yml2213
df8c63b829
协作台账：新表 huya_external_orders 替代 Excel 多人协同表（设计见
docs/设计-虎牙外部CK协作台账.md），一条 CK 一行，所有人共享、改动留痕。
- 前端 pages/HuyaCollabPage.tsx + 菜单「虎牙 → 外部单协作」/huya/collab：
内联编辑（带乐观锁 version）、图片缩略图、详情抽屉（字段表单 + 两个图片槽位 +
操作时间线）、粘贴导入弹窗（带 TSV 预览）、批量/单条验证 CK、页头统计条、筛选
- 两套状态机：业务 pending→bound→recharged→scripted→done（旁路 void）、
凭据 unverified/valid/invalid/disabled；唯一写入口在 services/huya_collab_service.py，
字段级审计 + 整数 version 乐观锁（不用 updated_at：DATETIME 只到秒）
- CK 只读探针 check_user_bind_game_account（真机判据：响应非 None 且 status == 200），
成功即覆盖写绑定快照 bind_game_account/bind_game_role/bind_game_channel（迁移 0056），
失败只改凭据状态、不动业务状态；人工只能停用/恢复（POST /{id}/ck/status）
- 贴/换/重建 CK、TSV 批量导入（SAVEPOINT 单行隔离，部分失败不整体失败）、
权限 huya:collab:view/edit/manage（删除只给运营）
- 孤儿图清理脚本 scripts/cleanup_orphan_file_assets.py（默认 --dry-run）
图片上传（协作台账截图用）：OSS/本地双后端 + 缩略图变体 + /api/files 代理回读，
权限 file:upload / file:view。
验证：pytest 880 passed / 6 skipped；ruff format 全绿、ruff check 仅剩既有告警；
前端 tsc -b / eslint / vite build 通过；接口端到端冒烟 29 项全绿（TestClient，探针用假客户端）；
探针真机冒烟 3 例（有效 CK=True、别人 uid=False、裸 cookie=False）。
379 lines
13 KiB
Python
379 lines
13 KiB
Python
"""上传服务：校验 → 生成对象 key → 落对象存储 → 生成缩略图变体 → 落台账。
|
||
|
||
流程与两个参考实现对齐（order_site 的 ``file-storage-service.ts``、
|
||
hfb_sys 的 ``modules/file/service.go``）：
|
||
|
||
1. 内存里收下整个文件（上限 10MB），先判大小再**按魔数嗅探真实类型**（不信
|
||
Content-Type）；
|
||
2. ``objectKey = scene/yyyy/mm/dd/<24位随机hex><ext>`` —— 随机段保证对象不可枚举；
|
||
3. 原图入库后再生成长边 480/1280 的 ``thumb`` / ``medium`` 变体，**变体失败不阻断
|
||
上传**（回读接口会自动回退原图）；
|
||
4. 任何一步失败都把已经写进对象存储的 key 全部删掉，避免留下孤儿对象；
|
||
5. 元数据落 ``file_assets`` 台账，便于溯源与后续清理。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import io
|
||
import re
|
||
import secrets
|
||
from dataclasses import dataclass
|
||
from datetime import UTC, datetime
|
||
from typing import Any
|
||
from urllib.parse import quote
|
||
|
||
from loguru import logger
|
||
from PIL import Image, ImageOps, features
|
||
from sqlalchemy.orm import Session
|
||
|
||
from ..models import FileAsset
|
||
from .file_storage import (
|
||
ObjectStorage,
|
||
StorageSettings,
|
||
StoredObject,
|
||
get_object_storage,
|
||
max_upload_size_bytes,
|
||
normalize_object_key,
|
||
)
|
||
|
||
ALLOWED_CONTENT_TYPES = frozenset(
|
||
{"image/jpeg", "image/png", "image/webp", "application/pdf"}
|
||
)
|
||
IMAGE_CONTENT_TYPES = frozenset({"image/jpeg", "image/png", "image/webp"})
|
||
|
||
VARIANT_THUMB = "thumb"
|
||
VARIANT_MEDIUM = "medium"
|
||
_VARIANT_NAMES = (VARIANT_THUMB, VARIANT_MEDIUM)
|
||
_VARIANT_MAX_SIDE = {VARIANT_THUMB: 480, VARIANT_MEDIUM: 1280}
|
||
_VARIANT_QUALITY = {VARIANT_THUMB: 76, VARIANT_MEDIUM: 82}
|
||
|
||
_EXTENSION_BY_CONTENT_TYPE = {
|
||
"image/jpeg": ".jpg",
|
||
"image/png": ".png",
|
||
"image/webp": ".webp",
|
||
"application/pdf": ".pdf",
|
||
}
|
||
|
||
_SCENE_PATTERN = re.compile(r"[^a-z0-9_-]+")
|
||
_SAFE_EXT_PATTERN = re.compile(r"^\.[a-z0-9]{1,8}$")
|
||
_MAX_SCENE_LENGTH = 32
|
||
|
||
|
||
class FileUploadError(RuntimeError):
|
||
"""上传/读取的业务错误，路由层据此转成对应 HTTP 状态码。"""
|
||
|
||
def __init__(self, detail: str, *, status_code: int = 400, code: str = ""):
|
||
super().__init__(detail)
|
||
self.detail = detail
|
||
self.status_code = status_code
|
||
self.code = code
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ImageVariant:
|
||
"""一张缩略图变体（内容保存在内存里，随原图一起写入对象存储）。"""
|
||
|
||
name: str
|
||
content: bytes
|
||
content_type: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class UploadedFile:
|
||
"""上传结果，同时作为接口响应体。"""
|
||
|
||
object_key: str
|
||
url: str
|
||
thumbnail_url: str
|
||
medium_url: str
|
||
filename: str
|
||
content_type: str
|
||
size: int
|
||
|
||
|
||
def normalize_scene(raw_scene: Any) -> str:
|
||
"""scene 归一化：只留小写字母/数字/下划线/连字符，缺省 misc。"""
|
||
scene = _SCENE_PATTERN.sub("-", str(raw_scene or "").strip().lower()).strip("-")
|
||
return (scene or "misc")[:_MAX_SCENE_LENGTH]
|
||
|
||
|
||
def sniff_content_type(data: bytes) -> str:
|
||
"""按文件头魔数判断类型；识别不出返回空串。"""
|
||
if data[:3] == b"\xff\xd8\xff":
|
||
return "image/jpeg"
|
||
if data[:8] == b"\x89PNG\r\n\x1a\n":
|
||
return "image/png"
|
||
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||
return "image/webp"
|
||
if data[:4] == b"%PDF":
|
||
return "application/pdf"
|
||
return ""
|
||
|
||
|
||
def resolve_content_type(hint: Any, data: bytes) -> str:
|
||
"""先信魔数，再信调用方给的 Content-Type。"""
|
||
detected = sniff_content_type(data)
|
||
if detected:
|
||
return detected
|
||
normalized = str(hint or "").strip().lower().split(";")[0]
|
||
return normalized or "application/octet-stream"
|
||
|
||
|
||
def build_object_key(scene: str, content_type: str, filename: str = "") -> str:
|
||
"""``scene/yyyy/mm/dd/<24位随机hex><ext>``。"""
|
||
now = datetime.now(UTC)
|
||
token = secrets.token_hex(12)
|
||
return f"{scene}/{now.year:04d}/{now.month:02d}/{now.day:02d}/{token}{_pick_extension(content_type, filename)}"
|
||
|
||
|
||
def image_variant_key(object_key: str, variant: str, extension: str) -> str:
|
||
"""变体 key：在扩展名前插入 ``.<variant>``。"""
|
||
base = object_key
|
||
for known in _EXTENSION_BY_CONTENT_TYPE.values():
|
||
if object_key.lower().endswith(known):
|
||
base = object_key[: -len(known)]
|
||
break
|
||
return f"{base}.{variant}{extension}"
|
||
|
||
|
||
def image_variant_fallback_keys(object_key: str, variant: str) -> list[str]:
|
||
"""变体 key 的候选顺序：webp → jpg → png → 原图。
|
||
|
||
变体的扩展名取决于源图是否有透明通道（png/webp 保 alpha 出 webp，
|
||
其余出 jpg），而回读接口只知道基础 key，所以这里按候选逐个试。
|
||
"""
|
||
if variant not in _VARIANT_NAMES:
|
||
return [object_key]
|
||
candidates = [
|
||
image_variant_key(object_key, variant, ext) for ext in (".webp", ".jpg", ".png")
|
||
]
|
||
candidates.append(object_key)
|
||
return list(dict.fromkeys(candidates))
|
||
|
||
|
||
def build_file_url(object_key: str, variant: str = "") -> str:
|
||
"""文件访问地址（走后端代理，不暴露对象存储直链）。"""
|
||
url = f"/api/files/object?key={quote(normalize_object_key(object_key), safe='')}"
|
||
if variant:
|
||
url = f"{url}&variant={quote(variant, safe='')}"
|
||
return url
|
||
|
||
|
||
def _webp_available() -> bool:
|
||
return bool(features.check("webp"))
|
||
|
||
|
||
def _pick_extension(content_type: str, filename: str) -> str:
|
||
known = _EXTENSION_BY_CONTENT_TYPE.get(content_type)
|
||
if known:
|
||
return known
|
||
candidate = ""
|
||
if "." in filename:
|
||
candidate = filename[filename.rfind(".") :].lower()
|
||
return candidate if _SAFE_EXT_PATTERN.match(candidate) else ".bin"
|
||
|
||
|
||
def _load_image(data: bytes) -> Image.Image | None:
|
||
try:
|
||
image = Image.open(io.BytesIO(data))
|
||
image.load()
|
||
except Exception: # noqa: BLE001 坏图不该让整次上传失败
|
||
return None
|
||
try:
|
||
# 手机截图常带 EXIF 方向，先摆正再缩放（对齐 sharp 的 .rotate()）。
|
||
return ImageOps.exif_transpose(image)
|
||
except Exception: # noqa: BLE001
|
||
return image
|
||
|
||
|
||
def _encode_variant(
|
||
image: Image.Image, *, preserve_alpha: bool, quality: int
|
||
) -> tuple[bytes, str]:
|
||
buffer = io.BytesIO()
|
||
if preserve_alpha:
|
||
prepared = image.convert("RGBA")
|
||
if _webp_available():
|
||
prepared.save(buffer, "WEBP", quality=quality, method=4)
|
||
return buffer.getvalue(), "image/webp"
|
||
prepared.save(buffer, "PNG", optimize=True)
|
||
return buffer.getvalue(), "image/png"
|
||
# JPEG 没有透明通道；源图也只会是无 alpha 的 JPEG，直接转 RGB 即可。
|
||
image.convert("RGB").save(buffer, "JPEG", quality=quality, optimize=True)
|
||
return buffer.getvalue(), "image/jpeg"
|
||
|
||
|
||
def generate_image_variants(data: bytes, content_type: str) -> list[ImageVariant]:
|
||
"""生成 thumb/medium 变体；不是图片或解码失败时返回空列表。"""
|
||
if content_type not in IMAGE_CONTENT_TYPES:
|
||
return []
|
||
source = _load_image(data)
|
||
if source is None:
|
||
logger.warning(f"图片解码失败，跳过变体生成 content_type={content_type}")
|
||
return []
|
||
|
||
# 与两个参考实现一致：png/webp 视为可能带透明通道，保留 alpha 输出 webp。
|
||
preserve_alpha = content_type in ("image/png", "image/webp")
|
||
variants: list[ImageVariant] = []
|
||
try:
|
||
for name in _VARIANT_NAMES:
|
||
max_side = _VARIANT_MAX_SIDE[name]
|
||
try:
|
||
frame = source.copy()
|
||
frame.thumbnail((max_side, max_side), Image.Resampling.LANCZOS)
|
||
content, variant_type = _encode_variant(
|
||
frame, preserve_alpha=preserve_alpha, quality=_VARIANT_QUALITY[name]
|
||
)
|
||
frame.close()
|
||
except Exception as exc: # noqa: BLE001 单张变体失败不影响其余变体
|
||
logger.warning(f"生成图片变体失败 variant={name}: {exc}")
|
||
continue
|
||
if content:
|
||
variants.append(
|
||
ImageVariant(name=name, content=content, content_type=variant_type)
|
||
)
|
||
finally:
|
||
source.close()
|
||
return variants
|
||
|
||
|
||
def upload_file_asset(
|
||
db: Session,
|
||
*,
|
||
data: bytes,
|
||
filename: str = "",
|
||
content_type: str = "",
|
||
scene: str = "misc",
|
||
uploader_type: str = "user",
|
||
uploader_id: str = "",
|
||
settings: StorageSettings | None = None,
|
||
) -> UploadedFile:
|
||
"""校验并写入一份上传文件。
|
||
|
||
``settings`` 显式传入时可覆盖环境配置（测试用非默认本地目录）。
|
||
"""
|
||
if not data:
|
||
raise FileUploadError(
|
||
"请选择要上传的文件", status_code=400, code="file_required"
|
||
)
|
||
|
||
limit = max_upload_size_bytes(settings)
|
||
if len(data) > limit:
|
||
raise FileUploadError(
|
||
f"文件不能超过 {limit // (1024 * 1024)}MB",
|
||
status_code=400,
|
||
code="file_too_large",
|
||
)
|
||
|
||
resolved_type = resolve_content_type(content_type, data)
|
||
if resolved_type not in ALLOWED_CONTENT_TYPES:
|
||
raise FileUploadError(
|
||
"文件不符合规则，仅支持 JPG、PNG、WebP 或 PDF",
|
||
status_code=400,
|
||
code="file_type_invalid",
|
||
)
|
||
|
||
resolved_scene = normalize_scene(scene)
|
||
object_key = build_object_key(resolved_scene, resolved_type, filename)
|
||
storage: ObjectStorage = get_object_storage(settings)
|
||
|
||
uploaded_keys: list[str] = []
|
||
try:
|
||
uploaded_keys.append(object_key)
|
||
storage.put_object(
|
||
key=object_key,
|
||
data=data,
|
||
content_type=resolved_type,
|
||
metadata={"original-filename": filename, "scene": resolved_scene},
|
||
)
|
||
|
||
thumbnail_key = ""
|
||
medium_key = ""
|
||
for variant in generate_image_variants(data, resolved_type):
|
||
variant_key = image_variant_key(
|
||
object_key, variant.name, _pick_extension(variant.content_type, "")
|
||
)
|
||
try:
|
||
storage.put_object(
|
||
key=variant_key,
|
||
data=variant.content,
|
||
content_type=variant.content_type,
|
||
metadata={"source-object": object_key},
|
||
)
|
||
except Exception as exc: # noqa: BLE001 变体失败不阻断上传
|
||
logger.warning(f"变体写入失败 key={variant_key}: {exc}")
|
||
continue
|
||
uploaded_keys.append(variant_key)
|
||
if variant.name == VARIANT_THUMB:
|
||
thumbnail_key = variant_key
|
||
elif variant.name == VARIANT_MEDIUM:
|
||
medium_key = variant_key
|
||
|
||
row = FileAsset(
|
||
object_key=object_key,
|
||
scene=resolved_scene,
|
||
original_filename=filename[:255],
|
||
content_type=resolved_type,
|
||
size_bytes=len(data),
|
||
thumbnail_object_key=thumbnail_key,
|
||
medium_object_key=medium_key,
|
||
uploader_type=str(uploader_type or "")[:32],
|
||
uploader_id=str(uploader_id or "")[:64],
|
||
status="active",
|
||
metadata_json={
|
||
"originalContentType": content_type or "",
|
||
"scene": resolved_scene,
|
||
},
|
||
)
|
||
db.add(row)
|
||
db.commit()
|
||
except FileUploadError:
|
||
raise
|
||
except Exception as exc:
|
||
db.rollback()
|
||
_cleanup_objects(storage, uploaded_keys)
|
||
logger.error(f"文件上传失败，已回滚对象: {exc}")
|
||
raise
|
||
|
||
return UploadedFile(
|
||
object_key=object_key,
|
||
url=build_file_url(object_key),
|
||
thumbnail_url=build_file_url(thumbnail_key or object_key),
|
||
medium_url=build_file_url(medium_key or object_key),
|
||
filename=filename,
|
||
content_type=resolved_type,
|
||
size=len(data),
|
||
)
|
||
|
||
|
||
def get_stored_object(object_key: Any, variant: Any = "") -> StoredObject:
|
||
"""读取对象（变体缺失自动回退原图）。"""
|
||
try:
|
||
key = normalize_object_key(object_key)
|
||
except ValueError as exc:
|
||
raise FileUploadError(
|
||
"文件 key 不正确", status_code=400, code="file_key_invalid"
|
||
) from exc
|
||
|
||
variant_name = str(variant or "").strip().lower()
|
||
candidates = (
|
||
image_variant_fallback_keys(key, variant_name) if variant_name else [key]
|
||
)
|
||
storage = get_object_storage()
|
||
for candidate in candidates:
|
||
try:
|
||
return storage.get_object(candidate)
|
||
except FileNotFoundError:
|
||
continue
|
||
raise FileUploadError(
|
||
"文件不存在或暂不可访问", status_code=404, code="file_not_found"
|
||
)
|
||
|
||
|
||
def _cleanup_objects(storage: ObjectStorage, keys: list[str]) -> None:
|
||
"""补偿删除：删不干净只告警，不再抛异常掩盖原始错误。"""
|
||
for key in keys:
|
||
try:
|
||
storage.delete_object(key)
|
||
except Exception as exc: # noqa: BLE001
|
||
logger.warning(f"上传失败后的对象清理未成功 key={key}: {exc}")
|