feat(douyu): 建立账号同步队列与供给库存

This commit is contained in:
yml
2026-11-21 21:48:51 +08:00
parent 1d262cdfd9
commit 3baf15afda
16 changed files with 1158 additions and 33 deletions
+200
View File
@@ -0,0 +1,200 @@
"""账号内部同步队列执行器。"""
from __future__ import annotations
from collections.abc import Callable
from concurrent.futures import as_completed
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.orm import Session
from ..database import WorkerSessionLocal
from ..models import Account, AccountWorkItem
from .account_sync_service import DOMAIN_STOREHOME, sync_storehome_snapshot
from .account_work_queue import (
claim_account_work,
complete_account_work,
fail_account_work,
)
from .douyu_activity_query import activity_client
from .douyu_read_gateway import StorehomeSnapshot, fetch_storehome_snapshot
from .douyu_service import ensure_douyu_config
from .task_runtime import TaskLane, available_slots, submit
WORK_STOREHOME_SYNC = DOMAIN_STOREHOME + "_sync"
ACCOUNT_SYNC_WORK_TYPES = (WORK_STOREHOME_SYNC,)
def _execute_storehome_sync(
db: Session,
*,
account: Account,
item: AccountWorkItem,
client_factory: Callable[[str], Any],
snapshot_fetcher: Callable[..., StorehomeSnapshot],
) -> dict[str, Any]:
if not account.cookie:
raise RuntimeError("账号没有可用 CK,不能同步商品能力")
if account.ck_check_status in {"invalid", "manual"}:
raise RuntimeError(f"账号 CK 状态为 {account.ck_check_status},暂不能同步商品能力")
config = ensure_douyu_config(db)
manual_id = str(getattr(config, "manual_id", "") or "")
rid = str(getattr(config, "rid", "") or "")
if not manual_id:
raise RuntimeError("未配置精英宝典 manualID")
client = client_factory(account.cookie)
snapshot = snapshot_fetcher(
client,
manual_id=manual_id,
rid=rid,
)
result = sync_storehome_snapshot(
db,
account,
snapshot,
source="account_work.storehome_sync",
)
return {
"ok": True,
"work_id": int(item.id),
"account_id": int(account.id),
"item_count": result.item_count,
"changed_count": result.changed_count,
}
def execute_account_work_item(
work_id: int,
*,
session_factory: Callable[[], Session] = WorkerSessionLocal,
client_factory: Callable[[str], Any] = activity_client,
snapshot_fetcher: Callable[..., StorehomeSnapshot] = fetch_storehome_snapshot,
) -> dict[str, Any]:
"""在一个短事务中执行单条账号维护工作。"""
session = session_factory()
item: AccountWorkItem | None = None
try:
item = session.get(AccountWorkItem, int(work_id))
if item is None or item.status != "running":
return {"ok": False, "work_id": int(work_id), "skipped": True}
account = session.get(Account, int(item.account_id))
if account is None:
raise RuntimeError("工作项对应账号不存在")
if item.work_type != WORK_STOREHOME_SYNC:
raise RuntimeError(f"不支持的工作类型: {item.work_type}")
result = _execute_storehome_sync(
session,
account=account,
item=item,
client_factory=client_factory,
snapshot_fetcher=snapshot_fetcher,
)
complete_account_work(session, item)
session.commit()
return result
except Exception as exc: # noqa: BLE001 - 单工作项失败必须收敛到队列状态
if not session.is_active:
session.rollback()
item = session.get(AccountWorkItem, int(work_id))
if item is not None:
fail_account_work(
session,
item,
error=exc,
retry_after_seconds=15 * 60,
)
session.commit()
return {
"ok": False,
"work_id": int(work_id),
"error": f"{type(exc).__name__}: {exc}"[:500],
}
finally:
session.close()
def run_account_sync_queue(
db: Session,
*,
batch_size: int = 20,
now: datetime | None = None,
session_factory: Callable[[], Session] = WorkerSessionLocal,
client_factory: Callable[[str], Any] = activity_client,
snapshot_fetcher: Callable[..., StorehomeSnapshot] = fetch_storehome_snapshot,
) -> dict[str, Any]:
"""领取少量到期工作并通过独立账号同步槽执行。"""
moment = now or datetime.now(UTC)
capacity = available_slots(TaskLane.ACCOUNT_SYNC)
if capacity <= 0:
return {
"scanned": 0,
"success": 0,
"failed": 0,
"message": "账号同步槽已满,本轮跳过",
}
limit = max(1, min(int(batch_size), capacity))
items = claim_account_work(
db,
work_types=ACCOUNT_SYNC_WORK_TYPES,
limit=limit,
now=moment,
)
db.commit()
if not items:
return {
"scanned": 0,
"success": 0,
"failed": 0,
"message": "暂无到期的账号同步工作",
}
futures = {}
for item in items:
work_id = int(item.id)
future = submit(
TaskLane.ACCOUNT_SYNC,
execute_account_work_item,
work_id,
session_factory=session_factory,
client_factory=client_factory,
snapshot_fetcher=snapshot_fetcher,
)
if future is not None:
futures[future] = work_id
success = 0
failed = 0
skipped = 0
for future in as_completed(futures):
try:
result = future.result()
except Exception: # noqa: BLE001 - worker 异常已由执行函数收敛,这里只计数
failed += 1
continue
if result.get("ok"):
success += 1
elif result.get("skipped"):
skipped += 1
else:
failed += 1
return {
"scanned": len(futures),
"success": success,
"failed": failed,
"skipped": skipped,
"message": (
f"账号同步完成 {success} 条"
+ (f",失败 {failed} 条" if failed else "")
+ (f",跳过 {skipped} 条" if skipped else "")
),
}
__all__ = [
"ACCOUNT_SYNC_WORK_TYPES",
"WORK_STOREHOME_SYNC",
"execute_account_work_item",
"run_account_sync_queue",
]