From 3b2d4f89fceca3e0bb616e5f61828bad44c001e8 Mon Sep 17 00:00:00 2001 From: yml Date: Sat, 21 Nov 2026 21:58:31 +0800 Subject: [PATCH] =?UTF-8?q?feat(douyu):=20=E6=8E=A5=E5=85=A5=E9=87=8D?= =?UTF-8?q?=E7=99=BB=E5=B7=A5=E4=BD=9C=E9=98=9F=E5=88=97=E4=B8=8E=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=E7=8A=B6=E6=80=81=E8=A7=82=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/斗鱼/斗鱼账号平台最终实施设计.md | 8 +- tests/test_account_platform_foundation.py | 68 +++++++ tests/test_account_supply_service.py | 47 +++++ web/backend/routers/douyu.py | 10 + web/backend/services/account_lifecycle.py | 1 + web/backend/services/account_observability.py | 187 ++++++++++++++++++ web/backend/services/account_work_queue.py | 71 ++++++- .../services/automation/jobs/ck_relogin.py | 69 +++++++ 8 files changed, 456 insertions(+), 5 deletions(-) create mode 100644 web/backend/services/account_observability.py diff --git a/docs/斗鱼/斗鱼账号平台最终实施设计.md b/docs/斗鱼/斗鱼账号平台最终实施设计.md index f7fa857..64f9c1d 100644 --- a/docs/斗鱼/斗鱼账号平台最终实施设计.md +++ b/docs/斗鱼/斗鱼账号平台最终实施设计.md @@ -11,15 +11,15 @@ 更新时间:2026-11-21 -整体完成度约 **60%**。 +整体完成度约 **62%**。 | 阶段 | 完成度 | 已落地 | 主要缺口 | | --- | ---: | --- | --- | | 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 | | 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 | -| 阶段 2:账号与 CK | 80% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算 | 配置管理 API、更多自动重登灰度指标 | -| 阶段 3:商品与供给 | 55% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API | 默认供给目标、供给看板、覆盖率告警 | -| 阶段 4:安全恢复自动化 | 20% | account_sync_queue 与供给补货作业默认小批量开启 | 自动重登灰度参数、成功率熔断和动态预算 | +| 阶段 2:账号与 CK | 85% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队 | 更多自动重登灰度指标 | +| 阶段 3:商品与供给 | 60% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API、统一平台状态快照 | 默认供给目标、供给看板、阈值告警 | +| 阶段 4:安全恢复自动化 | 30% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列 | 自动重登灰度参数、成功率熔断和动态预算 | | 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 | ## 为什么重构 diff --git a/tests/test_account_platform_foundation.py b/tests/test_account_platform_foundation.py index 2f76826..567b11d 100644 --- a/tests/test_account_platform_foundation.py +++ b/tests/test_account_platform_foundation.py @@ -21,8 +21,11 @@ from web.backend.services.account_sync_service import ( sync_storehome_snapshot, ) from web.backend.services.account_work_queue import ( + claim_account_work, complete_account_work, enqueue_account_work, + fail_account_work_type, + reschedule_account_work_type, ) from web.backend.services.automation.jobs import ck_health from web.backend.services.douyu_account_state import DouyuLiveAccountState @@ -175,6 +178,71 @@ def test_account_work_queue_deduplicates_only_active_items(db): assert db.query(AccountWorkItem).count() == 2 +def test_account_work_claim_can_be_scoped_for_relogin(db): + first = _account("relogin-scope-a") + second = _account("relogin-scope-b") + db.add_all([first, second]) + db.commit() + for account in (first, second): + enqueue_account_work( + db, + account_id=account.id, + work_type="login_relogin", + max_attempts=20, + ) + db.commit() + + claimed = claim_account_work( + db, + work_types=("login_relogin",), + account_ids=(first.id,), + limit=10, + ) + + assert [item.account_id for item in claimed] == [first.id] + assert claimed[0].attempt_count == 1 + assert claimed[0].max_attempts == 20 + + +def test_relogin_work_reschedules_then_closes_for_manual(db): + account = _account("relogin-work-state") + db.add(account) + db.commit() + enqueue_account_work( + db, + account_id=account.id, + work_type="login_relogin", + max_attempts=20, + ) + db.commit() + item = claim_account_work( + db, + work_types=("login_relogin",), + account_ids=(account.id,), + limit=1, + )[0] + retry_at = datetime.now(UTC) + timedelta(hours=1) + + reschedule_account_work_type( + db, + account_id=account.id, + work_type="login_relogin", + due_at=retry_at, + reason="验证码失败", + ) + assert item.status == "pending" + assert item.due_at == retry_at.replace(microsecond=0) + + fail_account_work_type( + db, + account_id=account.id, + work_type="login_relogin", + reason="转人工", + ) + assert item.status == "failed" + assert item.last_error == "转人工" + + def test_ck_verdict_updates_lifecycle_and_moves_relogin_work(db): issued_at = datetime.now(UTC) - timedelta(days=7, hours=9) account = _account("ck-lifecycle", cookie_updated_at=issued_at) diff --git a/tests/test_account_supply_service.py b/tests/test_account_supply_service.py index 0d1e303..e9b58f3 100644 --- a/tests/test_account_supply_service.py +++ b/tests/test_account_supply_service.py @@ -9,17 +9,20 @@ from web.backend.models import ( AccountCommodityState, AccountLease, AccountSupplyTarget, + AccountSyncState, AccountWorkItem, User, ) from web.backend.routers.douyu import create_supply_target, list_supply_targets from web.backend.schemas import DouyuSupplyTargetUpsert +from web.backend.services.account_observability import account_platform_status from web.backend.services.account_supply_service import ( VALUE_HIGH, enqueue_supply_replenishment, evaluate_supply_target, upsert_supply_target, ) +from web.backend.services.account_work_queue import enqueue_account_work NOW = datetime.now(UTC).replace(microsecond=0) @@ -191,3 +194,47 @@ def test_supply_target_configuration_route_returns_inventory(db): assert created.covered_shortage == 12 assert listed[0].id == created.id assert listed[0].needs_replenishment is True + + +def test_account_platform_status_aggregates_queue_sync_and_supply(db): + account = _account("platform-status", points=400) + account.ck_expected_expires_at = NOW + timedelta(hours=12) + db.add(account) + db.flush() + db.add_all( + [ + _state(account), + AccountSyncState( + account_id=account.id, + platform="douyu", + domain="storehome", + status="success", + last_success_at=NOW, + valid_until=NOW + timedelta(hours=1), + ), + AccountSupplyTarget( + platform="douyu", + scope="elite", + commodity_id="C", + value_class=VALUE_HIGH, + target_covered=2, + target_available=2, + target_allocatable=1, + ), + ] + ) + enqueue_account_work( + db, + account_id=account.id, + work_type="login_relogin", + ) + db.commit() + + status = account_platform_status(db, now=NOW) + + assert status["accounts"]["ck_valid"] == 1 + assert status["accounts"]["ck_expiring_24h"] == 1 + assert status["work_queue"]["login_relogin"]["pending"] == 1 + assert status["sync_domains"]["storehome"]["fresh"] == 1 + assert status["commodities"]["fresh_available"] == 1 + assert status["supply_targets"][0]["covered_shortage"] == 1 diff --git a/web/backend/routers/douyu.py b/web/backend/routers/douyu.py index 0101d2a..b92aa3f 100644 --- a/web/backend/routers/douyu.py +++ b/web/backend/routers/douyu.py @@ -62,6 +62,7 @@ from ..schemas import ( DouyuXpdGoodsOut, ) from ..services.account_binding import bind_payload +from ..services.account_observability import account_platform_status from ..services.account_supply_service import ( evaluate_supply_target, update_supply_target, @@ -724,6 +725,15 @@ def task_types(current: User = Depends(require_permission("douyu:task"))): return SUPPORTED_DOUYU_TASK_TYPES +@router.get("/account-platform-status") +def get_account_platform_status( + db: Session = Depends(get_db), + current: User = Depends(require_permission("douyu:task")), +): + """账号同步、CK 周期、商品覆盖和供给缺口的统一观测快照。""" + return account_platform_status(db) + + @router.get("/accounts") def list_task_accounts( search: str = Query(""), diff --git a/web/backend/services/account_lifecycle.py b/web/backend/services/account_lifecycle.py index 7a972e5..c61c723 100644 --- a/web/backend/services/account_lifecycle.py +++ b/web/backend/services/account_lifecycle.py @@ -360,6 +360,7 @@ def on_ck_verdict( account_id=int(account.id), work_type=WORK_LOGIN_RELOGIN, priority=30, + max_attempts=20, reason="CK 已判定失效,等待受预算控制的重登", ) except Exception as exc: # noqa: BLE001 - 生命周期主流程优先 diff --git a/web/backend/services/account_observability.py b/web/backend/services/account_observability.py new file mode 100644 index 0000000..23db6e6 --- /dev/null +++ b/web/backend/services/account_observability.py @@ -0,0 +1,187 @@ +"""斗鱼账号平台运行状态与覆盖率快照。""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import case, func +from sqlalchemy.orm import Session + +from ..models import ( + Account, + AccountCommodityState, + AccountSyncState, + AccountWorkItem, +) +from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED +from .account_supply_service import evaluate_supply_targets + + +def account_platform_status( + db: Session, + *, + now: datetime | None = None, +) -> dict[str, Any]: + moment = now or datetime.now(UTC) + in_24h = moment + timedelta(hours=24) + in_72h = moment + timedelta(hours=72) + + account_counts = dict( + db.query(Account.ck_check_status, func.count(Account.id)) + .group_by(Account.ck_check_status) + .all() + ) + total_accounts = int(db.query(func.count(Account.id)).scalar() or 0) + cookie_accounts = int( + db.query(func.count(Account.id)) + .filter(Account.cookie.isnot(None), Account.cookie != "") + .scalar() + or 0 + ) + + expiring_24h = int( + db.query(func.count(Account.id)) + .filter( + Account.ck_check_status == "valid", + Account.ck_expected_expires_at.isnot(None), + Account.ck_expected_expires_at > moment, + Account.ck_expected_expires_at <= in_24h, + ) + .scalar() + or 0 + ) + expiring_72h = int( + db.query(func.count(Account.id)) + .filter( + Account.ck_check_status == "valid", + Account.ck_expected_expires_at.isnot(None), + Account.ck_expected_expires_at > moment, + Account.ck_expected_expires_at <= in_72h, + ) + .scalar() + or 0 + ) + predicted_expired = int( + db.query(func.count(Account.id)) + .filter( + Account.ck_check_status == "valid", + Account.ck_expected_expires_at.isnot(None), + Account.ck_expected_expires_at <= moment, + ) + .scalar() + or 0 + ) + + queue_rows = ( + db.query( + AccountWorkItem.work_type, + AccountWorkItem.status, + func.count(AccountWorkItem.id), + func.min(AccountWorkItem.due_at), + ) + .group_by(AccountWorkItem.work_type, AccountWorkItem.status) + .all() + ) + work_queue: dict[str, dict[str, Any]] = {} + for work_type, status, count, oldest_due in queue_rows: + bucket = work_queue.setdefault(str(work_type), {}) + bucket[str(status)] = int(count or 0) + if status == "pending": + bucket["oldest_pending_due_at"] = oldest_due + + sync_rows = ( + db.query( + AccountSyncState.domain, + AccountSyncState.status, + func.count(AccountSyncState.id), + ) + .group_by(AccountSyncState.domain, AccountSyncState.status) + .all() + ) + sync_domains: dict[str, dict[str, Any]] = {} + for domain, status, count in sync_rows: + bucket = sync_domains.setdefault(str(domain), {}) + bucket[str(status)] = int(count or 0) + fresh_rows = ( + db.query(AccountSyncState.domain, func.count(AccountSyncState.id)) + .filter( + AccountSyncState.status == "success", + AccountSyncState.valid_until.isnot(None), + AccountSyncState.valid_until > moment, + ) + .group_by(AccountSyncState.domain) + .all() + ) + for domain, count in fresh_rows: + sync_domains.setdefault(str(domain), {})["fresh"] = int(count or 0) + + commodity_rows = ( + db.query( + AccountCommodityState.availability, + func.count(AccountCommodityState.id), + func.sum( + case( + ( + AccountCommodityState.valid_until > moment, + 1, + ), + else_=0, + ) + ), + ) + .filter(AccountCommodityState.platform == "douyu") + .group_by(AccountCommodityState.availability) + .all() + ) + commodities = { + str(availability): { + "total": int(total or 0), + "fresh": int(fresh or 0), + } + for availability, total, fresh in commodity_rows + } + fresh_available = int(commodities.get(AVAILABILITY_AVAILABLE, {}).get("fresh", 0)) + fresh_limited = int(commodities.get(AVAILABILITY_LIMITED, {}).get("fresh", 0)) + + inventories = evaluate_supply_targets(db, now=moment) + return { + "checked_at": moment, + "accounts": { + "total": total_accounts, + "with_cookie": cookie_accounts, + "ck_valid": int(account_counts.get("valid", 0)), + "ck_invalid": int(account_counts.get("invalid", 0)), + "ck_error": int(account_counts.get("error", 0)), + "ck_unchecked": int(account_counts.get("", 0) or 0), + "ck_expiring_24h": expiring_24h, + "ck_expiring_72h": expiring_72h, + "ck_predicted_expired": predicted_expired, + }, + "work_queue": work_queue, + "sync_domains": sync_domains, + "commodities": { + "by_availability": commodities, + "fresh_available": fresh_available, + "fresh_limited": fresh_limited, + }, + "supply_targets": [ + { + "target_id": item.target_id, + "scope": item.scope, + "commodity_id": item.commodity_id, + "value_class": item.value_class, + "covered": item.covered, + "available": item.available, + "allocatable": item.allocatable, + "covered_shortage": item.covered_shortage, + "available_shortage": item.available_shortage, + "allocatable_shortage": item.allocatable_shortage, + "needs_replenishment": item.needs_replenishment, + } + for item in inventories + ], + } + + +__all__ = ["account_platform_status"] diff --git a/web/backend/services/account_work_queue.py b/web/backend/services/account_work_queue.py index b2b25eb..e430c11 100644 --- a/web/backend/services/account_work_queue.py +++ b/web/backend/services/account_work_queue.py @@ -34,6 +34,7 @@ def enqueue_account_work( ) if existing is not None: existing.priority = min(existing.priority, priority) + existing.max_attempts = max(existing.max_attempts, max(1, int(max_attempts))) if due_at is not None and due_at < existing.due_at: existing.due_at = due_at db.flush() @@ -69,15 +70,23 @@ def claim_account_work( work_types: tuple[str, ...], limit: int = 10, now: datetime | None = None, + account_ids: tuple[int, ...] | None = None, ) -> list[AccountWorkItem]: moment = now or datetime.now(UTC) - rows = ( + query = ( db.query(AccountWorkItem) .filter( AccountWorkItem.work_type.in_(work_types), AccountWorkItem.status == "pending", AccountWorkItem.due_at <= moment, ) + ) + if account_ids is not None: + if not account_ids: + return [] + query = query.filter(AccountWorkItem.account_id.in_(account_ids)) + rows = ( + query .order_by( AccountWorkItem.priority.asc(), AccountWorkItem.due_at.asc(), @@ -179,11 +188,71 @@ def recover_running_account_work( return len(items) +def reschedule_account_work_type( + db: Session, + *, + account_id: int, + work_type: str, + due_at: datetime, + reason: str = "", +) -> int: + """把自动重登失败的工作项按账号退避时间重新排队。""" + moment = (due_at if due_at.tzinfo else due_at.replace(tzinfo=UTC)).replace( + microsecond=0 + ) + items = ( + db.query(AccountWorkItem) + .filter( + AccountWorkItem.account_id == account_id, + AccountWorkItem.work_type == work_type, + AccountWorkItem.status.in_(("pending", "running")), + ) + .all() + ) + for item in items: + item.status = "pending" + item.due_at = moment + item.last_error = reason[:2000] + item.updated_at = datetime.now(UTC) + db.flush() + return len(items) + + +def fail_account_work_type( + db: Session, + *, + account_id: int, + work_type: str, + reason: str, + now: datetime | None = None, +) -> int: + """关闭不再自动处理的工作项,例如重登已转人工。""" + moment = now or datetime.now(UTC) + items = ( + db.query(AccountWorkItem) + .filter( + AccountWorkItem.account_id == account_id, + AccountWorkItem.work_type == work_type, + AccountWorkItem.status.in_(("pending", "running")), + ) + .all() + ) + for item in items: + item.status = "failed" + item.finished_at = moment + item.last_error = reason[:2000] + item.updated_at = moment + db.flush() + return len(items) + + __all__ = [ "claim_account_work", "complete_account_work", "complete_account_work_type", "enqueue_account_work", "fail_account_work", + "fail_account_work_type", "recover_running_account_work", + "reschedule_account_work_type", ] diff --git a/web/backend/services/automation/jobs/ck_relogin.py b/web/backend/services/automation/jobs/ck_relogin.py index a69a186..21e023d 100644 --- a/web/backend/services/automation/jobs/ck_relogin.py +++ b/web/backend/services/automation/jobs/ck_relogin.py @@ -27,10 +27,17 @@ from ...account_events_service import EVENT_RELOGIN_FAILED, emit from ...account_lifecycle import ( MANUAL_KINDS, MANUAL_STATES, + WORK_LOGIN_RELOGIN, on_login_success, sync_status, ) from ...account_value import value_tier_expr +from ...account_work_queue import ( + claim_account_work, + enqueue_account_work, + fail_account_work_type, + reschedule_account_work_type, +) from ...douyu_service import cookie_holding_filter, cookie_operable_filter from ...login_failure import classify_failure from ...login_service import LoginBatchRunner, get_relogin_limits @@ -235,6 +242,47 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: if not accounts: return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0} + selected_account_ids = tuple(int(account.id) for account in accounts) + for account in accounts: + enqueue_account_work( + db, + account_id=int(account.id), + work_type=WORK_LOGIN_RELOGIN, + priority=30, + max_attempts=20, + due_at=now, + reason="CK 失效,等待自动重登", + ) + db.commit() + work_items = claim_account_work( + db, + work_types=(WORK_LOGIN_RELOGIN,), + account_ids=selected_account_ids, + limit=len(selected_account_ids), + now=now, + ) + db.commit() + claimed_account_ids = [int(item.account_id) for item in work_items] + if not claimed_account_ids: + return { + "scanned": 0, + "valid": 0, + "invalid": 0, + "skipped": len(selected_account_ids), + "message": "候选账号的重登工作项已被其他执行器领取", + } + account_map = { + int(account.id): account + for account in db.query(Account) + .filter(Account.id.in_(claimed_account_ids)) + .all() + } + accounts = [ + account_map[account_id] + for account_id in claimed_account_ids + if account_id in account_map + ] + # 调度单位是**账号**(CK 在账号上);登录记录只是这次尝试的历史。 account_ids = [account.id for account in accounts] proxy = db.query(ProxyConfigModel).first() @@ -325,6 +373,13 @@ def settle_results( for account_id in account_ids: entry = fresh.get(account_id) if entry is None: + reschedule_account_work_type( + db, + account_id=int(account_id), + work_type=WORK_LOGIN_RELOGIN, + due_at=now + timedelta(hours=1), + reason="自动重登未产生账号任务,重新排队", + ) continue account, task = entry if task.attempt_status == "success": @@ -359,11 +414,25 @@ def settle_results( ) # 状态跟着落到 inactive 的分支(invalid 或 manual)——推导函数负责判定 sync_status(db, account, occurred_at=now) + fail_account_work_type( + db, + account_id=int(account.id), + work_type=WORK_LOGIN_RELOGIN, + reason=f"自动重登转人工({kind})", + now=now, + ) continue if kind == "email_quota": counters["email_quota"] += 1 counters["retry"] += 1 account.relogin_next_at = _next_retry_at(kind, account.relogin_attempts, now) + reschedule_account_work_type( + db, + account_id=int(account.id), + work_type=WORK_LOGIN_RELOGIN, + due_at=account.relogin_next_at, + reason=f"自动重登未成功({kind}),按账号退避时间重试", + ) # 排了下一次重试 → 仍是 invalid(不是 manual) sync_status(db, account, occurred_at=now) return counters