feat(douyu): 接入重登工作队列与平台状态观测

This commit is contained in:
yml
2026-11-21 21:58:31 +08:00
parent e178a393e4
commit 3b2d4f89fc
8 changed files with 456 additions and 5 deletions
@@ -11,15 +11,15 @@
更新时间:2026-11-21
整体完成度约 **60%**。
整体完成度约 **62%**。
| 阶段 | 完成度 | 已落地 | 主要缺口 |
| --- | ---: | --- | --- |
| 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 |
| 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 |
| 阶段 2:账号与 CK | 80% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算 | 配置管理 API、更多自动重登灰度指标 |
| 阶段 3:商品与供给 | 55% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API | 默认供给目标、供给看板、覆盖率告警 |
| 阶段 4:安全恢复自动化 | 20% | account_sync_queue 与供给补货作业默认小批量开启 | 自动重登灰度参数、成功率熔断和动态预算 |
| 阶段 2:账号与 CK | 85% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队 | 更多自动重登灰度指标 |
| 阶段 3:商品与供给 | 60% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API、统一平台状态快照 | 默认供给目标、供给看板、阈值告警 |
| 阶段 4:安全恢复自动化 | 30% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列 | 自动重登灰度参数、成功率熔断和动态预算 |
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
## 为什么重构
+68
View File
@@ -21,8 +21,11 @@ from web.backend.services.account_sync_service import (
sync_storehome_snapshot,
)
from web.backend.services.account_work_queue import (
claim_account_work,
complete_account_work,
enqueue_account_work,
fail_account_work_type,
reschedule_account_work_type,
)
from web.backend.services.automation.jobs import ck_health
from web.backend.services.douyu_account_state import DouyuLiveAccountState
@@ -175,6 +178,71 @@ def test_account_work_queue_deduplicates_only_active_items(db):
assert db.query(AccountWorkItem).count() == 2
def test_account_work_claim_can_be_scoped_for_relogin(db):
first = _account("relogin-scope-a")
second = _account("relogin-scope-b")
db.add_all([first, second])
db.commit()
for account in (first, second):
enqueue_account_work(
db,
account_id=account.id,
work_type="login_relogin",
max_attempts=20,
)
db.commit()
claimed = claim_account_work(
db,
work_types=("login_relogin",),
account_ids=(first.id,),
limit=10,
)
assert [item.account_id for item in claimed] == [first.id]
assert claimed[0].attempt_count == 1
assert claimed[0].max_attempts == 20
def test_relogin_work_reschedules_then_closes_for_manual(db):
account = _account("relogin-work-state")
db.add(account)
db.commit()
enqueue_account_work(
db,
account_id=account.id,
work_type="login_relogin",
max_attempts=20,
)
db.commit()
item = claim_account_work(
db,
work_types=("login_relogin",),
account_ids=(account.id,),
limit=1,
)[0]
retry_at = datetime.now(UTC) + timedelta(hours=1)
reschedule_account_work_type(
db,
account_id=account.id,
work_type="login_relogin",
due_at=retry_at,
reason="验证码失败",
)
assert item.status == "pending"
assert item.due_at == retry_at.replace(microsecond=0)
fail_account_work_type(
db,
account_id=account.id,
work_type="login_relogin",
reason="转人工",
)
assert item.status == "failed"
assert item.last_error == "转人工"
def test_ck_verdict_updates_lifecycle_and_moves_relogin_work(db):
issued_at = datetime.now(UTC) - timedelta(days=7, hours=9)
account = _account("ck-lifecycle", cookie_updated_at=issued_at)
+47
View File
@@ -9,17 +9,20 @@ from web.backend.models import (
AccountCommodityState,
AccountLease,
AccountSupplyTarget,
AccountSyncState,
AccountWorkItem,
User,
)
from web.backend.routers.douyu import create_supply_target, list_supply_targets
from web.backend.schemas import DouyuSupplyTargetUpsert
from web.backend.services.account_observability import account_platform_status
from web.backend.services.account_supply_service import (
VALUE_HIGH,
enqueue_supply_replenishment,
evaluate_supply_target,
upsert_supply_target,
)
from web.backend.services.account_work_queue import enqueue_account_work
NOW = datetime.now(UTC).replace(microsecond=0)
@@ -191,3 +194,47 @@ def test_supply_target_configuration_route_returns_inventory(db):
assert created.covered_shortage == 12
assert listed[0].id == created.id
assert listed[0].needs_replenishment is True
def test_account_platform_status_aggregates_queue_sync_and_supply(db):
account = _account("platform-status", points=400)
account.ck_expected_expires_at = NOW + timedelta(hours=12)
db.add(account)
db.flush()
db.add_all(
[
_state(account),
AccountSyncState(
account_id=account.id,
platform="douyu",
domain="storehome",
status="success",
last_success_at=NOW,
valid_until=NOW + timedelta(hours=1),
),
AccountSupplyTarget(
platform="douyu",
scope="elite",
commodity_id="C",
value_class=VALUE_HIGH,
target_covered=2,
target_available=2,
target_allocatable=1,
),
]
)
enqueue_account_work(
db,
account_id=account.id,
work_type="login_relogin",
)
db.commit()
status = account_platform_status(db, now=NOW)
assert status["accounts"]["ck_valid"] == 1
assert status["accounts"]["ck_expiring_24h"] == 1
assert status["work_queue"]["login_relogin"]["pending"] == 1
assert status["sync_domains"]["storehome"]["fresh"] == 1
assert status["commodities"]["fresh_available"] == 1
assert status["supply_targets"][0]["covered_shortage"] == 1
+10
View File
@@ -62,6 +62,7 @@ from ..schemas import (
DouyuXpdGoodsOut,
)
from ..services.account_binding import bind_payload
from ..services.account_observability import account_platform_status
from ..services.account_supply_service import (
evaluate_supply_target,
update_supply_target,
@@ -724,6 +725,15 @@ def task_types(current: User = Depends(require_permission("douyu:task"))):
return SUPPORTED_DOUYU_TASK_TYPES
@router.get("/account-platform-status")
def get_account_platform_status(
db: Session = Depends(get_db),
current: User = Depends(require_permission("douyu:task")),
):
"""账号同步、CK 周期、商品覆盖和供给缺口的统一观测快照。"""
return account_platform_status(db)
@router.get("/accounts")
def list_task_accounts(
search: str = Query(""),
@@ -360,6 +360,7 @@ def on_ck_verdict(
account_id=int(account.id),
work_type=WORK_LOGIN_RELOGIN,
priority=30,
max_attempts=20,
reason="CK 已判定失效,等待受预算控制的重登",
)
except Exception as exc: # noqa: BLE001 - 生命周期主流程优先
@@ -0,0 +1,187 @@
"""斗鱼账号平台运行状态与覆盖率快照。"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import case, func
from sqlalchemy.orm import Session
from ..models import (
Account,
AccountCommodityState,
AccountSyncState,
AccountWorkItem,
)
from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED
from .account_supply_service import evaluate_supply_targets
def account_platform_status(
db: Session,
*,
now: datetime | None = None,
) -> dict[str, Any]:
moment = now or datetime.now(UTC)
in_24h = moment + timedelta(hours=24)
in_72h = moment + timedelta(hours=72)
account_counts = dict(
db.query(Account.ck_check_status, func.count(Account.id))
.group_by(Account.ck_check_status)
.all()
)
total_accounts = int(db.query(func.count(Account.id)).scalar() or 0)
cookie_accounts = int(
db.query(func.count(Account.id))
.filter(Account.cookie.isnot(None), Account.cookie != "")
.scalar()
or 0
)
expiring_24h = int(
db.query(func.count(Account.id))
.filter(
Account.ck_check_status == "valid",
Account.ck_expected_expires_at.isnot(None),
Account.ck_expected_expires_at > moment,
Account.ck_expected_expires_at <= in_24h,
)
.scalar()
or 0
)
expiring_72h = int(
db.query(func.count(Account.id))
.filter(
Account.ck_check_status == "valid",
Account.ck_expected_expires_at.isnot(None),
Account.ck_expected_expires_at > moment,
Account.ck_expected_expires_at <= in_72h,
)
.scalar()
or 0
)
predicted_expired = int(
db.query(func.count(Account.id))
.filter(
Account.ck_check_status == "valid",
Account.ck_expected_expires_at.isnot(None),
Account.ck_expected_expires_at <= moment,
)
.scalar()
or 0
)
queue_rows = (
db.query(
AccountWorkItem.work_type,
AccountWorkItem.status,
func.count(AccountWorkItem.id),
func.min(AccountWorkItem.due_at),
)
.group_by(AccountWorkItem.work_type, AccountWorkItem.status)
.all()
)
work_queue: dict[str, dict[str, Any]] = {}
for work_type, status, count, oldest_due in queue_rows:
bucket = work_queue.setdefault(str(work_type), {})
bucket[str(status)] = int(count or 0)
if status == "pending":
bucket["oldest_pending_due_at"] = oldest_due
sync_rows = (
db.query(
AccountSyncState.domain,
AccountSyncState.status,
func.count(AccountSyncState.id),
)
.group_by(AccountSyncState.domain, AccountSyncState.status)
.all()
)
sync_domains: dict[str, dict[str, Any]] = {}
for domain, status, count in sync_rows:
bucket = sync_domains.setdefault(str(domain), {})
bucket[str(status)] = int(count or 0)
fresh_rows = (
db.query(AccountSyncState.domain, func.count(AccountSyncState.id))
.filter(
AccountSyncState.status == "success",
AccountSyncState.valid_until.isnot(None),
AccountSyncState.valid_until > moment,
)
.group_by(AccountSyncState.domain)
.all()
)
for domain, count in fresh_rows:
sync_domains.setdefault(str(domain), {})["fresh"] = int(count or 0)
commodity_rows = (
db.query(
AccountCommodityState.availability,
func.count(AccountCommodityState.id),
func.sum(
case(
(
AccountCommodityState.valid_until > moment,
1,
),
else_=0,
)
),
)
.filter(AccountCommodityState.platform == "douyu")
.group_by(AccountCommodityState.availability)
.all()
)
commodities = {
str(availability): {
"total": int(total or 0),
"fresh": int(fresh or 0),
}
for availability, total, fresh in commodity_rows
}
fresh_available = int(commodities.get(AVAILABILITY_AVAILABLE, {}).get("fresh", 0))
fresh_limited = int(commodities.get(AVAILABILITY_LIMITED, {}).get("fresh", 0))
inventories = evaluate_supply_targets(db, now=moment)
return {
"checked_at": moment,
"accounts": {
"total": total_accounts,
"with_cookie": cookie_accounts,
"ck_valid": int(account_counts.get("valid", 0)),
"ck_invalid": int(account_counts.get("invalid", 0)),
"ck_error": int(account_counts.get("error", 0)),
"ck_unchecked": int(account_counts.get("", 0) or 0),
"ck_expiring_24h": expiring_24h,
"ck_expiring_72h": expiring_72h,
"ck_predicted_expired": predicted_expired,
},
"work_queue": work_queue,
"sync_domains": sync_domains,
"commodities": {
"by_availability": commodities,
"fresh_available": fresh_available,
"fresh_limited": fresh_limited,
},
"supply_targets": [
{
"target_id": item.target_id,
"scope": item.scope,
"commodity_id": item.commodity_id,
"value_class": item.value_class,
"covered": item.covered,
"available": item.available,
"allocatable": item.allocatable,
"covered_shortage": item.covered_shortage,
"available_shortage": item.available_shortage,
"allocatable_shortage": item.allocatable_shortage,
"needs_replenishment": item.needs_replenishment,
}
for item in inventories
],
}
__all__ = ["account_platform_status"]
+70 -1
View File
@@ -34,6 +34,7 @@ def enqueue_account_work(
)
if existing is not None:
existing.priority = min(existing.priority, priority)
existing.max_attempts = max(existing.max_attempts, max(1, int(max_attempts)))
if due_at is not None and due_at < existing.due_at:
existing.due_at = due_at
db.flush()
@@ -69,15 +70,23 @@ def claim_account_work(
work_types: tuple[str, ...],
limit: int = 10,
now: datetime | None = None,
account_ids: tuple[int, ...] | None = None,
) -> list[AccountWorkItem]:
moment = now or datetime.now(UTC)
rows = (
query = (
db.query(AccountWorkItem)
.filter(
AccountWorkItem.work_type.in_(work_types),
AccountWorkItem.status == "pending",
AccountWorkItem.due_at <= moment,
)
)
if account_ids is not None:
if not account_ids:
return []
query = query.filter(AccountWorkItem.account_id.in_(account_ids))
rows = (
query
.order_by(
AccountWorkItem.priority.asc(),
AccountWorkItem.due_at.asc(),
@@ -179,11 +188,71 @@ def recover_running_account_work(
return len(items)
def reschedule_account_work_type(
db: Session,
*,
account_id: int,
work_type: str,
due_at: datetime,
reason: str = "",
) -> int:
"""把自动重登失败的工作项按账号退避时间重新排队。"""
moment = (due_at if due_at.tzinfo else due_at.replace(tzinfo=UTC)).replace(
microsecond=0
)
items = (
db.query(AccountWorkItem)
.filter(
AccountWorkItem.account_id == account_id,
AccountWorkItem.work_type == work_type,
AccountWorkItem.status.in_(("pending", "running")),
)
.all()
)
for item in items:
item.status = "pending"
item.due_at = moment
item.last_error = reason[:2000]
item.updated_at = datetime.now(UTC)
db.flush()
return len(items)
def fail_account_work_type(
db: Session,
*,
account_id: int,
work_type: str,
reason: str,
now: datetime | None = None,
) -> int:
"""关闭不再自动处理的工作项,例如重登已转人工。"""
moment = now or datetime.now(UTC)
items = (
db.query(AccountWorkItem)
.filter(
AccountWorkItem.account_id == account_id,
AccountWorkItem.work_type == work_type,
AccountWorkItem.status.in_(("pending", "running")),
)
.all()
)
for item in items:
item.status = "failed"
item.finished_at = moment
item.last_error = reason[:2000]
item.updated_at = moment
db.flush()
return len(items)
__all__ = [
"claim_account_work",
"complete_account_work",
"complete_account_work_type",
"enqueue_account_work",
"fail_account_work",
"fail_account_work_type",
"recover_running_account_work",
"reschedule_account_work_type",
]
@@ -27,10 +27,17 @@ from ...account_events_service import EVENT_RELOGIN_FAILED, emit
from ...account_lifecycle import (
MANUAL_KINDS,
MANUAL_STATES,
WORK_LOGIN_RELOGIN,
on_login_success,
sync_status,
)
from ...account_value import value_tier_expr
from ...account_work_queue import (
claim_account_work,
enqueue_account_work,
fail_account_work_type,
reschedule_account_work_type,
)
from ...douyu_service import cookie_holding_filter, cookie_operable_filter
from ...login_failure import classify_failure
from ...login_service import LoginBatchRunner, get_relogin_limits
@@ -235,6 +242,47 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
if not accounts:
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
selected_account_ids = tuple(int(account.id) for account in accounts)
for account in accounts:
enqueue_account_work(
db,
account_id=int(account.id),
work_type=WORK_LOGIN_RELOGIN,
priority=30,
max_attempts=20,
due_at=now,
reason="CK 失效,等待自动重登",
)
db.commit()
work_items = claim_account_work(
db,
work_types=(WORK_LOGIN_RELOGIN,),
account_ids=selected_account_ids,
limit=len(selected_account_ids),
now=now,
)
db.commit()
claimed_account_ids = [int(item.account_id) for item in work_items]
if not claimed_account_ids:
return {
"scanned": 0,
"valid": 0,
"invalid": 0,
"skipped": len(selected_account_ids),
"message": "候选账号的重登工作项已被其他执行器领取",
}
account_map = {
int(account.id): account
for account in db.query(Account)
.filter(Account.id.in_(claimed_account_ids))
.all()
}
accounts = [
account_map[account_id]
for account_id in claimed_account_ids
if account_id in account_map
]
# 调度单位是**账号**(CK 在账号上);登录记录只是这次尝试的历史。
account_ids = [account.id for account in accounts]
proxy = db.query(ProxyConfigModel).first()
@@ -325,6 +373,13 @@ def settle_results(
for account_id in account_ids:
entry = fresh.get(account_id)
if entry is None:
reschedule_account_work_type(
db,
account_id=int(account_id),
work_type=WORK_LOGIN_RELOGIN,
due_at=now + timedelta(hours=1),
reason="自动重登未产生账号任务,重新排队",
)
continue
account, task = entry
if task.attempt_status == "success":
@@ -359,11 +414,25 @@ def settle_results(
)
# 状态跟着落到 inactive 的分支(invalid 或 manual)——推导函数负责判定
sync_status(db, account, occurred_at=now)
fail_account_work_type(
db,
account_id=int(account.id),
work_type=WORK_LOGIN_RELOGIN,
reason=f"自动重登转人工({kind})",
now=now,
)
continue
if kind == "email_quota":
counters["email_quota"] += 1
counters["retry"] += 1
account.relogin_next_at = _next_retry_at(kind, account.relogin_attempts, now)
reschedule_account_work_type(
db,
account_id=int(account.id),
work_type=WORK_LOGIN_RELOGIN,
due_at=account.relogin_next_at,
reason=f"自动重登未成功({kind}),按账号退避时间重试",
)
# 排了下一次重试 → 仍是 invalid(不是 manual)
sync_status(db, account, occurred_at=now)
return counters