feat(douyu): 接入重登工作队列与平台状态观测
This commit is contained in:
@@ -62,6 +62,7 @@ from ..schemas import (
|
||||
DouyuXpdGoodsOut,
|
||||
)
|
||||
from ..services.account_binding import bind_payload
|
||||
from ..services.account_observability import account_platform_status
|
||||
from ..services.account_supply_service import (
|
||||
evaluate_supply_target,
|
||||
update_supply_target,
|
||||
@@ -724,6 +725,15 @@ def task_types(current: User = Depends(require_permission("douyu:task"))):
|
||||
return SUPPORTED_DOUYU_TASK_TYPES
|
||||
|
||||
|
||||
@router.get("/account-platform-status")
|
||||
def get_account_platform_status(
|
||||
db: Session = Depends(get_db),
|
||||
current: User = Depends(require_permission("douyu:task")),
|
||||
):
|
||||
"""账号同步、CK 周期、商品覆盖和供给缺口的统一观测快照。"""
|
||||
return account_platform_status(db)
|
||||
|
||||
|
||||
@router.get("/accounts")
|
||||
def list_task_accounts(
|
||||
search: str = Query(""),
|
||||
|
||||
@@ -360,6 +360,7 @@ def on_ck_verdict(
|
||||
account_id=int(account.id),
|
||||
work_type=WORK_LOGIN_RELOGIN,
|
||||
priority=30,
|
||||
max_attempts=20,
|
||||
reason="CK 已判定失效,等待受预算控制的重登",
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - 生命周期主流程优先
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
"""斗鱼账号平台运行状态与覆盖率快照。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import case, func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
Account,
|
||||
AccountCommodityState,
|
||||
AccountSyncState,
|
||||
AccountWorkItem,
|
||||
)
|
||||
from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED
|
||||
from .account_supply_service import evaluate_supply_targets
|
||||
|
||||
|
||||
def account_platform_status(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
moment = now or datetime.now(UTC)
|
||||
in_24h = moment + timedelta(hours=24)
|
||||
in_72h = moment + timedelta(hours=72)
|
||||
|
||||
account_counts = dict(
|
||||
db.query(Account.ck_check_status, func.count(Account.id))
|
||||
.group_by(Account.ck_check_status)
|
||||
.all()
|
||||
)
|
||||
total_accounts = int(db.query(func.count(Account.id)).scalar() or 0)
|
||||
cookie_accounts = int(
|
||||
db.query(func.count(Account.id))
|
||||
.filter(Account.cookie.isnot(None), Account.cookie != "")
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
expiring_24h = int(
|
||||
db.query(func.count(Account.id))
|
||||
.filter(
|
||||
Account.ck_check_status == "valid",
|
||||
Account.ck_expected_expires_at.isnot(None),
|
||||
Account.ck_expected_expires_at > moment,
|
||||
Account.ck_expected_expires_at <= in_24h,
|
||||
)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
expiring_72h = int(
|
||||
db.query(func.count(Account.id))
|
||||
.filter(
|
||||
Account.ck_check_status == "valid",
|
||||
Account.ck_expected_expires_at.isnot(None),
|
||||
Account.ck_expected_expires_at > moment,
|
||||
Account.ck_expected_expires_at <= in_72h,
|
||||
)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
predicted_expired = int(
|
||||
db.query(func.count(Account.id))
|
||||
.filter(
|
||||
Account.ck_check_status == "valid",
|
||||
Account.ck_expected_expires_at.isnot(None),
|
||||
Account.ck_expected_expires_at <= moment,
|
||||
)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
queue_rows = (
|
||||
db.query(
|
||||
AccountWorkItem.work_type,
|
||||
AccountWorkItem.status,
|
||||
func.count(AccountWorkItem.id),
|
||||
func.min(AccountWorkItem.due_at),
|
||||
)
|
||||
.group_by(AccountWorkItem.work_type, AccountWorkItem.status)
|
||||
.all()
|
||||
)
|
||||
work_queue: dict[str, dict[str, Any]] = {}
|
||||
for work_type, status, count, oldest_due in queue_rows:
|
||||
bucket = work_queue.setdefault(str(work_type), {})
|
||||
bucket[str(status)] = int(count or 0)
|
||||
if status == "pending":
|
||||
bucket["oldest_pending_due_at"] = oldest_due
|
||||
|
||||
sync_rows = (
|
||||
db.query(
|
||||
AccountSyncState.domain,
|
||||
AccountSyncState.status,
|
||||
func.count(AccountSyncState.id),
|
||||
)
|
||||
.group_by(AccountSyncState.domain, AccountSyncState.status)
|
||||
.all()
|
||||
)
|
||||
sync_domains: dict[str, dict[str, Any]] = {}
|
||||
for domain, status, count in sync_rows:
|
||||
bucket = sync_domains.setdefault(str(domain), {})
|
||||
bucket[str(status)] = int(count or 0)
|
||||
fresh_rows = (
|
||||
db.query(AccountSyncState.domain, func.count(AccountSyncState.id))
|
||||
.filter(
|
||||
AccountSyncState.status == "success",
|
||||
AccountSyncState.valid_until.isnot(None),
|
||||
AccountSyncState.valid_until > moment,
|
||||
)
|
||||
.group_by(AccountSyncState.domain)
|
||||
.all()
|
||||
)
|
||||
for domain, count in fresh_rows:
|
||||
sync_domains.setdefault(str(domain), {})["fresh"] = int(count or 0)
|
||||
|
||||
commodity_rows = (
|
||||
db.query(
|
||||
AccountCommodityState.availability,
|
||||
func.count(AccountCommodityState.id),
|
||||
func.sum(
|
||||
case(
|
||||
(
|
||||
AccountCommodityState.valid_until > moment,
|
||||
1,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
),
|
||||
)
|
||||
.filter(AccountCommodityState.platform == "douyu")
|
||||
.group_by(AccountCommodityState.availability)
|
||||
.all()
|
||||
)
|
||||
commodities = {
|
||||
str(availability): {
|
||||
"total": int(total or 0),
|
||||
"fresh": int(fresh or 0),
|
||||
}
|
||||
for availability, total, fresh in commodity_rows
|
||||
}
|
||||
fresh_available = int(commodities.get(AVAILABILITY_AVAILABLE, {}).get("fresh", 0))
|
||||
fresh_limited = int(commodities.get(AVAILABILITY_LIMITED, {}).get("fresh", 0))
|
||||
|
||||
inventories = evaluate_supply_targets(db, now=moment)
|
||||
return {
|
||||
"checked_at": moment,
|
||||
"accounts": {
|
||||
"total": total_accounts,
|
||||
"with_cookie": cookie_accounts,
|
||||
"ck_valid": int(account_counts.get("valid", 0)),
|
||||
"ck_invalid": int(account_counts.get("invalid", 0)),
|
||||
"ck_error": int(account_counts.get("error", 0)),
|
||||
"ck_unchecked": int(account_counts.get("", 0) or 0),
|
||||
"ck_expiring_24h": expiring_24h,
|
||||
"ck_expiring_72h": expiring_72h,
|
||||
"ck_predicted_expired": predicted_expired,
|
||||
},
|
||||
"work_queue": work_queue,
|
||||
"sync_domains": sync_domains,
|
||||
"commodities": {
|
||||
"by_availability": commodities,
|
||||
"fresh_available": fresh_available,
|
||||
"fresh_limited": fresh_limited,
|
||||
},
|
||||
"supply_targets": [
|
||||
{
|
||||
"target_id": item.target_id,
|
||||
"scope": item.scope,
|
||||
"commodity_id": item.commodity_id,
|
||||
"value_class": item.value_class,
|
||||
"covered": item.covered,
|
||||
"available": item.available,
|
||||
"allocatable": item.allocatable,
|
||||
"covered_shortage": item.covered_shortage,
|
||||
"available_shortage": item.available_shortage,
|
||||
"allocatable_shortage": item.allocatable_shortage,
|
||||
"needs_replenishment": item.needs_replenishment,
|
||||
}
|
||||
for item in inventories
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["account_platform_status"]
|
||||
@@ -34,6 +34,7 @@ def enqueue_account_work(
|
||||
)
|
||||
if existing is not None:
|
||||
existing.priority = min(existing.priority, priority)
|
||||
existing.max_attempts = max(existing.max_attempts, max(1, int(max_attempts)))
|
||||
if due_at is not None and due_at < existing.due_at:
|
||||
existing.due_at = due_at
|
||||
db.flush()
|
||||
@@ -69,15 +70,23 @@ def claim_account_work(
|
||||
work_types: tuple[str, ...],
|
||||
limit: int = 10,
|
||||
now: datetime | None = None,
|
||||
account_ids: tuple[int, ...] | None = None,
|
||||
) -> list[AccountWorkItem]:
|
||||
moment = now or datetime.now(UTC)
|
||||
rows = (
|
||||
query = (
|
||||
db.query(AccountWorkItem)
|
||||
.filter(
|
||||
AccountWorkItem.work_type.in_(work_types),
|
||||
AccountWorkItem.status == "pending",
|
||||
AccountWorkItem.due_at <= moment,
|
||||
)
|
||||
)
|
||||
if account_ids is not None:
|
||||
if not account_ids:
|
||||
return []
|
||||
query = query.filter(AccountWorkItem.account_id.in_(account_ids))
|
||||
rows = (
|
||||
query
|
||||
.order_by(
|
||||
AccountWorkItem.priority.asc(),
|
||||
AccountWorkItem.due_at.asc(),
|
||||
@@ -179,11 +188,71 @@ def recover_running_account_work(
|
||||
return len(items)
|
||||
|
||||
|
||||
def reschedule_account_work_type(
|
||||
db: Session,
|
||||
*,
|
||||
account_id: int,
|
||||
work_type: str,
|
||||
due_at: datetime,
|
||||
reason: str = "",
|
||||
) -> int:
|
||||
"""把自动重登失败的工作项按账号退避时间重新排队。"""
|
||||
moment = (due_at if due_at.tzinfo else due_at.replace(tzinfo=UTC)).replace(
|
||||
microsecond=0
|
||||
)
|
||||
items = (
|
||||
db.query(AccountWorkItem)
|
||||
.filter(
|
||||
AccountWorkItem.account_id == account_id,
|
||||
AccountWorkItem.work_type == work_type,
|
||||
AccountWorkItem.status.in_(("pending", "running")),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for item in items:
|
||||
item.status = "pending"
|
||||
item.due_at = moment
|
||||
item.last_error = reason[:2000]
|
||||
item.updated_at = datetime.now(UTC)
|
||||
db.flush()
|
||||
return len(items)
|
||||
|
||||
|
||||
def fail_account_work_type(
|
||||
db: Session,
|
||||
*,
|
||||
account_id: int,
|
||||
work_type: str,
|
||||
reason: str,
|
||||
now: datetime | None = None,
|
||||
) -> int:
|
||||
"""关闭不再自动处理的工作项,例如重登已转人工。"""
|
||||
moment = now or datetime.now(UTC)
|
||||
items = (
|
||||
db.query(AccountWorkItem)
|
||||
.filter(
|
||||
AccountWorkItem.account_id == account_id,
|
||||
AccountWorkItem.work_type == work_type,
|
||||
AccountWorkItem.status.in_(("pending", "running")),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for item in items:
|
||||
item.status = "failed"
|
||||
item.finished_at = moment
|
||||
item.last_error = reason[:2000]
|
||||
item.updated_at = moment
|
||||
db.flush()
|
||||
return len(items)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"claim_account_work",
|
||||
"complete_account_work",
|
||||
"complete_account_work_type",
|
||||
"enqueue_account_work",
|
||||
"fail_account_work",
|
||||
"fail_account_work_type",
|
||||
"recover_running_account_work",
|
||||
"reschedule_account_work_type",
|
||||
]
|
||||
|
||||
@@ -27,10 +27,17 @@ from ...account_events_service import EVENT_RELOGIN_FAILED, emit
|
||||
from ...account_lifecycle import (
|
||||
MANUAL_KINDS,
|
||||
MANUAL_STATES,
|
||||
WORK_LOGIN_RELOGIN,
|
||||
on_login_success,
|
||||
sync_status,
|
||||
)
|
||||
from ...account_value import value_tier_expr
|
||||
from ...account_work_queue import (
|
||||
claim_account_work,
|
||||
enqueue_account_work,
|
||||
fail_account_work_type,
|
||||
reschedule_account_work_type,
|
||||
)
|
||||
from ...douyu_service import cookie_holding_filter, cookie_operable_filter
|
||||
from ...login_failure import classify_failure
|
||||
from ...login_service import LoginBatchRunner, get_relogin_limits
|
||||
@@ -235,6 +242,47 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
if not accounts:
|
||||
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
|
||||
|
||||
selected_account_ids = tuple(int(account.id) for account in accounts)
|
||||
for account in accounts:
|
||||
enqueue_account_work(
|
||||
db,
|
||||
account_id=int(account.id),
|
||||
work_type=WORK_LOGIN_RELOGIN,
|
||||
priority=30,
|
||||
max_attempts=20,
|
||||
due_at=now,
|
||||
reason="CK 失效,等待自动重登",
|
||||
)
|
||||
db.commit()
|
||||
work_items = claim_account_work(
|
||||
db,
|
||||
work_types=(WORK_LOGIN_RELOGIN,),
|
||||
account_ids=selected_account_ids,
|
||||
limit=len(selected_account_ids),
|
||||
now=now,
|
||||
)
|
||||
db.commit()
|
||||
claimed_account_ids = [int(item.account_id) for item in work_items]
|
||||
if not claimed_account_ids:
|
||||
return {
|
||||
"scanned": 0,
|
||||
"valid": 0,
|
||||
"invalid": 0,
|
||||
"skipped": len(selected_account_ids),
|
||||
"message": "候选账号的重登工作项已被其他执行器领取",
|
||||
}
|
||||
account_map = {
|
||||
int(account.id): account
|
||||
for account in db.query(Account)
|
||||
.filter(Account.id.in_(claimed_account_ids))
|
||||
.all()
|
||||
}
|
||||
accounts = [
|
||||
account_map[account_id]
|
||||
for account_id in claimed_account_ids
|
||||
if account_id in account_map
|
||||
]
|
||||
|
||||
# 调度单位是**账号**(CK 在账号上);登录记录只是这次尝试的历史。
|
||||
account_ids = [account.id for account in accounts]
|
||||
proxy = db.query(ProxyConfigModel).first()
|
||||
@@ -325,6 +373,13 @@ def settle_results(
|
||||
for account_id in account_ids:
|
||||
entry = fresh.get(account_id)
|
||||
if entry is None:
|
||||
reschedule_account_work_type(
|
||||
db,
|
||||
account_id=int(account_id),
|
||||
work_type=WORK_LOGIN_RELOGIN,
|
||||
due_at=now + timedelta(hours=1),
|
||||
reason="自动重登未产生账号任务,重新排队",
|
||||
)
|
||||
continue
|
||||
account, task = entry
|
||||
if task.attempt_status == "success":
|
||||
@@ -359,11 +414,25 @@ def settle_results(
|
||||
)
|
||||
# 状态跟着落到 inactive 的分支(invalid 或 manual)——推导函数负责判定
|
||||
sync_status(db, account, occurred_at=now)
|
||||
fail_account_work_type(
|
||||
db,
|
||||
account_id=int(account.id),
|
||||
work_type=WORK_LOGIN_RELOGIN,
|
||||
reason=f"自动重登转人工({kind})",
|
||||
now=now,
|
||||
)
|
||||
continue
|
||||
if kind == "email_quota":
|
||||
counters["email_quota"] += 1
|
||||
counters["retry"] += 1
|
||||
account.relogin_next_at = _next_retry_at(kind, account.relogin_attempts, now)
|
||||
reschedule_account_work_type(
|
||||
db,
|
||||
account_id=int(account.id),
|
||||
work_type=WORK_LOGIN_RELOGIN,
|
||||
due_at=account.relogin_next_at,
|
||||
reason=f"自动重登未成功({kind}),按账号退避时间重试",
|
||||
)
|
||||
# 排了下一次重试 → 仍是 invalid(不是 manual)
|
||||
sync_status(db, account, occurred_at=now)
|
||||
return counters
|
||||
|
||||
Reference in New Issue
Block a user