feat(douyu): 接入重登工作队列与平台状态观测
This commit is contained in:
@@ -11,15 +11,15 @@
|
|||||||
|
|
||||||
更新时间:2026-11-21
|
更新时间:2026-11-21
|
||||||
|
|
||||||
整体完成度约 **60%**。
|
整体完成度约 **62%**。
|
||||||
|
|
||||||
| 阶段 | 完成度 | 已落地 | 主要缺口 |
|
| 阶段 | 完成度 | 已落地 | 主要缺口 |
|
||||||
| --- | ---: | --- | --- |
|
| --- | ---: | --- | --- |
|
||||||
| 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 |
|
| 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 |
|
||||||
| 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 |
|
| 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 |
|
||||||
| 阶段 2:账号与 CK | 80% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算 | 配置管理 API、更多自动重登灰度指标 |
|
| 阶段 2:账号与 CK | 85% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队 | 更多自动重登灰度指标 |
|
||||||
| 阶段 3:商品与供给 | 55% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API | 默认供给目标、供给看板、覆盖率告警 |
|
| 阶段 3:商品与供给 | 60% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API、统一平台状态快照 | 默认供给目标、供给看板、阈值告警 |
|
||||||
| 阶段 4:安全恢复自动化 | 20% | account_sync_queue 与供给补货作业默认小批量开启 | 自动重登灰度参数、成功率熔断和动态预算 |
|
| 阶段 4:安全恢复自动化 | 30% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列 | 自动重登灰度参数、成功率熔断和动态预算 |
|
||||||
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
|
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
|
||||||
|
|
||||||
## 为什么重构
|
## 为什么重构
|
||||||
|
|||||||
@@ -21,8 +21,11 @@ from web.backend.services.account_sync_service import (
|
|||||||
sync_storehome_snapshot,
|
sync_storehome_snapshot,
|
||||||
)
|
)
|
||||||
from web.backend.services.account_work_queue import (
|
from web.backend.services.account_work_queue import (
|
||||||
|
claim_account_work,
|
||||||
complete_account_work,
|
complete_account_work,
|
||||||
enqueue_account_work,
|
enqueue_account_work,
|
||||||
|
fail_account_work_type,
|
||||||
|
reschedule_account_work_type,
|
||||||
)
|
)
|
||||||
from web.backend.services.automation.jobs import ck_health
|
from web.backend.services.automation.jobs import ck_health
|
||||||
from web.backend.services.douyu_account_state import DouyuLiveAccountState
|
from web.backend.services.douyu_account_state import DouyuLiveAccountState
|
||||||
@@ -175,6 +178,71 @@ def test_account_work_queue_deduplicates_only_active_items(db):
|
|||||||
assert db.query(AccountWorkItem).count() == 2
|
assert db.query(AccountWorkItem).count() == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_account_work_claim_can_be_scoped_for_relogin(db):
|
||||||
|
first = _account("relogin-scope-a")
|
||||||
|
second = _account("relogin-scope-b")
|
||||||
|
db.add_all([first, second])
|
||||||
|
db.commit()
|
||||||
|
for account in (first, second):
|
||||||
|
enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type="login_relogin",
|
||||||
|
max_attempts=20,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
claimed = claim_account_work(
|
||||||
|
db,
|
||||||
|
work_types=("login_relogin",),
|
||||||
|
account_ids=(first.id,),
|
||||||
|
limit=10,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [item.account_id for item in claimed] == [first.id]
|
||||||
|
assert claimed[0].attempt_count == 1
|
||||||
|
assert claimed[0].max_attempts == 20
|
||||||
|
|
||||||
|
|
||||||
|
def test_relogin_work_reschedules_then_closes_for_manual(db):
|
||||||
|
account = _account("relogin-work-state")
|
||||||
|
db.add(account)
|
||||||
|
db.commit()
|
||||||
|
enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type="login_relogin",
|
||||||
|
max_attempts=20,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
item = claim_account_work(
|
||||||
|
db,
|
||||||
|
work_types=("login_relogin",),
|
||||||
|
account_ids=(account.id,),
|
||||||
|
limit=1,
|
||||||
|
)[0]
|
||||||
|
retry_at = datetime.now(UTC) + timedelta(hours=1)
|
||||||
|
|
||||||
|
reschedule_account_work_type(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type="login_relogin",
|
||||||
|
due_at=retry_at,
|
||||||
|
reason="验证码失败",
|
||||||
|
)
|
||||||
|
assert item.status == "pending"
|
||||||
|
assert item.due_at == retry_at.replace(microsecond=0)
|
||||||
|
|
||||||
|
fail_account_work_type(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type="login_relogin",
|
||||||
|
reason="转人工",
|
||||||
|
)
|
||||||
|
assert item.status == "failed"
|
||||||
|
assert item.last_error == "转人工"
|
||||||
|
|
||||||
|
|
||||||
def test_ck_verdict_updates_lifecycle_and_moves_relogin_work(db):
|
def test_ck_verdict_updates_lifecycle_and_moves_relogin_work(db):
|
||||||
issued_at = datetime.now(UTC) - timedelta(days=7, hours=9)
|
issued_at = datetime.now(UTC) - timedelta(days=7, hours=9)
|
||||||
account = _account("ck-lifecycle", cookie_updated_at=issued_at)
|
account = _account("ck-lifecycle", cookie_updated_at=issued_at)
|
||||||
|
|||||||
@@ -9,17 +9,20 @@ from web.backend.models import (
|
|||||||
AccountCommodityState,
|
AccountCommodityState,
|
||||||
AccountLease,
|
AccountLease,
|
||||||
AccountSupplyTarget,
|
AccountSupplyTarget,
|
||||||
|
AccountSyncState,
|
||||||
AccountWorkItem,
|
AccountWorkItem,
|
||||||
User,
|
User,
|
||||||
)
|
)
|
||||||
from web.backend.routers.douyu import create_supply_target, list_supply_targets
|
from web.backend.routers.douyu import create_supply_target, list_supply_targets
|
||||||
from web.backend.schemas import DouyuSupplyTargetUpsert
|
from web.backend.schemas import DouyuSupplyTargetUpsert
|
||||||
|
from web.backend.services.account_observability import account_platform_status
|
||||||
from web.backend.services.account_supply_service import (
|
from web.backend.services.account_supply_service import (
|
||||||
VALUE_HIGH,
|
VALUE_HIGH,
|
||||||
enqueue_supply_replenishment,
|
enqueue_supply_replenishment,
|
||||||
evaluate_supply_target,
|
evaluate_supply_target,
|
||||||
upsert_supply_target,
|
upsert_supply_target,
|
||||||
)
|
)
|
||||||
|
from web.backend.services.account_work_queue import enqueue_account_work
|
||||||
|
|
||||||
NOW = datetime.now(UTC).replace(microsecond=0)
|
NOW = datetime.now(UTC).replace(microsecond=0)
|
||||||
|
|
||||||
@@ -191,3 +194,47 @@ def test_supply_target_configuration_route_returns_inventory(db):
|
|||||||
assert created.covered_shortage == 12
|
assert created.covered_shortage == 12
|
||||||
assert listed[0].id == created.id
|
assert listed[0].id == created.id
|
||||||
assert listed[0].needs_replenishment is True
|
assert listed[0].needs_replenishment is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_account_platform_status_aggregates_queue_sync_and_supply(db):
|
||||||
|
account = _account("platform-status", points=400)
|
||||||
|
account.ck_expected_expires_at = NOW + timedelta(hours=12)
|
||||||
|
db.add(account)
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
_state(account),
|
||||||
|
AccountSyncState(
|
||||||
|
account_id=account.id,
|
||||||
|
platform="douyu",
|
||||||
|
domain="storehome",
|
||||||
|
status="success",
|
||||||
|
last_success_at=NOW,
|
||||||
|
valid_until=NOW + timedelta(hours=1),
|
||||||
|
),
|
||||||
|
AccountSupplyTarget(
|
||||||
|
platform="douyu",
|
||||||
|
scope="elite",
|
||||||
|
commodity_id="C",
|
||||||
|
value_class=VALUE_HIGH,
|
||||||
|
target_covered=2,
|
||||||
|
target_available=2,
|
||||||
|
target_allocatable=1,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type="login_relogin",
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
status = account_platform_status(db, now=NOW)
|
||||||
|
|
||||||
|
assert status["accounts"]["ck_valid"] == 1
|
||||||
|
assert status["accounts"]["ck_expiring_24h"] == 1
|
||||||
|
assert status["work_queue"]["login_relogin"]["pending"] == 1
|
||||||
|
assert status["sync_domains"]["storehome"]["fresh"] == 1
|
||||||
|
assert status["commodities"]["fresh_available"] == 1
|
||||||
|
assert status["supply_targets"][0]["covered_shortage"] == 1
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ from ..schemas import (
|
|||||||
DouyuXpdGoodsOut,
|
DouyuXpdGoodsOut,
|
||||||
)
|
)
|
||||||
from ..services.account_binding import bind_payload
|
from ..services.account_binding import bind_payload
|
||||||
|
from ..services.account_observability import account_platform_status
|
||||||
from ..services.account_supply_service import (
|
from ..services.account_supply_service import (
|
||||||
evaluate_supply_target,
|
evaluate_supply_target,
|
||||||
update_supply_target,
|
update_supply_target,
|
||||||
@@ -724,6 +725,15 @@ def task_types(current: User = Depends(require_permission("douyu:task"))):
|
|||||||
return SUPPORTED_DOUYU_TASK_TYPES
|
return SUPPORTED_DOUYU_TASK_TYPES
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/account-platform-status")
|
||||||
|
def get_account_platform_status(
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current: User = Depends(require_permission("douyu:task")),
|
||||||
|
):
|
||||||
|
"""账号同步、CK 周期、商品覆盖和供给缺口的统一观测快照。"""
|
||||||
|
return account_platform_status(db)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/accounts")
|
@router.get("/accounts")
|
||||||
def list_task_accounts(
|
def list_task_accounts(
|
||||||
search: str = Query(""),
|
search: str = Query(""),
|
||||||
|
|||||||
@@ -360,6 +360,7 @@ def on_ck_verdict(
|
|||||||
account_id=int(account.id),
|
account_id=int(account.id),
|
||||||
work_type=WORK_LOGIN_RELOGIN,
|
work_type=WORK_LOGIN_RELOGIN,
|
||||||
priority=30,
|
priority=30,
|
||||||
|
max_attempts=20,
|
||||||
reason="CK 已判定失效,等待受预算控制的重登",
|
reason="CK 已判定失效,等待受预算控制的重登",
|
||||||
)
|
)
|
||||||
except Exception as exc: # noqa: BLE001 - 生命周期主流程优先
|
except Exception as exc: # noqa: BLE001 - 生命周期主流程优先
|
||||||
|
|||||||
@@ -0,0 +1,187 @@
|
|||||||
|
"""斗鱼账号平台运行状态与覆盖率快照。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import case, func
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..models import (
|
||||||
|
Account,
|
||||||
|
AccountCommodityState,
|
||||||
|
AccountSyncState,
|
||||||
|
AccountWorkItem,
|
||||||
|
)
|
||||||
|
from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED
|
||||||
|
from .account_supply_service import evaluate_supply_targets
|
||||||
|
|
||||||
|
|
||||||
|
def account_platform_status(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
moment = now or datetime.now(UTC)
|
||||||
|
in_24h = moment + timedelta(hours=24)
|
||||||
|
in_72h = moment + timedelta(hours=72)
|
||||||
|
|
||||||
|
account_counts = dict(
|
||||||
|
db.query(Account.ck_check_status, func.count(Account.id))
|
||||||
|
.group_by(Account.ck_check_status)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
total_accounts = int(db.query(func.count(Account.id)).scalar() or 0)
|
||||||
|
cookie_accounts = int(
|
||||||
|
db.query(func.count(Account.id))
|
||||||
|
.filter(Account.cookie.isnot(None), Account.cookie != "")
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
|
||||||
|
expiring_24h = int(
|
||||||
|
db.query(func.count(Account.id))
|
||||||
|
.filter(
|
||||||
|
Account.ck_check_status == "valid",
|
||||||
|
Account.ck_expected_expires_at.isnot(None),
|
||||||
|
Account.ck_expected_expires_at > moment,
|
||||||
|
Account.ck_expected_expires_at <= in_24h,
|
||||||
|
)
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
expiring_72h = int(
|
||||||
|
db.query(func.count(Account.id))
|
||||||
|
.filter(
|
||||||
|
Account.ck_check_status == "valid",
|
||||||
|
Account.ck_expected_expires_at.isnot(None),
|
||||||
|
Account.ck_expected_expires_at > moment,
|
||||||
|
Account.ck_expected_expires_at <= in_72h,
|
||||||
|
)
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
predicted_expired = int(
|
||||||
|
db.query(func.count(Account.id))
|
||||||
|
.filter(
|
||||||
|
Account.ck_check_status == "valid",
|
||||||
|
Account.ck_expected_expires_at.isnot(None),
|
||||||
|
Account.ck_expected_expires_at <= moment,
|
||||||
|
)
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
|
||||||
|
queue_rows = (
|
||||||
|
db.query(
|
||||||
|
AccountWorkItem.work_type,
|
||||||
|
AccountWorkItem.status,
|
||||||
|
func.count(AccountWorkItem.id),
|
||||||
|
func.min(AccountWorkItem.due_at),
|
||||||
|
)
|
||||||
|
.group_by(AccountWorkItem.work_type, AccountWorkItem.status)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
work_queue: dict[str, dict[str, Any]] = {}
|
||||||
|
for work_type, status, count, oldest_due in queue_rows:
|
||||||
|
bucket = work_queue.setdefault(str(work_type), {})
|
||||||
|
bucket[str(status)] = int(count or 0)
|
||||||
|
if status == "pending":
|
||||||
|
bucket["oldest_pending_due_at"] = oldest_due
|
||||||
|
|
||||||
|
sync_rows = (
|
||||||
|
db.query(
|
||||||
|
AccountSyncState.domain,
|
||||||
|
AccountSyncState.status,
|
||||||
|
func.count(AccountSyncState.id),
|
||||||
|
)
|
||||||
|
.group_by(AccountSyncState.domain, AccountSyncState.status)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
sync_domains: dict[str, dict[str, Any]] = {}
|
||||||
|
for domain, status, count in sync_rows:
|
||||||
|
bucket = sync_domains.setdefault(str(domain), {})
|
||||||
|
bucket[str(status)] = int(count or 0)
|
||||||
|
fresh_rows = (
|
||||||
|
db.query(AccountSyncState.domain, func.count(AccountSyncState.id))
|
||||||
|
.filter(
|
||||||
|
AccountSyncState.status == "success",
|
||||||
|
AccountSyncState.valid_until.isnot(None),
|
||||||
|
AccountSyncState.valid_until > moment,
|
||||||
|
)
|
||||||
|
.group_by(AccountSyncState.domain)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for domain, count in fresh_rows:
|
||||||
|
sync_domains.setdefault(str(domain), {})["fresh"] = int(count or 0)
|
||||||
|
|
||||||
|
commodity_rows = (
|
||||||
|
db.query(
|
||||||
|
AccountCommodityState.availability,
|
||||||
|
func.count(AccountCommodityState.id),
|
||||||
|
func.sum(
|
||||||
|
case(
|
||||||
|
(
|
||||||
|
AccountCommodityState.valid_until > moment,
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
else_=0,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.filter(AccountCommodityState.platform == "douyu")
|
||||||
|
.group_by(AccountCommodityState.availability)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
commodities = {
|
||||||
|
str(availability): {
|
||||||
|
"total": int(total or 0),
|
||||||
|
"fresh": int(fresh or 0),
|
||||||
|
}
|
||||||
|
for availability, total, fresh in commodity_rows
|
||||||
|
}
|
||||||
|
fresh_available = int(commodities.get(AVAILABILITY_AVAILABLE, {}).get("fresh", 0))
|
||||||
|
fresh_limited = int(commodities.get(AVAILABILITY_LIMITED, {}).get("fresh", 0))
|
||||||
|
|
||||||
|
inventories = evaluate_supply_targets(db, now=moment)
|
||||||
|
return {
|
||||||
|
"checked_at": moment,
|
||||||
|
"accounts": {
|
||||||
|
"total": total_accounts,
|
||||||
|
"with_cookie": cookie_accounts,
|
||||||
|
"ck_valid": int(account_counts.get("valid", 0)),
|
||||||
|
"ck_invalid": int(account_counts.get("invalid", 0)),
|
||||||
|
"ck_error": int(account_counts.get("error", 0)),
|
||||||
|
"ck_unchecked": int(account_counts.get("", 0) or 0),
|
||||||
|
"ck_expiring_24h": expiring_24h,
|
||||||
|
"ck_expiring_72h": expiring_72h,
|
||||||
|
"ck_predicted_expired": predicted_expired,
|
||||||
|
},
|
||||||
|
"work_queue": work_queue,
|
||||||
|
"sync_domains": sync_domains,
|
||||||
|
"commodities": {
|
||||||
|
"by_availability": commodities,
|
||||||
|
"fresh_available": fresh_available,
|
||||||
|
"fresh_limited": fresh_limited,
|
||||||
|
},
|
||||||
|
"supply_targets": [
|
||||||
|
{
|
||||||
|
"target_id": item.target_id,
|
||||||
|
"scope": item.scope,
|
||||||
|
"commodity_id": item.commodity_id,
|
||||||
|
"value_class": item.value_class,
|
||||||
|
"covered": item.covered,
|
||||||
|
"available": item.available,
|
||||||
|
"allocatable": item.allocatable,
|
||||||
|
"covered_shortage": item.covered_shortage,
|
||||||
|
"available_shortage": item.available_shortage,
|
||||||
|
"allocatable_shortage": item.allocatable_shortage,
|
||||||
|
"needs_replenishment": item.needs_replenishment,
|
||||||
|
}
|
||||||
|
for item in inventories
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["account_platform_status"]
|
||||||
@@ -34,6 +34,7 @@ def enqueue_account_work(
|
|||||||
)
|
)
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
existing.priority = min(existing.priority, priority)
|
existing.priority = min(existing.priority, priority)
|
||||||
|
existing.max_attempts = max(existing.max_attempts, max(1, int(max_attempts)))
|
||||||
if due_at is not None and due_at < existing.due_at:
|
if due_at is not None and due_at < existing.due_at:
|
||||||
existing.due_at = due_at
|
existing.due_at = due_at
|
||||||
db.flush()
|
db.flush()
|
||||||
@@ -69,15 +70,23 @@ def claim_account_work(
|
|||||||
work_types: tuple[str, ...],
|
work_types: tuple[str, ...],
|
||||||
limit: int = 10,
|
limit: int = 10,
|
||||||
now: datetime | None = None,
|
now: datetime | None = None,
|
||||||
|
account_ids: tuple[int, ...] | None = None,
|
||||||
) -> list[AccountWorkItem]:
|
) -> list[AccountWorkItem]:
|
||||||
moment = now or datetime.now(UTC)
|
moment = now or datetime.now(UTC)
|
||||||
rows = (
|
query = (
|
||||||
db.query(AccountWorkItem)
|
db.query(AccountWorkItem)
|
||||||
.filter(
|
.filter(
|
||||||
AccountWorkItem.work_type.in_(work_types),
|
AccountWorkItem.work_type.in_(work_types),
|
||||||
AccountWorkItem.status == "pending",
|
AccountWorkItem.status == "pending",
|
||||||
AccountWorkItem.due_at <= moment,
|
AccountWorkItem.due_at <= moment,
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
if account_ids is not None:
|
||||||
|
if not account_ids:
|
||||||
|
return []
|
||||||
|
query = query.filter(AccountWorkItem.account_id.in_(account_ids))
|
||||||
|
rows = (
|
||||||
|
query
|
||||||
.order_by(
|
.order_by(
|
||||||
AccountWorkItem.priority.asc(),
|
AccountWorkItem.priority.asc(),
|
||||||
AccountWorkItem.due_at.asc(),
|
AccountWorkItem.due_at.asc(),
|
||||||
@@ -179,11 +188,71 @@ def recover_running_account_work(
|
|||||||
return len(items)
|
return len(items)
|
||||||
|
|
||||||
|
|
||||||
|
def reschedule_account_work_type(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
account_id: int,
|
||||||
|
work_type: str,
|
||||||
|
due_at: datetime,
|
||||||
|
reason: str = "",
|
||||||
|
) -> int:
|
||||||
|
"""把自动重登失败的工作项按账号退避时间重新排队。"""
|
||||||
|
moment = (due_at if due_at.tzinfo else due_at.replace(tzinfo=UTC)).replace(
|
||||||
|
microsecond=0
|
||||||
|
)
|
||||||
|
items = (
|
||||||
|
db.query(AccountWorkItem)
|
||||||
|
.filter(
|
||||||
|
AccountWorkItem.account_id == account_id,
|
||||||
|
AccountWorkItem.work_type == work_type,
|
||||||
|
AccountWorkItem.status.in_(("pending", "running")),
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for item in items:
|
||||||
|
item.status = "pending"
|
||||||
|
item.due_at = moment
|
||||||
|
item.last_error = reason[:2000]
|
||||||
|
item.updated_at = datetime.now(UTC)
|
||||||
|
db.flush()
|
||||||
|
return len(items)
|
||||||
|
|
||||||
|
|
||||||
|
def fail_account_work_type(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
account_id: int,
|
||||||
|
work_type: str,
|
||||||
|
reason: str,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""关闭不再自动处理的工作项,例如重登已转人工。"""
|
||||||
|
moment = now or datetime.now(UTC)
|
||||||
|
items = (
|
||||||
|
db.query(AccountWorkItem)
|
||||||
|
.filter(
|
||||||
|
AccountWorkItem.account_id == account_id,
|
||||||
|
AccountWorkItem.work_type == work_type,
|
||||||
|
AccountWorkItem.status.in_(("pending", "running")),
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for item in items:
|
||||||
|
item.status = "failed"
|
||||||
|
item.finished_at = moment
|
||||||
|
item.last_error = reason[:2000]
|
||||||
|
item.updated_at = moment
|
||||||
|
db.flush()
|
||||||
|
return len(items)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"claim_account_work",
|
"claim_account_work",
|
||||||
"complete_account_work",
|
"complete_account_work",
|
||||||
"complete_account_work_type",
|
"complete_account_work_type",
|
||||||
"enqueue_account_work",
|
"enqueue_account_work",
|
||||||
"fail_account_work",
|
"fail_account_work",
|
||||||
|
"fail_account_work_type",
|
||||||
"recover_running_account_work",
|
"recover_running_account_work",
|
||||||
|
"reschedule_account_work_type",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -27,10 +27,17 @@ from ...account_events_service import EVENT_RELOGIN_FAILED, emit
|
|||||||
from ...account_lifecycle import (
|
from ...account_lifecycle import (
|
||||||
MANUAL_KINDS,
|
MANUAL_KINDS,
|
||||||
MANUAL_STATES,
|
MANUAL_STATES,
|
||||||
|
WORK_LOGIN_RELOGIN,
|
||||||
on_login_success,
|
on_login_success,
|
||||||
sync_status,
|
sync_status,
|
||||||
)
|
)
|
||||||
from ...account_value import value_tier_expr
|
from ...account_value import value_tier_expr
|
||||||
|
from ...account_work_queue import (
|
||||||
|
claim_account_work,
|
||||||
|
enqueue_account_work,
|
||||||
|
fail_account_work_type,
|
||||||
|
reschedule_account_work_type,
|
||||||
|
)
|
||||||
from ...douyu_service import cookie_holding_filter, cookie_operable_filter
|
from ...douyu_service import cookie_holding_filter, cookie_operable_filter
|
||||||
from ...login_failure import classify_failure
|
from ...login_failure import classify_failure
|
||||||
from ...login_service import LoginBatchRunner, get_relogin_limits
|
from ...login_service import LoginBatchRunner, get_relogin_limits
|
||||||
@@ -235,6 +242,47 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
|||||||
if not accounts:
|
if not accounts:
|
||||||
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
|
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
|
||||||
|
|
||||||
|
selected_account_ids = tuple(int(account.id) for account in accounts)
|
||||||
|
for account in accounts:
|
||||||
|
enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=int(account.id),
|
||||||
|
work_type=WORK_LOGIN_RELOGIN,
|
||||||
|
priority=30,
|
||||||
|
max_attempts=20,
|
||||||
|
due_at=now,
|
||||||
|
reason="CK 失效,等待自动重登",
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
work_items = claim_account_work(
|
||||||
|
db,
|
||||||
|
work_types=(WORK_LOGIN_RELOGIN,),
|
||||||
|
account_ids=selected_account_ids,
|
||||||
|
limit=len(selected_account_ids),
|
||||||
|
now=now,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
claimed_account_ids = [int(item.account_id) for item in work_items]
|
||||||
|
if not claimed_account_ids:
|
||||||
|
return {
|
||||||
|
"scanned": 0,
|
||||||
|
"valid": 0,
|
||||||
|
"invalid": 0,
|
||||||
|
"skipped": len(selected_account_ids),
|
||||||
|
"message": "候选账号的重登工作项已被其他执行器领取",
|
||||||
|
}
|
||||||
|
account_map = {
|
||||||
|
int(account.id): account
|
||||||
|
for account in db.query(Account)
|
||||||
|
.filter(Account.id.in_(claimed_account_ids))
|
||||||
|
.all()
|
||||||
|
}
|
||||||
|
accounts = [
|
||||||
|
account_map[account_id]
|
||||||
|
for account_id in claimed_account_ids
|
||||||
|
if account_id in account_map
|
||||||
|
]
|
||||||
|
|
||||||
# 调度单位是**账号**(CK 在账号上);登录记录只是这次尝试的历史。
|
# 调度单位是**账号**(CK 在账号上);登录记录只是这次尝试的历史。
|
||||||
account_ids = [account.id for account in accounts]
|
account_ids = [account.id for account in accounts]
|
||||||
proxy = db.query(ProxyConfigModel).first()
|
proxy = db.query(ProxyConfigModel).first()
|
||||||
@@ -325,6 +373,13 @@ def settle_results(
|
|||||||
for account_id in account_ids:
|
for account_id in account_ids:
|
||||||
entry = fresh.get(account_id)
|
entry = fresh.get(account_id)
|
||||||
if entry is None:
|
if entry is None:
|
||||||
|
reschedule_account_work_type(
|
||||||
|
db,
|
||||||
|
account_id=int(account_id),
|
||||||
|
work_type=WORK_LOGIN_RELOGIN,
|
||||||
|
due_at=now + timedelta(hours=1),
|
||||||
|
reason="自动重登未产生账号任务,重新排队",
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
account, task = entry
|
account, task = entry
|
||||||
if task.attempt_status == "success":
|
if task.attempt_status == "success":
|
||||||
@@ -359,11 +414,25 @@ def settle_results(
|
|||||||
)
|
)
|
||||||
# 状态跟着落到 inactive 的分支(invalid 或 manual)——推导函数负责判定
|
# 状态跟着落到 inactive 的分支(invalid 或 manual)——推导函数负责判定
|
||||||
sync_status(db, account, occurred_at=now)
|
sync_status(db, account, occurred_at=now)
|
||||||
|
fail_account_work_type(
|
||||||
|
db,
|
||||||
|
account_id=int(account.id),
|
||||||
|
work_type=WORK_LOGIN_RELOGIN,
|
||||||
|
reason=f"自动重登转人工({kind})",
|
||||||
|
now=now,
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
if kind == "email_quota":
|
if kind == "email_quota":
|
||||||
counters["email_quota"] += 1
|
counters["email_quota"] += 1
|
||||||
counters["retry"] += 1
|
counters["retry"] += 1
|
||||||
account.relogin_next_at = _next_retry_at(kind, account.relogin_attempts, now)
|
account.relogin_next_at = _next_retry_at(kind, account.relogin_attempts, now)
|
||||||
|
reschedule_account_work_type(
|
||||||
|
db,
|
||||||
|
account_id=int(account.id),
|
||||||
|
work_type=WORK_LOGIN_RELOGIN,
|
||||||
|
due_at=account.relogin_next_at,
|
||||||
|
reason=f"自动重登未成功({kind}),按账号退避时间重试",
|
||||||
|
)
|
||||||
# 排了下一次重试 → 仍是 invalid(不是 manual)
|
# 排了下一次重试 → 仍是 invalid(不是 manual)
|
||||||
sync_status(db, account, occurred_at=now)
|
sync_status(db, account, occurred_at=now)
|
||||||
return counters
|
return counters
|
||||||
|
|||||||
Reference in New Issue
Block a user