feat(douyu): 建立账号同步队列与供给库存

This commit is contained in:
yml
2026-11-21 21:48:51 +08:00
parent 1d262cdfd9
commit 3baf15afda
16 changed files with 1158 additions and 33 deletions
@@ -18,6 +18,18 @@ from .jobs.account_overview_refresh import (
from .jobs.account_overview_refresh import (
run_account_overview_refresh,
)
from .jobs.account_supply_replenishment import (
JOB_KEY as ACCOUNT_SUPPLY_REPLENISHMENT_KEY,
)
from .jobs.account_supply_replenishment import (
run_account_supply_replenishment,
)
from .jobs.account_sync_queue import (
JOB_KEY as ACCOUNT_SYNC_QUEUE_KEY,
)
from .jobs.account_sync_queue import (
run_account_sync_queue_job,
)
from .jobs.ck_health import JOB_KEY as CK_HEALTH_KEY
from .jobs.ck_health import run_ck_health
from .jobs.ck_relogin import JOB_KEY as CK_RELOGIN_KEY
@@ -53,6 +65,8 @@ from .scheduler import (
__all__ = [
"ACCOUNT_CAPABILITY_REFRESH_KEY",
"ACCOUNT_OVERVIEW_REFRESH_KEY",
"ACCOUNT_SUPPLY_REPLENISHMENT_KEY",
"ACCOUNT_SYNC_QUEUE_KEY",
"CK_HEALTH_KEY",
"CK_RELOGIN_KEY",
"DOUYU_TASKS_RETENTION_KEY",
@@ -67,6 +81,8 @@ __all__ = [
"reset_registry",
"run_account_capability_refresh",
"run_account_overview_refresh",
"run_account_supply_replenishment",
"run_account_sync_queue_job",
"run_ck_health",
"run_ck_relogin",
"run_douyu_tasks_retention",
@@ -82,6 +98,30 @@ __all__ = [
def register_default_jobs() -> None:
"""登记内置作业(幂等)。"""
register_job(
JobSpec(
key=ACCOUNT_SYNC_QUEUE_KEY,
title="账号内部同步队列",
interval_seconds=60,
run_interval_seconds=60,
batch_size=20,
concurrency=3,
run=run_account_sync_queue_job,
default_enabled=True,
)
)
register_job(
JobSpec(
key=ACCOUNT_SUPPLY_REPLENISHMENT_KEY,
title="账号供给目标补货",
interval_seconds=5 * 60,
run_interval_seconds=5 * 60,
batch_size=50,
concurrency=1,
run=run_account_supply_replenishment,
default_enabled=True,
)
)
register_job(
JobSpec(
key=ACCOUNT_CAPABILITY_REFRESH_KEY,
@@ -1,8 +1,8 @@
"""按直播履约需求刷新账号商品能力。
"""按直播履约需求把过期商品能力账号送入内部同步队列。
全量资料补全仍由 ``account_overview_refresh`` 负责;本作业只处理“有 pending
任务、但对应商品能力快照已经过期”的账号,避免为了秒匹配把两万多个账号
无条件轮询一遍。
本作业只做本地候选筛选,不直接请求官网。真正的 ``storehome`` 查询由
``account_sync_queue`` 作业在独立执行槽中低并发消费,避免补货查询抢占
履约实时复核、登录或批量任务的执行资源。
"""
from __future__ import annotations
@@ -21,8 +21,9 @@ from ....models import (
PlatformFulfillmentTask,
)
from ...account_lifecycle import MANUAL_STATES
from ...account_sync_runner import WORK_STOREHOME_SYNC
from ...account_work_queue import enqueue_account_work
from ...douyu_service import cookie_operable_filter
from .account_overview_refresh import run_refresh_for_accounts
JOB_KEY = "account_capability_refresh"
@@ -94,18 +95,29 @@ def run_account_capability_refresh(
) -> dict[str, Any]:
now = datetime.now(UTC)
accounts = build_select_query(db, job, now).all()
return run_refresh_for_accounts(
db,
job,
accounts,
now=now,
# 精确剩余量要完整评估 accounts × commodity EXISTS,生产实测单次约
# 1.3 秒、扫描 38 万行;它只用于展示,直接省略,避免每轮制造慢查询。
remaining_count=None,
source="capability_demand_refresh",
log_label="capability",
empty_message="当前履约商品没有到期待刷新的账号能力",
)
for account in accounts:
enqueue_account_work(
db,
account_id=int(account.id),
work_type=WORK_STOREHOME_SYNC,
priority=40,
reason="履约商品能力过期,等待补货刷新",
due_at=now,
)
db.commit()
queued = len(accounts)
return {
"scanned": len(accounts),
"success": queued,
"invalid": 0,
"skipped": 0,
"message": (
f"本轮筛选 {len(accounts)} 个到期账号,新入队 {queued} 个"
if accounts
else "当前履约商品没有到期待刷新的账号能力"
),
"detail": {"queued": queued},
}
__all__ = [
@@ -0,0 +1,41 @@
"""按账号供给目标缺口定向补货入队。"""
from __future__ import annotations
from typing import Any
from sqlalchemy.orm import Session
from ....models import AutomationJob
from ...account_supply_service import enqueue_supply_replenishment
JOB_KEY = "account_supply_replenishment"
def run_account_supply_replenishment(
db: Session,
job: AutomationJob,
) -> dict[str, Any]:
result = enqueue_supply_replenishment(
db,
max_per_target=max(1, int(job.batch_size or 50)),
)
return {
"scanned": result.targets,
"success": result.queued,
"invalid": 0,
"skipped": max(0, result.targets - result.shortages),
"message": (
f"评估 {result.targets} 个供给目标,"
f"短缺 {result.shortages} 个,新排队 {result.queued} 个账号同步"
if result.targets
else "未配置账号供给目标"
),
"detail": {
"shortages": result.shortages,
"queued": result.queued,
},
}
__all__ = ["JOB_KEY", "run_account_supply_replenishment"]
@@ -0,0 +1,47 @@
"""账号内部同步队列作业。"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.orm import Session
from ....models import AccountWorkItem, AutomationJob
from ...account_sync_runner import ACCOUNT_SYNC_WORK_TYPES, run_account_sync_queue
JOB_KEY = "account_sync_queue"
def count_pending_work(db: Session, *, now: datetime | None = None) -> int:
moment = now or datetime.now(UTC)
return int(
db.query(AccountWorkItem.id)
.filter(
AccountWorkItem.work_type.in_(ACCOUNT_SYNC_WORK_TYPES),
AccountWorkItem.status == "pending",
AccountWorkItem.due_at <= moment,
)
.count()
or 0
)
def run_account_sync_queue_job(
db: Session,
job: AutomationJob,
) -> dict[str, Any]:
result = run_account_sync_queue(
db,
batch_size=max(1, int(job.batch_size or 20)),
)
if result.get("scanned"):
result["remaining"] = count_pending_work(db)
return result
__all__ = [
"JOB_KEY",
"count_pending_work",
"run_account_sync_queue_job",
]