feat(douyu): 建立账号同步队列与供给库存
This commit is contained in:
@@ -99,6 +99,7 @@ TASK_LANE_BINDING_POLL_WORKERS=2
|
|||||||
TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT=2
|
TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT=2
|
||||||
TASK_LANE_LOGIN_WORKERS=4
|
TASK_LANE_LOGIN_WORKERS=4
|
||||||
TASK_LANE_ACCOUNT_CHECK_WORKERS=4
|
TASK_LANE_ACCOUNT_CHECK_WORKERS=4
|
||||||
|
TASK_LANE_ACCOUNT_SYNC_WORKERS=3
|
||||||
TASK_LANE_OVERVIEW_QUERY_WORKERS=5
|
TASK_LANE_OVERVIEW_QUERY_WORKERS=5
|
||||||
TASK_LANE_QUERY_BATCH_WORKERS=3
|
TASK_LANE_QUERY_BATCH_WORKERS=3
|
||||||
TASK_LANE_PAYMENT_QR_WORKERS=10
|
TASK_LANE_PAYMENT_QR_WORKERS=10
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ services:
|
|||||||
- TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT=${TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT:-2}
|
- TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT=${TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT:-2}
|
||||||
- TASK_LANE_LOGIN_WORKERS=${TASK_LANE_LOGIN_WORKERS:-4}
|
- TASK_LANE_LOGIN_WORKERS=${TASK_LANE_LOGIN_WORKERS:-4}
|
||||||
- TASK_LANE_ACCOUNT_CHECK_WORKERS=${TASK_LANE_ACCOUNT_CHECK_WORKERS:-4}
|
- TASK_LANE_ACCOUNT_CHECK_WORKERS=${TASK_LANE_ACCOUNT_CHECK_WORKERS:-4}
|
||||||
|
- TASK_LANE_ACCOUNT_SYNC_WORKERS=${TASK_LANE_ACCOUNT_SYNC_WORKERS:-3}
|
||||||
- TASK_LANE_OVERVIEW_QUERY_WORKERS=${TASK_LANE_OVERVIEW_QUERY_WORKERS:-5}
|
- TASK_LANE_OVERVIEW_QUERY_WORKERS=${TASK_LANE_OVERVIEW_QUERY_WORKERS:-5}
|
||||||
- TASK_LANE_QUERY_BATCH_WORKERS=${TASK_LANE_QUERY_BATCH_WORKERS:-3}
|
- TASK_LANE_QUERY_BATCH_WORKERS=${TASK_LANE_QUERY_BATCH_WORKERS:-3}
|
||||||
- TASK_LANE_PAYMENT_QR_WORKERS=${TASK_LANE_PAYMENT_QR_WORKERS:-10}
|
- TASK_LANE_PAYMENT_QR_WORKERS=${TASK_LANE_PAYMENT_QR_WORKERS:-10}
|
||||||
|
|||||||
@@ -7,6 +7,21 @@
|
|||||||
本文是斗鱼账号平台后续实施的唯一主线。协议细节、字段真值和专项分析保留在其他
|
本文是斗鱼账号平台后续实施的唯一主线。协议细节、字段真值和专项分析保留在其他
|
||||||
文档中,但模块职责、数据流、调度和实施顺序以本文为准。
|
文档中,但模块职责、数据流、调度和实施顺序以本文为准。
|
||||||
|
|
||||||
|
## 当前实施进度
|
||||||
|
|
||||||
|
更新时间:2026-11-21
|
||||||
|
|
||||||
|
整体完成度约 **55%**。
|
||||||
|
|
||||||
|
| 阶段 | 完成度 | 已落地 | 主要缺口 |
|
||||||
|
| --- | ---: | --- | --- |
|
||||||
|
| 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 |
|
||||||
|
| 阶段 1:同步基础 | 85% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器 | points/gold/role/exchange 域尚未全部统一 |
|
||||||
|
| 阶段 2:账号与 CK | 80% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算 | 配置管理 API、更多自动重登灰度指标 |
|
||||||
|
| 阶段 3:商品与供给 | 45% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队 | 默认供给目标、供给看板、覆盖率告警 |
|
||||||
|
| 阶段 4:安全恢复自动化 | 20% | account_sync_queue 与供给补货作业默认小批量开启 | 自动重登灰度参数、成功率熔断和动态预算 |
|
||||||
|
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
|
||||||
|
|
||||||
## 为什么重构
|
## 为什么重构
|
||||||
|
|
||||||
当前能力并不缺,主要问题是职责分散:
|
当前能力并不缺,主要问题是职责分散:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from web.backend.models import (
|
|||||||
Account,
|
Account,
|
||||||
AccountCommodityState,
|
AccountCommodityState,
|
||||||
AccountLease,
|
AccountLease,
|
||||||
|
AccountWorkItem,
|
||||||
AutomationJob,
|
AutomationJob,
|
||||||
PlatformFulfillmentTask,
|
PlatformFulfillmentTask,
|
||||||
User,
|
User,
|
||||||
@@ -124,7 +125,7 @@ def test_no_pending_task_means_no_refresh(db):
|
|||||||
assert job.build_select_query(db, automation_job, NOW).all() == []
|
assert job.build_select_query(db, automation_job, NOW).all() == []
|
||||||
|
|
||||||
|
|
||||||
def test_run_does_not_execute_exact_remaining_count(db, monkeypatch):
|
def test_run_enqueues_stale_accounts_without_external_query(db):
|
||||||
db.add(
|
db.add(
|
||||||
PlatformFulfillmentTask(
|
PlatformFulfillmentTask(
|
||||||
source_system="order_site",
|
source_system="order_site",
|
||||||
@@ -148,19 +149,17 @@ def test_run_does_not_execute_exact_remaining_count(db, monkeypatch):
|
|||||||
batch_size=10,
|
batch_size=10,
|
||||||
concurrency=3,
|
concurrency=3,
|
||||||
)
|
)
|
||||||
captured: dict[str, object] = {}
|
result = job.run_account_capability_refresh(db, automation_job)
|
||||||
|
|
||||||
def fake_run_refresh(_db, _job, accounts, **kwargs):
|
work = (
|
||||||
captured["accounts"] = [row.id for row in accounts]
|
db.query(AccountWorkItem)
|
||||||
captured["remaining_count"] = kwargs.get("remaining_count")
|
.filter_by(account_id=stale.id, work_type="storehome_sync")
|
||||||
return {"scanned": len(accounts)}
|
.one()
|
||||||
|
)
|
||||||
monkeypatch.setattr(job, "run_refresh_for_accounts", fake_run_refresh)
|
assert result["scanned"] == 1
|
||||||
|
assert result["success"] == 1
|
||||||
job.run_account_capability_refresh(db, automation_job)
|
assert work.status == "pending"
|
||||||
|
assert work.priority == 40
|
||||||
assert captured["accounts"] == [stale.id]
|
|
||||||
assert captured["remaining_count"] is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_default_enabled_job_is_created_enabled(db, monkeypatch):
|
def test_default_enabled_job_is_created_enabled(db, monkeypatch):
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""账号供给库存与定向补货测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from web.backend.models import (
|
||||||
|
Account,
|
||||||
|
AccountCommodityState,
|
||||||
|
AccountLease,
|
||||||
|
AccountSupplyTarget,
|
||||||
|
AccountWorkItem,
|
||||||
|
User,
|
||||||
|
)
|
||||||
|
from web.backend.services.account_supply_service import (
|
||||||
|
VALUE_HIGH,
|
||||||
|
enqueue_supply_replenishment,
|
||||||
|
evaluate_supply_target,
|
||||||
|
)
|
||||||
|
|
||||||
|
NOW = datetime.now(UTC).replace(microsecond=0)
|
||||||
|
|
||||||
|
|
||||||
|
def _account(name: str, *, points: int = 0) -> Account:
|
||||||
|
return Account(
|
||||||
|
username=name,
|
||||||
|
password="p",
|
||||||
|
email=f"{name}@example.com",
|
||||||
|
email_password="ep",
|
||||||
|
cookie="acf_uid=100",
|
||||||
|
cookie_updated_at=NOW,
|
||||||
|
ck_check_status="valid",
|
||||||
|
ck_checked_at=NOW,
|
||||||
|
status="active",
|
||||||
|
auto_managed=True,
|
||||||
|
points=points,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _state(account: Account, commodity_id: str = "C") -> AccountCommodityState:
|
||||||
|
return AccountCommodityState(
|
||||||
|
account_id=account.id,
|
||||||
|
platform="douyu",
|
||||||
|
scope="elite",
|
||||||
|
commodity_id=commodity_id,
|
||||||
|
availability="available",
|
||||||
|
observed_at=NOW,
|
||||||
|
valid_until=NOW + timedelta(hours=1),
|
||||||
|
available_from=NOW - timedelta(hours=1),
|
||||||
|
available_until=NOW + timedelta(hours=1),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_supply_inventory_separates_available_and_allocatable(db):
|
||||||
|
user = User(
|
||||||
|
username="supply-owner",
|
||||||
|
password_hash="h",
|
||||||
|
role="super_admin",
|
||||||
|
is_active=True,
|
||||||
|
)
|
||||||
|
first = _account("supply-free", points=400)
|
||||||
|
leased = _account("supply-leased", points=400)
|
||||||
|
db.add_all([user, first, leased])
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
_state(first),
|
||||||
|
_state(leased),
|
||||||
|
AccountLease(
|
||||||
|
account_id=leased.id,
|
||||||
|
holder_user_id=user.id,
|
||||||
|
scope="elite",
|
||||||
|
purpose="platform_fulfillment",
|
||||||
|
status="active",
|
||||||
|
),
|
||||||
|
AccountSupplyTarget(
|
||||||
|
platform="douyu",
|
||||||
|
scope="elite",
|
||||||
|
commodity_id="C",
|
||||||
|
value_class=VALUE_HIGH,
|
||||||
|
target_covered=2,
|
||||||
|
target_available=2,
|
||||||
|
target_allocatable=2,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
target = db.query(AccountSupplyTarget).one()
|
||||||
|
|
||||||
|
inventory = evaluate_supply_target(db, target, now=NOW)
|
||||||
|
|
||||||
|
assert inventory.covered == 2
|
||||||
|
assert inventory.available == 2
|
||||||
|
assert inventory.allocatable == 1
|
||||||
|
assert inventory.allocatable_shortage == 1
|
||||||
|
assert inventory.needs_replenishment is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_supply_shortage_enqueues_missing_coverage(db):
|
||||||
|
account = _account("supply-missing", points=400)
|
||||||
|
db.add(account)
|
||||||
|
db.flush()
|
||||||
|
db.add(
|
||||||
|
AccountSupplyTarget(
|
||||||
|
platform="douyu",
|
||||||
|
scope="elite",
|
||||||
|
commodity_id="C",
|
||||||
|
value_class=VALUE_HIGH,
|
||||||
|
target_covered=1,
|
||||||
|
target_available=1,
|
||||||
|
target_allocatable=0,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
result = enqueue_supply_replenishment(db, now=NOW, max_per_target=10)
|
||||||
|
|
||||||
|
work = db.query(AccountWorkItem).one()
|
||||||
|
assert result.targets == 1
|
||||||
|
assert result.shortages == 1
|
||||||
|
assert result.queued == 1
|
||||||
|
assert work.account_id == account.id
|
||||||
|
assert work.work_type == "storehome_sync"
|
||||||
|
assert work.priority == 35
|
||||||
@@ -0,0 +1,219 @@
|
|||||||
|
"""账号同步队列执行器测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from tests.support.db import make_test_session
|
||||||
|
from web.backend.models import (
|
||||||
|
Account,
|
||||||
|
AccountCommodityState,
|
||||||
|
AccountSyncState,
|
||||||
|
AccountWorkItem,
|
||||||
|
DouyuConfig,
|
||||||
|
)
|
||||||
|
from web.backend.services.account_sync_runner import (
|
||||||
|
WORK_STOREHOME_SYNC,
|
||||||
|
execute_account_work_item,
|
||||||
|
run_account_sync_queue,
|
||||||
|
)
|
||||||
|
from web.backend.services.account_work_queue import (
|
||||||
|
claim_account_work,
|
||||||
|
enqueue_account_work,
|
||||||
|
recover_running_account_work,
|
||||||
|
)
|
||||||
|
from web.backend.services.douyu_read_gateway import StorehomeSnapshot
|
||||||
|
|
||||||
|
|
||||||
|
def _config(db) -> None:
|
||||||
|
db.add(
|
||||||
|
DouyuConfig(
|
||||||
|
manual_id="manual",
|
||||||
|
rid="room",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _account(name: str = "sync-account") -> Account:
|
||||||
|
return Account(
|
||||||
|
username=name,
|
||||||
|
password="p",
|
||||||
|
email=f"{name}@example.com",
|
||||||
|
email_password="ep",
|
||||||
|
cookie="acf_uid=100",
|
||||||
|
cookie_updated_at=datetime.now(UTC),
|
||||||
|
ck_check_status="valid",
|
||||||
|
status="active",
|
||||||
|
auto_managed=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot() -> StorehomeSnapshot:
|
||||||
|
items = (
|
||||||
|
{
|
||||||
|
"commodityId": "limited",
|
||||||
|
"commodityName": "限兑商品",
|
||||||
|
"status": 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"commodityId": "available",
|
||||||
|
"commodityName": "可兑商品",
|
||||||
|
"status": 0,
|
||||||
|
"startTime": str(int((datetime.now(UTC) - timedelta(hours=1)).timestamp())),
|
||||||
|
"endTime": str(int((datetime.now(UTC) + timedelta(hours=1)).timestamp())),
|
||||||
|
"timeLimited": 1,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return StorehomeSnapshot(
|
||||||
|
manual_id="manual",
|
||||||
|
rid="room",
|
||||||
|
observed_at=datetime.now(UTC),
|
||||||
|
items=items,
|
||||||
|
limited_ids=frozenset({"limited"}),
|
||||||
|
raw={"data": {"list": list(items)}},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execute_work_item_syncs_all_goods_and_completes(db):
|
||||||
|
_config(db)
|
||||||
|
account = _account()
|
||||||
|
db.add(account)
|
||||||
|
db.commit()
|
||||||
|
work = enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type=WORK_STOREHOME_SYNC,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
claimed = claim_account_work(
|
||||||
|
db,
|
||||||
|
work_types=(WORK_STOREHOME_SYNC,),
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
assert [item.id for item in claimed] == [work.id]
|
||||||
|
assert claimed[0].attempt_count == 1
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
result = execute_account_work_item(
|
||||||
|
work.id,
|
||||||
|
session_factory=make_test_session,
|
||||||
|
client_factory=lambda _cookie: object(),
|
||||||
|
snapshot_fetcher=lambda *_args, **_kwargs: _snapshot(),
|
||||||
|
)
|
||||||
|
|
||||||
|
with make_test_session() as verify:
|
||||||
|
work_row = verify.get(AccountWorkItem, work.id)
|
||||||
|
states = (
|
||||||
|
verify.query(AccountCommodityState)
|
||||||
|
.filter(AccountCommodityState.account_id == account.id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
sync_state = (
|
||||||
|
verify.query(AccountSyncState)
|
||||||
|
.filter_by(account_id=account.id, domain="storehome")
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
assert result["ok"] is True
|
||||||
|
assert result["item_count"] == 2
|
||||||
|
assert work_row.status == "success"
|
||||||
|
assert len(states) == 2
|
||||||
|
assert sync_state.status == "success"
|
||||||
|
|
||||||
|
|
||||||
|
def test_work_item_failure_returns_to_pending_for_retry(db):
|
||||||
|
_config(db)
|
||||||
|
account = _account("sync-retry")
|
||||||
|
db.add(account)
|
||||||
|
db.commit()
|
||||||
|
work = enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type=WORK_STOREHOME_SYNC,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
claimed = claim_account_work(
|
||||||
|
db,
|
||||||
|
work_types=(WORK_STOREHOME_SYNC,),
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
assert claimed[0].attempt_count == 1
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
def fail(*_args, **_kwargs):
|
||||||
|
raise RuntimeError("network unavailable")
|
||||||
|
|
||||||
|
result = execute_account_work_item(
|
||||||
|
work.id,
|
||||||
|
session_factory=make_test_session,
|
||||||
|
client_factory=lambda _cookie: object(),
|
||||||
|
snapshot_fetcher=fail,
|
||||||
|
)
|
||||||
|
|
||||||
|
with make_test_session() as verify:
|
||||||
|
work_row = verify.get(AccountWorkItem, work.id)
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert work_row.status == "pending"
|
||||||
|
assert work_row.attempt_count == 1
|
||||||
|
assert "network unavailable" in work_row.last_error
|
||||||
|
assert work_row.due_at > datetime.now(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_account_sync_queue_claims_and_executes(db):
|
||||||
|
_config(db)
|
||||||
|
account = _account("sync-runner")
|
||||||
|
db.add(account)
|
||||||
|
db.commit()
|
||||||
|
enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type=WORK_STOREHOME_SYNC,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
result = run_account_sync_queue(
|
||||||
|
db,
|
||||||
|
batch_size=5,
|
||||||
|
session_factory=make_test_session,
|
||||||
|
client_factory=lambda _cookie: object(),
|
||||||
|
snapshot_fetcher=lambda *_args, **_kwargs: _snapshot(),
|
||||||
|
)
|
||||||
|
|
||||||
|
with make_test_session() as verify:
|
||||||
|
work = (
|
||||||
|
verify.query(AccountWorkItem)
|
||||||
|
.filter_by(account_id=account.id, work_type=WORK_STOREHOME_SYNC)
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
assert result["scanned"] == 1
|
||||||
|
assert result["success"] == 1
|
||||||
|
assert work.status == "success"
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_recovers_running_work(db):
|
||||||
|
account = _account("sync-recovery")
|
||||||
|
db.add(account)
|
||||||
|
db.commit()
|
||||||
|
work = enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=account.id,
|
||||||
|
work_type=WORK_STOREHOME_SYNC,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
claim_account_work(
|
||||||
|
db,
|
||||||
|
work_types=(WORK_STOREHOME_SYNC,),
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
recovered = recover_running_account_work(db)
|
||||||
|
|
||||||
|
with make_test_session() as verify:
|
||||||
|
row = verify.get(AccountWorkItem, work.id)
|
||||||
|
assert recovered == 1
|
||||||
|
assert row.status == "pending"
|
||||||
|
assert row.due_at <= datetime.now(UTC)
|
||||||
|
assert "服务重启" in row.last_error
|
||||||
@@ -408,7 +408,8 @@ class TestScheduler(_Base):
|
|||||||
self.session.commit()
|
self.session.commit()
|
||||||
|
|
||||||
outcomes = run_due_jobs(self.session)
|
outcomes = run_due_jobs(self.session)
|
||||||
assert outcomes[0]["status"] == "failed"
|
failed = next(item for item in outcomes if item["job_key"] == "boom")
|
||||||
|
assert failed["status"] == "failed"
|
||||||
run = (
|
run = (
|
||||||
self.session.query(AutomationRun)
|
self.session.query(AutomationRun)
|
||||||
.filter(AutomationRun.job_key == "boom")
|
.filter(AutomationRun.job_key == "boom")
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ async def lifespan(app: FastAPI):
|
|||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
from .database import SessionLocal, dispose_engines
|
from .database import SessionLocal, dispose_engines
|
||||||
|
from .services.account_work_queue import recover_running_account_work
|
||||||
from .services.douyu_service import cleanup_orphan_douyu_tasks
|
from .services.douyu_service import cleanup_orphan_douyu_tasks
|
||||||
from .services.huya_service import cleanup_orphan_huya_tasks
|
from .services.huya_service import cleanup_orphan_huya_tasks
|
||||||
from .services.login_quota_service import cleanup_orphan_login_attempts
|
from .services.login_quota_service import cleanup_orphan_login_attempts
|
||||||
@@ -88,6 +89,9 @@ async def lifespan(app: FastAPI):
|
|||||||
cleaned_attempts = cleanup_orphan_login_attempts(db)
|
cleaned_attempts = cleanup_orphan_login_attempts(db)
|
||||||
if cleaned_attempts:
|
if cleaned_attempts:
|
||||||
logger.info(f"启动收敛中断的登录尝试流水: {cleaned_attempts} 条")
|
logger.info(f"启动收敛中断的登录尝试流水: {cleaned_attempts} 条")
|
||||||
|
recovered_work = recover_running_account_work(db)
|
||||||
|
if recovered_work:
|
||||||
|
logger.info(f"启动恢复中断的账号内部工作项: {recovered_work} 条")
|
||||||
from .services.yyb_service import cleanup_orphan_yyb_tasks
|
from .services.yyb_service import cleanup_orphan_yyb_tasks
|
||||||
|
|
||||||
cleaned_yyb = cleanup_orphan_yyb_tasks(db, message="任务已中断(服务重启)")
|
cleaned_yyb = cleanup_orphan_yyb_tasks(db, message="任务已中断(服务重启)")
|
||||||
|
|||||||
@@ -0,0 +1,392 @@
|
|||||||
|
"""斗鱼账号供给库存、目标缺口和定向补货入队。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from sqlalchemy import and_, or_, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..models import (
|
||||||
|
Account,
|
||||||
|
AccountCommodityState,
|
||||||
|
AccountLease,
|
||||||
|
AccountSupplyTarget,
|
||||||
|
AccountSyncState,
|
||||||
|
DouyuExternalOrder,
|
||||||
|
)
|
||||||
|
from .account_capability_service import AVAILABILITY_AVAILABLE
|
||||||
|
from .account_lifecycle import MANUAL_STATES
|
||||||
|
from .account_sync_runner import WORK_STOREHOME_SYNC
|
||||||
|
from .account_value import is_high_value_expr
|
||||||
|
from .account_work_queue import enqueue_account_work
|
||||||
|
|
||||||
|
VALUE_HIGH = "high_value"
|
||||||
|
VALUE_ORDINARY = "ordinary"
|
||||||
|
VALID_VALUES = frozenset({VALUE_HIGH, VALUE_ORDINARY})
|
||||||
|
CK_FRESH_SECONDS = 24 * 3600
|
||||||
|
ACTIVE_COLLAB_STATUSES = (
|
||||||
|
"incomplete",
|
||||||
|
"to_recharge",
|
||||||
|
"recharged",
|
||||||
|
"scripted",
|
||||||
|
"points_error",
|
||||||
|
"role_error",
|
||||||
|
"failed",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SupplyInventory:
|
||||||
|
target_id: int
|
||||||
|
scope: str
|
||||||
|
commodity_id: str
|
||||||
|
value_class: str
|
||||||
|
covered: int
|
||||||
|
available: int
|
||||||
|
allocatable: int
|
||||||
|
target_covered: int
|
||||||
|
target_available: int
|
||||||
|
target_allocatable: int
|
||||||
|
safety_buffer: int
|
||||||
|
covered_shortage: int
|
||||||
|
available_shortage: int
|
||||||
|
allocatable_shortage: int
|
||||||
|
|
||||||
|
@property
|
||||||
|
def needs_replenishment(self) -> bool:
|
||||||
|
return bool(
|
||||||
|
self.covered_shortage
|
||||||
|
or self.available_shortage
|
||||||
|
or self.allocatable_shortage
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SupplyReplenishmentResult:
|
||||||
|
targets: int
|
||||||
|
queued: int
|
||||||
|
shortages: int
|
||||||
|
|
||||||
|
|
||||||
|
def _active_lease_exists():
|
||||||
|
return (
|
||||||
|
select(AccountLease.id)
|
||||||
|
.where(
|
||||||
|
AccountLease.account_id == Account.id,
|
||||||
|
AccountLease.status == "active",
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _active_collab_exists():
|
||||||
|
return (
|
||||||
|
select(DouyuExternalOrder.id)
|
||||||
|
.where(
|
||||||
|
DouyuExternalOrder.account_id == Account.id,
|
||||||
|
DouyuExternalOrder.archived_at.is_(None),
|
||||||
|
DouyuExternalOrder.status.in_(ACTIVE_COLLAB_STATUSES),
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _value_filter(value_class: str, *, now: datetime):
|
||||||
|
high_value = is_high_value_expr(now=now)
|
||||||
|
if value_class == VALUE_HIGH:
|
||||||
|
return high_value
|
||||||
|
if value_class == VALUE_ORDINARY:
|
||||||
|
return ~high_value
|
||||||
|
raise ValueError(f"不支持的账号价值分类: {value_class}")
|
||||||
|
|
||||||
|
|
||||||
|
def _base_account_query(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
now: datetime,
|
||||||
|
value_class: str,
|
||||||
|
):
|
||||||
|
ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS)
|
||||||
|
return (
|
||||||
|
db.query(Account)
|
||||||
|
.filter(
|
||||||
|
Account.cookie.isnot(None),
|
||||||
|
Account.cookie != "",
|
||||||
|
Account.ck_check_status == "valid",
|
||||||
|
Account.ck_checked_at.isnot(None),
|
||||||
|
Account.ck_checked_at >= ck_fresh_cutoff,
|
||||||
|
Account.auto_managed.is_(True),
|
||||||
|
Account.status.notin_(MANUAL_STATES),
|
||||||
|
_value_filter(value_class, now=now),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _allocatable_filter(now: datetime):
|
||||||
|
return and_(
|
||||||
|
~_active_lease_exists(),
|
||||||
|
~_active_collab_exists(),
|
||||||
|
or_(
|
||||||
|
Account.role_change_available_at.is_(None),
|
||||||
|
Account.role_change_available_at <= now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _fresh_available_filter(now: datetime):
|
||||||
|
return and_(
|
||||||
|
AccountCommodityState.availability == AVAILABILITY_AVAILABLE,
|
||||||
|
AccountCommodityState.valid_until.isnot(None),
|
||||||
|
AccountCommodityState.valid_until > now,
|
||||||
|
or_(
|
||||||
|
AccountCommodityState.available_from.is_(None),
|
||||||
|
AccountCommodityState.available_from <= now,
|
||||||
|
),
|
||||||
|
or_(
|
||||||
|
AccountCommodityState.available_until.is_(None),
|
||||||
|
AccountCommodityState.available_until >= now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _commodity_query(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
now: datetime,
|
||||||
|
target: AccountSupplyTarget,
|
||||||
|
):
|
||||||
|
query = _base_account_query(
|
||||||
|
db,
|
||||||
|
now=now,
|
||||||
|
value_class=target.value_class,
|
||||||
|
).join(
|
||||||
|
AccountCommodityState,
|
||||||
|
AccountCommodityState.account_id == Account.id,
|
||||||
|
).filter(
|
||||||
|
AccountCommodityState.platform == target.platform,
|
||||||
|
AccountCommodityState.scope == target.scope,
|
||||||
|
AccountCommodityState.commodity_id == target.commodity_id,
|
||||||
|
AccountCommodityState.observed_at.isnot(None),
|
||||||
|
)
|
||||||
|
return query
|
||||||
|
|
||||||
|
|
||||||
|
def _global_state_exists(target: AccountSupplyTarget):
|
||||||
|
return (
|
||||||
|
select(AccountCommodityState.id)
|
||||||
|
.where(
|
||||||
|
AccountCommodityState.account_id == Account.id,
|
||||||
|
AccountCommodityState.platform == target.platform,
|
||||||
|
AccountCommodityState.scope == target.scope,
|
||||||
|
AccountCommodityState.observed_at.isnot(None),
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _global_available_exists(target: AccountSupplyTarget, *, now: datetime):
|
||||||
|
return (
|
||||||
|
select(AccountCommodityState.id)
|
||||||
|
.where(
|
||||||
|
AccountCommodityState.account_id == Account.id,
|
||||||
|
AccountCommodityState.platform == target.platform,
|
||||||
|
AccountCommodityState.scope == target.scope,
|
||||||
|
_fresh_available_filter(now),
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_supply_target(
|
||||||
|
db: Session,
|
||||||
|
target: AccountSupplyTarget,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> SupplyInventory:
|
||||||
|
"""计算一条供给目标的 covered / available / allocatable 库存。"""
|
||||||
|
moment = now or datetime.now(UTC)
|
||||||
|
if target.value_class not in VALID_VALUES:
|
||||||
|
raise ValueError(f"不支持的账号价值分类: {target.value_class}")
|
||||||
|
|
||||||
|
if target.commodity_id:
|
||||||
|
base = _commodity_query(db, now=moment, target=target)
|
||||||
|
covered = base.count()
|
||||||
|
available_query = base.filter(_fresh_available_filter(moment))
|
||||||
|
available = available_query.count()
|
||||||
|
allocatable = available_query.filter(
|
||||||
|
_allocatable_filter(moment)
|
||||||
|
).count()
|
||||||
|
else:
|
||||||
|
base = _base_account_query(
|
||||||
|
db,
|
||||||
|
now=moment,
|
||||||
|
value_class=target.value_class,
|
||||||
|
)
|
||||||
|
covered = base.filter(_global_state_exists(target)).count()
|
||||||
|
available_query = base.filter(
|
||||||
|
_global_available_exists(target, now=moment)
|
||||||
|
)
|
||||||
|
available = available_query.count()
|
||||||
|
allocatable = available_query.filter(
|
||||||
|
_allocatable_filter(moment)
|
||||||
|
).count()
|
||||||
|
|
||||||
|
return SupplyInventory(
|
||||||
|
target_id=int(target.id),
|
||||||
|
scope=target.scope,
|
||||||
|
commodity_id=target.commodity_id,
|
||||||
|
value_class=target.value_class,
|
||||||
|
covered=covered,
|
||||||
|
available=available,
|
||||||
|
allocatable=allocatable,
|
||||||
|
target_covered=int(target.target_covered or 0),
|
||||||
|
target_available=int(target.target_available or 0),
|
||||||
|
target_allocatable=int(target.target_allocatable or 0),
|
||||||
|
safety_buffer=int(target.safety_buffer or 0),
|
||||||
|
covered_shortage=max(
|
||||||
|
0,
|
||||||
|
int(target.target_covered or 0) + int(target.safety_buffer or 0)
|
||||||
|
- covered,
|
||||||
|
),
|
||||||
|
available_shortage=max(
|
||||||
|
0,
|
||||||
|
int(target.target_available or 0) + int(target.safety_buffer or 0)
|
||||||
|
- available,
|
||||||
|
),
|
||||||
|
allocatable_shortage=max(
|
||||||
|
0,
|
||||||
|
int(target.target_allocatable or 0) + int(target.safety_buffer or 0)
|
||||||
|
- allocatable,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_supply_targets(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> list[SupplyInventory]:
|
||||||
|
moment = now or datetime.now(UTC)
|
||||||
|
targets = (
|
||||||
|
db.query(AccountSupplyTarget)
|
||||||
|
.filter(AccountSupplyTarget.enabled.is_(True))
|
||||||
|
.order_by(
|
||||||
|
AccountSupplyTarget.platform.asc(),
|
||||||
|
AccountSupplyTarget.scope.asc(),
|
||||||
|
AccountSupplyTarget.value_class.asc(),
|
||||||
|
AccountSupplyTarget.commodity_id.asc(),
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
evaluate_supply_target(db, target, now=moment)
|
||||||
|
for target in targets
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _sync_candidate_query(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
target: AccountSupplyTarget,
|
||||||
|
now: datetime,
|
||||||
|
):
|
||||||
|
query = _base_account_query(
|
||||||
|
db,
|
||||||
|
now=now,
|
||||||
|
value_class=target.value_class,
|
||||||
|
)
|
||||||
|
if target.commodity_id:
|
||||||
|
state_exists = (
|
||||||
|
select(AccountCommodityState.id)
|
||||||
|
.where(
|
||||||
|
AccountCommodityState.account_id == Account.id,
|
||||||
|
AccountCommodityState.platform == target.platform,
|
||||||
|
AccountCommodityState.scope == target.scope,
|
||||||
|
AccountCommodityState.commodity_id == target.commodity_id,
|
||||||
|
AccountCommodityState.valid_until.isnot(None),
|
||||||
|
AccountCommodityState.valid_until > now,
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
state_exists = (
|
||||||
|
select(AccountSyncState.id)
|
||||||
|
.where(
|
||||||
|
AccountSyncState.account_id == Account.id,
|
||||||
|
AccountSyncState.platform == target.platform,
|
||||||
|
AccountSyncState.domain == "storehome",
|
||||||
|
AccountSyncState.status == "success",
|
||||||
|
AccountSyncState.valid_until.isnot(None),
|
||||||
|
AccountSyncState.valid_until > now,
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
return query.filter(~state_exists).order_by(
|
||||||
|
Account.overview_queried_at.is_(None).desc(),
|
||||||
|
Account.overview_queried_at.asc(),
|
||||||
|
Account.id.asc(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def enqueue_supply_replenishment(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
max_per_target: int = 50,
|
||||||
|
) -> SupplyReplenishmentResult:
|
||||||
|
"""按本地目标缺口定向入队 storehome 同步,不发起外部请求。"""
|
||||||
|
moment = now or datetime.now(UTC)
|
||||||
|
inventories = evaluate_supply_targets(db, now=moment)
|
||||||
|
queued = 0
|
||||||
|
shortages = 0
|
||||||
|
for inventory in inventories:
|
||||||
|
if not inventory.needs_replenishment:
|
||||||
|
continue
|
||||||
|
shortages += 1
|
||||||
|
target = db.get(AccountSupplyTarget, inventory.target_id)
|
||||||
|
if target is None:
|
||||||
|
continue
|
||||||
|
missing = max(
|
||||||
|
inventory.covered_shortage,
|
||||||
|
inventory.available_shortage,
|
||||||
|
inventory.allocatable_shortage,
|
||||||
|
)
|
||||||
|
limit = max(1, min(int(max_per_target), missing))
|
||||||
|
accounts = _sync_candidate_query(
|
||||||
|
db,
|
||||||
|
target=target,
|
||||||
|
now=moment,
|
||||||
|
).limit(limit).all()
|
||||||
|
for account in accounts:
|
||||||
|
enqueue_account_work(
|
||||||
|
db,
|
||||||
|
account_id=int(account.id),
|
||||||
|
work_type=WORK_STOREHOME_SYNC,
|
||||||
|
priority=35,
|
||||||
|
due_at=moment,
|
||||||
|
reason=(
|
||||||
|
f"供给短缺: {target.scope}/{target.commodity_id or 'all'}"
|
||||||
|
f"/{target.value_class}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
queued += 1
|
||||||
|
if queued:
|
||||||
|
db.commit()
|
||||||
|
return SupplyReplenishmentResult(
|
||||||
|
targets=len(inventories),
|
||||||
|
queued=queued,
|
||||||
|
shortages=shortages,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"VALUE_HIGH",
|
||||||
|
"VALUE_ORDINARY",
|
||||||
|
"SupplyInventory",
|
||||||
|
"SupplyReplenishmentResult",
|
||||||
|
"enqueue_supply_replenishment",
|
||||||
|
"evaluate_supply_target",
|
||||||
|
"evaluate_supply_targets",
|
||||||
|
]
|
||||||
@@ -0,0 +1,200 @@
|
|||||||
|
"""账号内部同步队列执行器。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from concurrent.futures import as_completed
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..database import WorkerSessionLocal
|
||||||
|
from ..models import Account, AccountWorkItem
|
||||||
|
from .account_sync_service import DOMAIN_STOREHOME, sync_storehome_snapshot
|
||||||
|
from .account_work_queue import (
|
||||||
|
claim_account_work,
|
||||||
|
complete_account_work,
|
||||||
|
fail_account_work,
|
||||||
|
)
|
||||||
|
from .douyu_activity_query import activity_client
|
||||||
|
from .douyu_read_gateway import StorehomeSnapshot, fetch_storehome_snapshot
|
||||||
|
from .douyu_service import ensure_douyu_config
|
||||||
|
from .task_runtime import TaskLane, available_slots, submit
|
||||||
|
|
||||||
|
WORK_STOREHOME_SYNC = DOMAIN_STOREHOME + "_sync"
|
||||||
|
ACCOUNT_SYNC_WORK_TYPES = (WORK_STOREHOME_SYNC,)
|
||||||
|
|
||||||
|
|
||||||
|
def _execute_storehome_sync(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
account: Account,
|
||||||
|
item: AccountWorkItem,
|
||||||
|
client_factory: Callable[[str], Any],
|
||||||
|
snapshot_fetcher: Callable[..., StorehomeSnapshot],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not account.cookie:
|
||||||
|
raise RuntimeError("账号没有可用 CK,不能同步商品能力")
|
||||||
|
if account.ck_check_status in {"invalid", "manual"}:
|
||||||
|
raise RuntimeError(f"账号 CK 状态为 {account.ck_check_status},暂不能同步商品能力")
|
||||||
|
config = ensure_douyu_config(db)
|
||||||
|
manual_id = str(getattr(config, "manual_id", "") or "")
|
||||||
|
rid = str(getattr(config, "rid", "") or "")
|
||||||
|
if not manual_id:
|
||||||
|
raise RuntimeError("未配置精英宝典 manualID")
|
||||||
|
client = client_factory(account.cookie)
|
||||||
|
snapshot = snapshot_fetcher(
|
||||||
|
client,
|
||||||
|
manual_id=manual_id,
|
||||||
|
rid=rid,
|
||||||
|
)
|
||||||
|
result = sync_storehome_snapshot(
|
||||||
|
db,
|
||||||
|
account,
|
||||||
|
snapshot,
|
||||||
|
source="account_work.storehome_sync",
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"work_id": int(item.id),
|
||||||
|
"account_id": int(account.id),
|
||||||
|
"item_count": result.item_count,
|
||||||
|
"changed_count": result.changed_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def execute_account_work_item(
|
||||||
|
work_id: int,
|
||||||
|
*,
|
||||||
|
session_factory: Callable[[], Session] = WorkerSessionLocal,
|
||||||
|
client_factory: Callable[[str], Any] = activity_client,
|
||||||
|
snapshot_fetcher: Callable[..., StorehomeSnapshot] = fetch_storehome_snapshot,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""在一个短事务中执行单条账号维护工作。"""
|
||||||
|
session = session_factory()
|
||||||
|
item: AccountWorkItem | None = None
|
||||||
|
try:
|
||||||
|
item = session.get(AccountWorkItem, int(work_id))
|
||||||
|
if item is None or item.status != "running":
|
||||||
|
return {"ok": False, "work_id": int(work_id), "skipped": True}
|
||||||
|
account = session.get(Account, int(item.account_id))
|
||||||
|
if account is None:
|
||||||
|
raise RuntimeError("工作项对应账号不存在")
|
||||||
|
if item.work_type != WORK_STOREHOME_SYNC:
|
||||||
|
raise RuntimeError(f"不支持的工作类型: {item.work_type}")
|
||||||
|
result = _execute_storehome_sync(
|
||||||
|
session,
|
||||||
|
account=account,
|
||||||
|
item=item,
|
||||||
|
client_factory=client_factory,
|
||||||
|
snapshot_fetcher=snapshot_fetcher,
|
||||||
|
)
|
||||||
|
complete_account_work(session, item)
|
||||||
|
session.commit()
|
||||||
|
return result
|
||||||
|
except Exception as exc: # noqa: BLE001 - 单工作项失败必须收敛到队列状态
|
||||||
|
if not session.is_active:
|
||||||
|
session.rollback()
|
||||||
|
item = session.get(AccountWorkItem, int(work_id))
|
||||||
|
if item is not None:
|
||||||
|
fail_account_work(
|
||||||
|
session,
|
||||||
|
item,
|
||||||
|
error=exc,
|
||||||
|
retry_after_seconds=15 * 60,
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return {
|
||||||
|
"ok": False,
|
||||||
|
"work_id": int(work_id),
|
||||||
|
"error": f"{type(exc).__name__}: {exc}"[:500],
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
|
def run_account_sync_queue(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
batch_size: int = 20,
|
||||||
|
now: datetime | None = None,
|
||||||
|
session_factory: Callable[[], Session] = WorkerSessionLocal,
|
||||||
|
client_factory: Callable[[str], Any] = activity_client,
|
||||||
|
snapshot_fetcher: Callable[..., StorehomeSnapshot] = fetch_storehome_snapshot,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""领取少量到期工作并通过独立账号同步槽执行。"""
|
||||||
|
moment = now or datetime.now(UTC)
|
||||||
|
capacity = available_slots(TaskLane.ACCOUNT_SYNC)
|
||||||
|
if capacity <= 0:
|
||||||
|
return {
|
||||||
|
"scanned": 0,
|
||||||
|
"success": 0,
|
||||||
|
"failed": 0,
|
||||||
|
"message": "账号同步槽已满,本轮跳过",
|
||||||
|
}
|
||||||
|
limit = max(1, min(int(batch_size), capacity))
|
||||||
|
items = claim_account_work(
|
||||||
|
db,
|
||||||
|
work_types=ACCOUNT_SYNC_WORK_TYPES,
|
||||||
|
limit=limit,
|
||||||
|
now=moment,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
if not items:
|
||||||
|
return {
|
||||||
|
"scanned": 0,
|
||||||
|
"success": 0,
|
||||||
|
"failed": 0,
|
||||||
|
"message": "暂无到期的账号同步工作",
|
||||||
|
}
|
||||||
|
|
||||||
|
futures = {}
|
||||||
|
for item in items:
|
||||||
|
work_id = int(item.id)
|
||||||
|
future = submit(
|
||||||
|
TaskLane.ACCOUNT_SYNC,
|
||||||
|
execute_account_work_item,
|
||||||
|
work_id,
|
||||||
|
session_factory=session_factory,
|
||||||
|
client_factory=client_factory,
|
||||||
|
snapshot_fetcher=snapshot_fetcher,
|
||||||
|
)
|
||||||
|
if future is not None:
|
||||||
|
futures[future] = work_id
|
||||||
|
|
||||||
|
success = 0
|
||||||
|
failed = 0
|
||||||
|
skipped = 0
|
||||||
|
for future in as_completed(futures):
|
||||||
|
try:
|
||||||
|
result = future.result()
|
||||||
|
except Exception: # noqa: BLE001 - worker 异常已由执行函数收敛,这里只计数
|
||||||
|
failed += 1
|
||||||
|
continue
|
||||||
|
if result.get("ok"):
|
||||||
|
success += 1
|
||||||
|
elif result.get("skipped"):
|
||||||
|
skipped += 1
|
||||||
|
else:
|
||||||
|
failed += 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"scanned": len(futures),
|
||||||
|
"success": success,
|
||||||
|
"failed": failed,
|
||||||
|
"skipped": skipped,
|
||||||
|
"message": (
|
||||||
|
f"账号同步完成 {success} 条"
|
||||||
|
+ (f",失败 {failed} 条" if failed else "")
|
||||||
|
+ (f",跳过 {skipped} 条" if skipped else "")
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ACCOUNT_SYNC_WORK_TYPES",
|
||||||
|
"WORK_STOREHOME_SYNC",
|
||||||
|
"execute_account_work_item",
|
||||||
|
"run_account_sync_queue",
|
||||||
|
]
|
||||||
@@ -42,7 +42,9 @@ def enqueue_account_work(
|
|||||||
account_id=account_id,
|
account_id=account_id,
|
||||||
work_type=work_type[:32],
|
work_type=work_type[:32],
|
||||||
priority=int(priority),
|
priority=int(priority),
|
||||||
due_at=due_at or datetime.now(UTC),
|
# MySQL DATETIME 不保存微秒;立即到期的工作项必须落在当前秒,
|
||||||
|
# 否则四舍五入到下一秒后,本轮 claim 会误判“还没到期”。
|
||||||
|
due_at=(due_at or datetime.now(UTC)).replace(microsecond=0),
|
||||||
max_attempts=max(1, int(max_attempts)),
|
max_attempts=max(1, int(max_attempts)),
|
||||||
dedupe_key=dedupe_key[:191],
|
dedupe_key=dedupe_key[:191],
|
||||||
reason=reason[:255],
|
reason=reason[:255],
|
||||||
@@ -149,14 +151,39 @@ def fail_account_work(
|
|||||||
item.finished_at = moment
|
item.finished_at = moment
|
||||||
else:
|
else:
|
||||||
item.status = "pending"
|
item.status = "pending"
|
||||||
item.due_at = moment + timedelta(seconds=max(1, retry_after_seconds))
|
item.due_at = (
|
||||||
|
moment + timedelta(seconds=max(1, retry_after_seconds))
|
||||||
|
).replace(microsecond=0)
|
||||||
db.flush()
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def recover_running_account_work(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""服务重启后把中断的 running 工作项退回 pending。"""
|
||||||
|
moment = (now or datetime.now(UTC)).replace(microsecond=0)
|
||||||
|
items = (
|
||||||
|
db.query(AccountWorkItem)
|
||||||
|
.filter(AccountWorkItem.status == "running")
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for item in items:
|
||||||
|
item.status = "pending"
|
||||||
|
item.due_at = moment
|
||||||
|
item.last_error = "工作执行中断(服务重启),已重新排队"
|
||||||
|
item.updated_at = moment
|
||||||
|
if items:
|
||||||
|
db.commit()
|
||||||
|
return len(items)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"claim_account_work",
|
"claim_account_work",
|
||||||
"complete_account_work",
|
"complete_account_work",
|
||||||
"complete_account_work_type",
|
"complete_account_work_type",
|
||||||
"enqueue_account_work",
|
"enqueue_account_work",
|
||||||
"fail_account_work",
|
"fail_account_work",
|
||||||
|
"recover_running_account_work",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -18,6 +18,18 @@ from .jobs.account_overview_refresh import (
|
|||||||
from .jobs.account_overview_refresh import (
|
from .jobs.account_overview_refresh import (
|
||||||
run_account_overview_refresh,
|
run_account_overview_refresh,
|
||||||
)
|
)
|
||||||
|
from .jobs.account_supply_replenishment import (
|
||||||
|
JOB_KEY as ACCOUNT_SUPPLY_REPLENISHMENT_KEY,
|
||||||
|
)
|
||||||
|
from .jobs.account_supply_replenishment import (
|
||||||
|
run_account_supply_replenishment,
|
||||||
|
)
|
||||||
|
from .jobs.account_sync_queue import (
|
||||||
|
JOB_KEY as ACCOUNT_SYNC_QUEUE_KEY,
|
||||||
|
)
|
||||||
|
from .jobs.account_sync_queue import (
|
||||||
|
run_account_sync_queue_job,
|
||||||
|
)
|
||||||
from .jobs.ck_health import JOB_KEY as CK_HEALTH_KEY
|
from .jobs.ck_health import JOB_KEY as CK_HEALTH_KEY
|
||||||
from .jobs.ck_health import run_ck_health
|
from .jobs.ck_health import run_ck_health
|
||||||
from .jobs.ck_relogin import JOB_KEY as CK_RELOGIN_KEY
|
from .jobs.ck_relogin import JOB_KEY as CK_RELOGIN_KEY
|
||||||
@@ -53,6 +65,8 @@ from .scheduler import (
|
|||||||
__all__ = [
|
__all__ = [
|
||||||
"ACCOUNT_CAPABILITY_REFRESH_KEY",
|
"ACCOUNT_CAPABILITY_REFRESH_KEY",
|
||||||
"ACCOUNT_OVERVIEW_REFRESH_KEY",
|
"ACCOUNT_OVERVIEW_REFRESH_KEY",
|
||||||
|
"ACCOUNT_SUPPLY_REPLENISHMENT_KEY",
|
||||||
|
"ACCOUNT_SYNC_QUEUE_KEY",
|
||||||
"CK_HEALTH_KEY",
|
"CK_HEALTH_KEY",
|
||||||
"CK_RELOGIN_KEY",
|
"CK_RELOGIN_KEY",
|
||||||
"DOUYU_TASKS_RETENTION_KEY",
|
"DOUYU_TASKS_RETENTION_KEY",
|
||||||
@@ -67,6 +81,8 @@ __all__ = [
|
|||||||
"reset_registry",
|
"reset_registry",
|
||||||
"run_account_capability_refresh",
|
"run_account_capability_refresh",
|
||||||
"run_account_overview_refresh",
|
"run_account_overview_refresh",
|
||||||
|
"run_account_supply_replenishment",
|
||||||
|
"run_account_sync_queue_job",
|
||||||
"run_ck_health",
|
"run_ck_health",
|
||||||
"run_ck_relogin",
|
"run_ck_relogin",
|
||||||
"run_douyu_tasks_retention",
|
"run_douyu_tasks_retention",
|
||||||
@@ -82,6 +98,30 @@ __all__ = [
|
|||||||
|
|
||||||
def register_default_jobs() -> None:
|
def register_default_jobs() -> None:
|
||||||
"""登记内置作业(幂等)。"""
|
"""登记内置作业(幂等)。"""
|
||||||
|
register_job(
|
||||||
|
JobSpec(
|
||||||
|
key=ACCOUNT_SYNC_QUEUE_KEY,
|
||||||
|
title="账号内部同步队列",
|
||||||
|
interval_seconds=60,
|
||||||
|
run_interval_seconds=60,
|
||||||
|
batch_size=20,
|
||||||
|
concurrency=3,
|
||||||
|
run=run_account_sync_queue_job,
|
||||||
|
default_enabled=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
register_job(
|
||||||
|
JobSpec(
|
||||||
|
key=ACCOUNT_SUPPLY_REPLENISHMENT_KEY,
|
||||||
|
title="账号供给目标补货",
|
||||||
|
interval_seconds=5 * 60,
|
||||||
|
run_interval_seconds=5 * 60,
|
||||||
|
batch_size=50,
|
||||||
|
concurrency=1,
|
||||||
|
run=run_account_supply_replenishment,
|
||||||
|
default_enabled=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
register_job(
|
register_job(
|
||||||
JobSpec(
|
JobSpec(
|
||||||
key=ACCOUNT_CAPABILITY_REFRESH_KEY,
|
key=ACCOUNT_CAPABILITY_REFRESH_KEY,
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
"""按直播履约需求刷新账号商品能力。
|
"""按直播履约需求把过期商品能力账号送入内部同步队列。
|
||||||
|
|
||||||
全量资料补全仍由 ``account_overview_refresh`` 负责;本作业只处理“有 pending
|
本作业只做本地候选筛选,不直接请求官网。真正的 ``storehome`` 查询由
|
||||||
任务、但对应商品能力快照已经过期”的账号,避免为了秒匹配把两万多个账号
|
``account_sync_queue`` 作业在独立执行槽中低并发消费,避免补货查询抢占
|
||||||
无条件轮询一遍。
|
履约实时复核、登录或批量任务的执行资源。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -21,8 +21,9 @@ from ....models import (
|
|||||||
PlatformFulfillmentTask,
|
PlatformFulfillmentTask,
|
||||||
)
|
)
|
||||||
from ...account_lifecycle import MANUAL_STATES
|
from ...account_lifecycle import MANUAL_STATES
|
||||||
|
from ...account_sync_runner import WORK_STOREHOME_SYNC
|
||||||
|
from ...account_work_queue import enqueue_account_work
|
||||||
from ...douyu_service import cookie_operable_filter
|
from ...douyu_service import cookie_operable_filter
|
||||||
from .account_overview_refresh import run_refresh_for_accounts
|
|
||||||
|
|
||||||
JOB_KEY = "account_capability_refresh"
|
JOB_KEY = "account_capability_refresh"
|
||||||
|
|
||||||
@@ -94,18 +95,29 @@ def run_account_capability_refresh(
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
accounts = build_select_query(db, job, now).all()
|
accounts = build_select_query(db, job, now).all()
|
||||||
return run_refresh_for_accounts(
|
for account in accounts:
|
||||||
db,
|
enqueue_account_work(
|
||||||
job,
|
db,
|
||||||
accounts,
|
account_id=int(account.id),
|
||||||
now=now,
|
work_type=WORK_STOREHOME_SYNC,
|
||||||
# 精确剩余量要完整评估 accounts × commodity EXISTS,生产实测单次约
|
priority=40,
|
||||||
# 1.3 秒、扫描 38 万行;它只用于展示,直接省略,避免每轮制造慢查询。
|
reason="履约商品能力过期,等待补货刷新",
|
||||||
remaining_count=None,
|
due_at=now,
|
||||||
source="capability_demand_refresh",
|
)
|
||||||
log_label="capability",
|
db.commit()
|
||||||
empty_message="当前履约商品没有到期待刷新的账号能力",
|
queued = len(accounts)
|
||||||
)
|
return {
|
||||||
|
"scanned": len(accounts),
|
||||||
|
"success": queued,
|
||||||
|
"invalid": 0,
|
||||||
|
"skipped": 0,
|
||||||
|
"message": (
|
||||||
|
f"本轮筛选 {len(accounts)} 个到期账号,新入队 {queued} 个"
|
||||||
|
if accounts
|
||||||
|
else "当前履约商品没有到期待刷新的账号能力"
|
||||||
|
),
|
||||||
|
"detail": {"queued": queued},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""按账号供给目标缺口定向补货入队。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ....models import AutomationJob
|
||||||
|
from ...account_supply_service import enqueue_supply_replenishment
|
||||||
|
|
||||||
|
JOB_KEY = "account_supply_replenishment"
|
||||||
|
|
||||||
|
|
||||||
|
def run_account_supply_replenishment(
|
||||||
|
db: Session,
|
||||||
|
job: AutomationJob,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
result = enqueue_supply_replenishment(
|
||||||
|
db,
|
||||||
|
max_per_target=max(1, int(job.batch_size or 50)),
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"scanned": result.targets,
|
||||||
|
"success": result.queued,
|
||||||
|
"invalid": 0,
|
||||||
|
"skipped": max(0, result.targets - result.shortages),
|
||||||
|
"message": (
|
||||||
|
f"评估 {result.targets} 个供给目标,"
|
||||||
|
f"短缺 {result.shortages} 个,新排队 {result.queued} 个账号同步"
|
||||||
|
if result.targets
|
||||||
|
else "未配置账号供给目标"
|
||||||
|
),
|
||||||
|
"detail": {
|
||||||
|
"shortages": result.shortages,
|
||||||
|
"queued": result.queued,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["JOB_KEY", "run_account_supply_replenishment"]
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
"""账号内部同步队列作业。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ....models import AccountWorkItem, AutomationJob
|
||||||
|
from ...account_sync_runner import ACCOUNT_SYNC_WORK_TYPES, run_account_sync_queue
|
||||||
|
|
||||||
|
JOB_KEY = "account_sync_queue"
|
||||||
|
|
||||||
|
|
||||||
|
def count_pending_work(db: Session, *, now: datetime | None = None) -> int:
|
||||||
|
moment = now or datetime.now(UTC)
|
||||||
|
return int(
|
||||||
|
db.query(AccountWorkItem.id)
|
||||||
|
.filter(
|
||||||
|
AccountWorkItem.work_type.in_(ACCOUNT_SYNC_WORK_TYPES),
|
||||||
|
AccountWorkItem.status == "pending",
|
||||||
|
AccountWorkItem.due_at <= moment,
|
||||||
|
)
|
||||||
|
.count()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def run_account_sync_queue_job(
|
||||||
|
db: Session,
|
||||||
|
job: AutomationJob,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
result = run_account_sync_queue(
|
||||||
|
db,
|
||||||
|
batch_size=max(1, int(job.batch_size or 20)),
|
||||||
|
)
|
||||||
|
if result.get("scanned"):
|
||||||
|
result["remaining"] = count_pending_work(db)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"JOB_KEY",
|
||||||
|
"count_pending_work",
|
||||||
|
"run_account_sync_queue_job",
|
||||||
|
]
|
||||||
@@ -23,6 +23,7 @@ class TaskLane(StrEnum):
|
|||||||
BINDING_POLL = "binding_poll"
|
BINDING_POLL = "binding_poll"
|
||||||
LOGIN = "login"
|
LOGIN = "login"
|
||||||
ACCOUNT_CHECK = "account_check"
|
ACCOUNT_CHECK = "account_check"
|
||||||
|
ACCOUNT_SYNC = "account_sync"
|
||||||
AUTOMATION = "automation"
|
AUTOMATION = "automation"
|
||||||
OVERVIEW_QUERY = "overview_query"
|
OVERVIEW_QUERY = "overview_query"
|
||||||
QUERY_BATCH = "query_batch"
|
QUERY_BATCH = "query_batch"
|
||||||
@@ -56,6 +57,7 @@ _DEFAULT_WORKERS: dict[TaskLane, int] = {
|
|||||||
TaskLane.BINDING_POLL: 2,
|
TaskLane.BINDING_POLL: 2,
|
||||||
TaskLane.LOGIN: 4,
|
TaskLane.LOGIN: 4,
|
||||||
TaskLane.ACCOUNT_CHECK: 4,
|
TaskLane.ACCOUNT_CHECK: 4,
|
||||||
|
TaskLane.ACCOUNT_SYNC: 3,
|
||||||
TaskLane.AUTOMATION: 2,
|
TaskLane.AUTOMATION: 2,
|
||||||
TaskLane.OVERVIEW_QUERY: _env_int("DOUYU_MAX_ACTIVE_OVERVIEW_TASKS", 5),
|
TaskLane.OVERVIEW_QUERY: _env_int("DOUYU_MAX_ACTIVE_OVERVIEW_TASKS", 5),
|
||||||
TaskLane.QUERY_BATCH: 3,
|
TaskLane.QUERY_BATCH: 3,
|
||||||
|
|||||||
Reference in New Issue
Block a user