feat(douyu): 建立账号同步队列与供给库存
This commit is contained in:
@@ -0,0 +1,392 @@
|
||||
"""斗鱼账号供给库存、目标缺口和定向补货入队。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import and_, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
Account,
|
||||
AccountCommodityState,
|
||||
AccountLease,
|
||||
AccountSupplyTarget,
|
||||
AccountSyncState,
|
||||
DouyuExternalOrder,
|
||||
)
|
||||
from .account_capability_service import AVAILABILITY_AVAILABLE
|
||||
from .account_lifecycle import MANUAL_STATES
|
||||
from .account_sync_runner import WORK_STOREHOME_SYNC
|
||||
from .account_value import is_high_value_expr
|
||||
from .account_work_queue import enqueue_account_work
|
||||
|
||||
VALUE_HIGH = "high_value"
|
||||
VALUE_ORDINARY = "ordinary"
|
||||
VALID_VALUES = frozenset({VALUE_HIGH, VALUE_ORDINARY})
|
||||
CK_FRESH_SECONDS = 24 * 3600
|
||||
ACTIVE_COLLAB_STATUSES = (
|
||||
"incomplete",
|
||||
"to_recharge",
|
||||
"recharged",
|
||||
"scripted",
|
||||
"points_error",
|
||||
"role_error",
|
||||
"failed",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SupplyInventory:
|
||||
target_id: int
|
||||
scope: str
|
||||
commodity_id: str
|
||||
value_class: str
|
||||
covered: int
|
||||
available: int
|
||||
allocatable: int
|
||||
target_covered: int
|
||||
target_available: int
|
||||
target_allocatable: int
|
||||
safety_buffer: int
|
||||
covered_shortage: int
|
||||
available_shortage: int
|
||||
allocatable_shortage: int
|
||||
|
||||
@property
|
||||
def needs_replenishment(self) -> bool:
|
||||
return bool(
|
||||
self.covered_shortage
|
||||
or self.available_shortage
|
||||
or self.allocatable_shortage
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SupplyReplenishmentResult:
|
||||
targets: int
|
||||
queued: int
|
||||
shortages: int
|
||||
|
||||
|
||||
def _active_lease_exists():
|
||||
return (
|
||||
select(AccountLease.id)
|
||||
.where(
|
||||
AccountLease.account_id == Account.id,
|
||||
AccountLease.status == "active",
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def _active_collab_exists():
|
||||
return (
|
||||
select(DouyuExternalOrder.id)
|
||||
.where(
|
||||
DouyuExternalOrder.account_id == Account.id,
|
||||
DouyuExternalOrder.archived_at.is_(None),
|
||||
DouyuExternalOrder.status.in_(ACTIVE_COLLAB_STATUSES),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def _value_filter(value_class: str, *, now: datetime):
|
||||
high_value = is_high_value_expr(now=now)
|
||||
if value_class == VALUE_HIGH:
|
||||
return high_value
|
||||
if value_class == VALUE_ORDINARY:
|
||||
return ~high_value
|
||||
raise ValueError(f"不支持的账号价值分类: {value_class}")
|
||||
|
||||
|
||||
def _base_account_query(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime,
|
||||
value_class: str,
|
||||
):
|
||||
ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS)
|
||||
return (
|
||||
db.query(Account)
|
||||
.filter(
|
||||
Account.cookie.isnot(None),
|
||||
Account.cookie != "",
|
||||
Account.ck_check_status == "valid",
|
||||
Account.ck_checked_at.isnot(None),
|
||||
Account.ck_checked_at >= ck_fresh_cutoff,
|
||||
Account.auto_managed.is_(True),
|
||||
Account.status.notin_(MANUAL_STATES),
|
||||
_value_filter(value_class, now=now),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _allocatable_filter(now: datetime):
|
||||
return and_(
|
||||
~_active_lease_exists(),
|
||||
~_active_collab_exists(),
|
||||
or_(
|
||||
Account.role_change_available_at.is_(None),
|
||||
Account.role_change_available_at <= now,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _fresh_available_filter(now: datetime):
|
||||
return and_(
|
||||
AccountCommodityState.availability == AVAILABILITY_AVAILABLE,
|
||||
AccountCommodityState.valid_until.isnot(None),
|
||||
AccountCommodityState.valid_until > now,
|
||||
or_(
|
||||
AccountCommodityState.available_from.is_(None),
|
||||
AccountCommodityState.available_from <= now,
|
||||
),
|
||||
or_(
|
||||
AccountCommodityState.available_until.is_(None),
|
||||
AccountCommodityState.available_until >= now,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _commodity_query(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime,
|
||||
target: AccountSupplyTarget,
|
||||
):
|
||||
query = _base_account_query(
|
||||
db,
|
||||
now=now,
|
||||
value_class=target.value_class,
|
||||
).join(
|
||||
AccountCommodityState,
|
||||
AccountCommodityState.account_id == Account.id,
|
||||
).filter(
|
||||
AccountCommodityState.platform == target.platform,
|
||||
AccountCommodityState.scope == target.scope,
|
||||
AccountCommodityState.commodity_id == target.commodity_id,
|
||||
AccountCommodityState.observed_at.isnot(None),
|
||||
)
|
||||
return query
|
||||
|
||||
|
||||
def _global_state_exists(target: AccountSupplyTarget):
|
||||
return (
|
||||
select(AccountCommodityState.id)
|
||||
.where(
|
||||
AccountCommodityState.account_id == Account.id,
|
||||
AccountCommodityState.platform == target.platform,
|
||||
AccountCommodityState.scope == target.scope,
|
||||
AccountCommodityState.observed_at.isnot(None),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def _global_available_exists(target: AccountSupplyTarget, *, now: datetime):
|
||||
return (
|
||||
select(AccountCommodityState.id)
|
||||
.where(
|
||||
AccountCommodityState.account_id == Account.id,
|
||||
AccountCommodityState.platform == target.platform,
|
||||
AccountCommodityState.scope == target.scope,
|
||||
_fresh_available_filter(now),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def evaluate_supply_target(
|
||||
db: Session,
|
||||
target: AccountSupplyTarget,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> SupplyInventory:
|
||||
"""计算一条供给目标的 covered / available / allocatable 库存。"""
|
||||
moment = now or datetime.now(UTC)
|
||||
if target.value_class not in VALID_VALUES:
|
||||
raise ValueError(f"不支持的账号价值分类: {target.value_class}")
|
||||
|
||||
if target.commodity_id:
|
||||
base = _commodity_query(db, now=moment, target=target)
|
||||
covered = base.count()
|
||||
available_query = base.filter(_fresh_available_filter(moment))
|
||||
available = available_query.count()
|
||||
allocatable = available_query.filter(
|
||||
_allocatable_filter(moment)
|
||||
).count()
|
||||
else:
|
||||
base = _base_account_query(
|
||||
db,
|
||||
now=moment,
|
||||
value_class=target.value_class,
|
||||
)
|
||||
covered = base.filter(_global_state_exists(target)).count()
|
||||
available_query = base.filter(
|
||||
_global_available_exists(target, now=moment)
|
||||
)
|
||||
available = available_query.count()
|
||||
allocatable = available_query.filter(
|
||||
_allocatable_filter(moment)
|
||||
).count()
|
||||
|
||||
return SupplyInventory(
|
||||
target_id=int(target.id),
|
||||
scope=target.scope,
|
||||
commodity_id=target.commodity_id,
|
||||
value_class=target.value_class,
|
||||
covered=covered,
|
||||
available=available,
|
||||
allocatable=allocatable,
|
||||
target_covered=int(target.target_covered or 0),
|
||||
target_available=int(target.target_available or 0),
|
||||
target_allocatable=int(target.target_allocatable or 0),
|
||||
safety_buffer=int(target.safety_buffer or 0),
|
||||
covered_shortage=max(
|
||||
0,
|
||||
int(target.target_covered or 0) + int(target.safety_buffer or 0)
|
||||
- covered,
|
||||
),
|
||||
available_shortage=max(
|
||||
0,
|
||||
int(target.target_available or 0) + int(target.safety_buffer or 0)
|
||||
- available,
|
||||
),
|
||||
allocatable_shortage=max(
|
||||
0,
|
||||
int(target.target_allocatable or 0) + int(target.safety_buffer or 0)
|
||||
- allocatable,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def evaluate_supply_targets(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> list[SupplyInventory]:
|
||||
moment = now or datetime.now(UTC)
|
||||
targets = (
|
||||
db.query(AccountSupplyTarget)
|
||||
.filter(AccountSupplyTarget.enabled.is_(True))
|
||||
.order_by(
|
||||
AccountSupplyTarget.platform.asc(),
|
||||
AccountSupplyTarget.scope.asc(),
|
||||
AccountSupplyTarget.value_class.asc(),
|
||||
AccountSupplyTarget.commodity_id.asc(),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
evaluate_supply_target(db, target, now=moment)
|
||||
for target in targets
|
||||
]
|
||||
|
||||
|
||||
def _sync_candidate_query(
|
||||
db: Session,
|
||||
*,
|
||||
target: AccountSupplyTarget,
|
||||
now: datetime,
|
||||
):
|
||||
query = _base_account_query(
|
||||
db,
|
||||
now=now,
|
||||
value_class=target.value_class,
|
||||
)
|
||||
if target.commodity_id:
|
||||
state_exists = (
|
||||
select(AccountCommodityState.id)
|
||||
.where(
|
||||
AccountCommodityState.account_id == Account.id,
|
||||
AccountCommodityState.platform == target.platform,
|
||||
AccountCommodityState.scope == target.scope,
|
||||
AccountCommodityState.commodity_id == target.commodity_id,
|
||||
AccountCommodityState.valid_until.isnot(None),
|
||||
AccountCommodityState.valid_until > now,
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
else:
|
||||
state_exists = (
|
||||
select(AccountSyncState.id)
|
||||
.where(
|
||||
AccountSyncState.account_id == Account.id,
|
||||
AccountSyncState.platform == target.platform,
|
||||
AccountSyncState.domain == "storehome",
|
||||
AccountSyncState.status == "success",
|
||||
AccountSyncState.valid_until.isnot(None),
|
||||
AccountSyncState.valid_until > now,
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
return query.filter(~state_exists).order_by(
|
||||
Account.overview_queried_at.is_(None).desc(),
|
||||
Account.overview_queried_at.asc(),
|
||||
Account.id.asc(),
|
||||
)
|
||||
|
||||
|
||||
def enqueue_supply_replenishment(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
max_per_target: int = 50,
|
||||
) -> SupplyReplenishmentResult:
|
||||
"""按本地目标缺口定向入队 storehome 同步,不发起外部请求。"""
|
||||
moment = now or datetime.now(UTC)
|
||||
inventories = evaluate_supply_targets(db, now=moment)
|
||||
queued = 0
|
||||
shortages = 0
|
||||
for inventory in inventories:
|
||||
if not inventory.needs_replenishment:
|
||||
continue
|
||||
shortages += 1
|
||||
target = db.get(AccountSupplyTarget, inventory.target_id)
|
||||
if target is None:
|
||||
continue
|
||||
missing = max(
|
||||
inventory.covered_shortage,
|
||||
inventory.available_shortage,
|
||||
inventory.allocatable_shortage,
|
||||
)
|
||||
limit = max(1, min(int(max_per_target), missing))
|
||||
accounts = _sync_candidate_query(
|
||||
db,
|
||||
target=target,
|
||||
now=moment,
|
||||
).limit(limit).all()
|
||||
for account in accounts:
|
||||
enqueue_account_work(
|
||||
db,
|
||||
account_id=int(account.id),
|
||||
work_type=WORK_STOREHOME_SYNC,
|
||||
priority=35,
|
||||
due_at=moment,
|
||||
reason=(
|
||||
f"供给短缺: {target.scope}/{target.commodity_id or 'all'}"
|
||||
f"/{target.value_class}"
|
||||
),
|
||||
)
|
||||
queued += 1
|
||||
if queued:
|
||||
db.commit()
|
||||
return SupplyReplenishmentResult(
|
||||
targets=len(inventories),
|
||||
queued=queued,
|
||||
shortages=shortages,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"VALUE_HIGH",
|
||||
"VALUE_ORDINARY",
|
||||
"SupplyInventory",
|
||||
"SupplyReplenishmentResult",
|
||||
"enqueue_supply_replenishment",
|
||||
"evaluate_supply_target",
|
||||
"evaluate_supply_targets",
|
||||
]
|
||||
@@ -0,0 +1,200 @@
|
||||
"""账号内部同步队列执行器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import as_completed
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import Account, AccountWorkItem
|
||||
from .account_sync_service import DOMAIN_STOREHOME, sync_storehome_snapshot
|
||||
from .account_work_queue import (
|
||||
claim_account_work,
|
||||
complete_account_work,
|
||||
fail_account_work,
|
||||
)
|
||||
from .douyu_activity_query import activity_client
|
||||
from .douyu_read_gateway import StorehomeSnapshot, fetch_storehome_snapshot
|
||||
from .douyu_service import ensure_douyu_config
|
||||
from .task_runtime import TaskLane, available_slots, submit
|
||||
|
||||
WORK_STOREHOME_SYNC = DOMAIN_STOREHOME + "_sync"
|
||||
ACCOUNT_SYNC_WORK_TYPES = (WORK_STOREHOME_SYNC,)
|
||||
|
||||
|
||||
def _execute_storehome_sync(
|
||||
db: Session,
|
||||
*,
|
||||
account: Account,
|
||||
item: AccountWorkItem,
|
||||
client_factory: Callable[[str], Any],
|
||||
snapshot_fetcher: Callable[..., StorehomeSnapshot],
|
||||
) -> dict[str, Any]:
|
||||
if not account.cookie:
|
||||
raise RuntimeError("账号没有可用 CK,不能同步商品能力")
|
||||
if account.ck_check_status in {"invalid", "manual"}:
|
||||
raise RuntimeError(f"账号 CK 状态为 {account.ck_check_status},暂不能同步商品能力")
|
||||
config = ensure_douyu_config(db)
|
||||
manual_id = str(getattr(config, "manual_id", "") or "")
|
||||
rid = str(getattr(config, "rid", "") or "")
|
||||
if not manual_id:
|
||||
raise RuntimeError("未配置精英宝典 manualID")
|
||||
client = client_factory(account.cookie)
|
||||
snapshot = snapshot_fetcher(
|
||||
client,
|
||||
manual_id=manual_id,
|
||||
rid=rid,
|
||||
)
|
||||
result = sync_storehome_snapshot(
|
||||
db,
|
||||
account,
|
||||
snapshot,
|
||||
source="account_work.storehome_sync",
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"work_id": int(item.id),
|
||||
"account_id": int(account.id),
|
||||
"item_count": result.item_count,
|
||||
"changed_count": result.changed_count,
|
||||
}
|
||||
|
||||
|
||||
def execute_account_work_item(
|
||||
work_id: int,
|
||||
*,
|
||||
session_factory: Callable[[], Session] = WorkerSessionLocal,
|
||||
client_factory: Callable[[str], Any] = activity_client,
|
||||
snapshot_fetcher: Callable[..., StorehomeSnapshot] = fetch_storehome_snapshot,
|
||||
) -> dict[str, Any]:
|
||||
"""在一个短事务中执行单条账号维护工作。"""
|
||||
session = session_factory()
|
||||
item: AccountWorkItem | None = None
|
||||
try:
|
||||
item = session.get(AccountWorkItem, int(work_id))
|
||||
if item is None or item.status != "running":
|
||||
return {"ok": False, "work_id": int(work_id), "skipped": True}
|
||||
account = session.get(Account, int(item.account_id))
|
||||
if account is None:
|
||||
raise RuntimeError("工作项对应账号不存在")
|
||||
if item.work_type != WORK_STOREHOME_SYNC:
|
||||
raise RuntimeError(f"不支持的工作类型: {item.work_type}")
|
||||
result = _execute_storehome_sync(
|
||||
session,
|
||||
account=account,
|
||||
item=item,
|
||||
client_factory=client_factory,
|
||||
snapshot_fetcher=snapshot_fetcher,
|
||||
)
|
||||
complete_account_work(session, item)
|
||||
session.commit()
|
||||
return result
|
||||
except Exception as exc: # noqa: BLE001 - 单工作项失败必须收敛到队列状态
|
||||
if not session.is_active:
|
||||
session.rollback()
|
||||
item = session.get(AccountWorkItem, int(work_id))
|
||||
if item is not None:
|
||||
fail_account_work(
|
||||
session,
|
||||
item,
|
||||
error=exc,
|
||||
retry_after_seconds=15 * 60,
|
||||
)
|
||||
session.commit()
|
||||
return {
|
||||
"ok": False,
|
||||
"work_id": int(work_id),
|
||||
"error": f"{type(exc).__name__}: {exc}"[:500],
|
||||
}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def run_account_sync_queue(
|
||||
db: Session,
|
||||
*,
|
||||
batch_size: int = 20,
|
||||
now: datetime | None = None,
|
||||
session_factory: Callable[[], Session] = WorkerSessionLocal,
|
||||
client_factory: Callable[[str], Any] = activity_client,
|
||||
snapshot_fetcher: Callable[..., StorehomeSnapshot] = fetch_storehome_snapshot,
|
||||
) -> dict[str, Any]:
|
||||
"""领取少量到期工作并通过独立账号同步槽执行。"""
|
||||
moment = now or datetime.now(UTC)
|
||||
capacity = available_slots(TaskLane.ACCOUNT_SYNC)
|
||||
if capacity <= 0:
|
||||
return {
|
||||
"scanned": 0,
|
||||
"success": 0,
|
||||
"failed": 0,
|
||||
"message": "账号同步槽已满,本轮跳过",
|
||||
}
|
||||
limit = max(1, min(int(batch_size), capacity))
|
||||
items = claim_account_work(
|
||||
db,
|
||||
work_types=ACCOUNT_SYNC_WORK_TYPES,
|
||||
limit=limit,
|
||||
now=moment,
|
||||
)
|
||||
db.commit()
|
||||
if not items:
|
||||
return {
|
||||
"scanned": 0,
|
||||
"success": 0,
|
||||
"failed": 0,
|
||||
"message": "暂无到期的账号同步工作",
|
||||
}
|
||||
|
||||
futures = {}
|
||||
for item in items:
|
||||
work_id = int(item.id)
|
||||
future = submit(
|
||||
TaskLane.ACCOUNT_SYNC,
|
||||
execute_account_work_item,
|
||||
work_id,
|
||||
session_factory=session_factory,
|
||||
client_factory=client_factory,
|
||||
snapshot_fetcher=snapshot_fetcher,
|
||||
)
|
||||
if future is not None:
|
||||
futures[future] = work_id
|
||||
|
||||
success = 0
|
||||
failed = 0
|
||||
skipped = 0
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
result = future.result()
|
||||
except Exception: # noqa: BLE001 - worker 异常已由执行函数收敛,这里只计数
|
||||
failed += 1
|
||||
continue
|
||||
if result.get("ok"):
|
||||
success += 1
|
||||
elif result.get("skipped"):
|
||||
skipped += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
return {
|
||||
"scanned": len(futures),
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"skipped": skipped,
|
||||
"message": (
|
||||
f"账号同步完成 {success} 条"
|
||||
+ (f",失败 {failed} 条" if failed else "")
|
||||
+ (f",跳过 {skipped} 条" if skipped else "")
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ACCOUNT_SYNC_WORK_TYPES",
|
||||
"WORK_STOREHOME_SYNC",
|
||||
"execute_account_work_item",
|
||||
"run_account_sync_queue",
|
||||
]
|
||||
@@ -42,7 +42,9 @@ def enqueue_account_work(
|
||||
account_id=account_id,
|
||||
work_type=work_type[:32],
|
||||
priority=int(priority),
|
||||
due_at=due_at or datetime.now(UTC),
|
||||
# MySQL DATETIME 不保存微秒;立即到期的工作项必须落在当前秒,
|
||||
# 否则四舍五入到下一秒后,本轮 claim 会误判“还没到期”。
|
||||
due_at=(due_at or datetime.now(UTC)).replace(microsecond=0),
|
||||
max_attempts=max(1, int(max_attempts)),
|
||||
dedupe_key=dedupe_key[:191],
|
||||
reason=reason[:255],
|
||||
@@ -149,14 +151,39 @@ def fail_account_work(
|
||||
item.finished_at = moment
|
||||
else:
|
||||
item.status = "pending"
|
||||
item.due_at = moment + timedelta(seconds=max(1, retry_after_seconds))
|
||||
item.due_at = (
|
||||
moment + timedelta(seconds=max(1, retry_after_seconds))
|
||||
).replace(microsecond=0)
|
||||
db.flush()
|
||||
|
||||
|
||||
def recover_running_account_work(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> int:
|
||||
"""服务重启后把中断的 running 工作项退回 pending。"""
|
||||
moment = (now or datetime.now(UTC)).replace(microsecond=0)
|
||||
items = (
|
||||
db.query(AccountWorkItem)
|
||||
.filter(AccountWorkItem.status == "running")
|
||||
.all()
|
||||
)
|
||||
for item in items:
|
||||
item.status = "pending"
|
||||
item.due_at = moment
|
||||
item.last_error = "工作执行中断(服务重启),已重新排队"
|
||||
item.updated_at = moment
|
||||
if items:
|
||||
db.commit()
|
||||
return len(items)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"claim_account_work",
|
||||
"complete_account_work",
|
||||
"complete_account_work_type",
|
||||
"enqueue_account_work",
|
||||
"fail_account_work",
|
||||
"recover_running_account_work",
|
||||
]
|
||||
|
||||
@@ -18,6 +18,18 @@ from .jobs.account_overview_refresh import (
|
||||
from .jobs.account_overview_refresh import (
|
||||
run_account_overview_refresh,
|
||||
)
|
||||
from .jobs.account_supply_replenishment import (
|
||||
JOB_KEY as ACCOUNT_SUPPLY_REPLENISHMENT_KEY,
|
||||
)
|
||||
from .jobs.account_supply_replenishment import (
|
||||
run_account_supply_replenishment,
|
||||
)
|
||||
from .jobs.account_sync_queue import (
|
||||
JOB_KEY as ACCOUNT_SYNC_QUEUE_KEY,
|
||||
)
|
||||
from .jobs.account_sync_queue import (
|
||||
run_account_sync_queue_job,
|
||||
)
|
||||
from .jobs.ck_health import JOB_KEY as CK_HEALTH_KEY
|
||||
from .jobs.ck_health import run_ck_health
|
||||
from .jobs.ck_relogin import JOB_KEY as CK_RELOGIN_KEY
|
||||
@@ -53,6 +65,8 @@ from .scheduler import (
|
||||
__all__ = [
|
||||
"ACCOUNT_CAPABILITY_REFRESH_KEY",
|
||||
"ACCOUNT_OVERVIEW_REFRESH_KEY",
|
||||
"ACCOUNT_SUPPLY_REPLENISHMENT_KEY",
|
||||
"ACCOUNT_SYNC_QUEUE_KEY",
|
||||
"CK_HEALTH_KEY",
|
||||
"CK_RELOGIN_KEY",
|
||||
"DOUYU_TASKS_RETENTION_KEY",
|
||||
@@ -67,6 +81,8 @@ __all__ = [
|
||||
"reset_registry",
|
||||
"run_account_capability_refresh",
|
||||
"run_account_overview_refresh",
|
||||
"run_account_supply_replenishment",
|
||||
"run_account_sync_queue_job",
|
||||
"run_ck_health",
|
||||
"run_ck_relogin",
|
||||
"run_douyu_tasks_retention",
|
||||
@@ -82,6 +98,30 @@ __all__ = [
|
||||
|
||||
def register_default_jobs() -> None:
|
||||
"""登记内置作业(幂等)。"""
|
||||
register_job(
|
||||
JobSpec(
|
||||
key=ACCOUNT_SYNC_QUEUE_KEY,
|
||||
title="账号内部同步队列",
|
||||
interval_seconds=60,
|
||||
run_interval_seconds=60,
|
||||
batch_size=20,
|
||||
concurrency=3,
|
||||
run=run_account_sync_queue_job,
|
||||
default_enabled=True,
|
||||
)
|
||||
)
|
||||
register_job(
|
||||
JobSpec(
|
||||
key=ACCOUNT_SUPPLY_REPLENISHMENT_KEY,
|
||||
title="账号供给目标补货",
|
||||
interval_seconds=5 * 60,
|
||||
run_interval_seconds=5 * 60,
|
||||
batch_size=50,
|
||||
concurrency=1,
|
||||
run=run_account_supply_replenishment,
|
||||
default_enabled=True,
|
||||
)
|
||||
)
|
||||
register_job(
|
||||
JobSpec(
|
||||
key=ACCOUNT_CAPABILITY_REFRESH_KEY,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""按直播履约需求刷新账号商品能力。
|
||||
"""按直播履约需求把过期商品能力账号送入内部同步队列。
|
||||
|
||||
全量资料补全仍由 ``account_overview_refresh`` 负责;本作业只处理“有 pending
|
||||
任务、但对应商品能力快照已经过期”的账号,避免为了秒匹配把两万多个账号
|
||||
无条件轮询一遍。
|
||||
本作业只做本地候选筛选,不直接请求官网。真正的 ``storehome`` 查询由
|
||||
``account_sync_queue`` 作业在独立执行槽中低并发消费,避免补货查询抢占
|
||||
履约实时复核、登录或批量任务的执行资源。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,8 +21,9 @@ from ....models import (
|
||||
PlatformFulfillmentTask,
|
||||
)
|
||||
from ...account_lifecycle import MANUAL_STATES
|
||||
from ...account_sync_runner import WORK_STOREHOME_SYNC
|
||||
from ...account_work_queue import enqueue_account_work
|
||||
from ...douyu_service import cookie_operable_filter
|
||||
from .account_overview_refresh import run_refresh_for_accounts
|
||||
|
||||
JOB_KEY = "account_capability_refresh"
|
||||
|
||||
@@ -94,18 +95,29 @@ def run_account_capability_refresh(
|
||||
) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
accounts = build_select_query(db, job, now).all()
|
||||
return run_refresh_for_accounts(
|
||||
db,
|
||||
job,
|
||||
accounts,
|
||||
now=now,
|
||||
# 精确剩余量要完整评估 accounts × commodity EXISTS,生产实测单次约
|
||||
# 1.3 秒、扫描 38 万行;它只用于展示,直接省略,避免每轮制造慢查询。
|
||||
remaining_count=None,
|
||||
source="capability_demand_refresh",
|
||||
log_label="capability",
|
||||
empty_message="当前履约商品没有到期待刷新的账号能力",
|
||||
)
|
||||
for account in accounts:
|
||||
enqueue_account_work(
|
||||
db,
|
||||
account_id=int(account.id),
|
||||
work_type=WORK_STOREHOME_SYNC,
|
||||
priority=40,
|
||||
reason="履约商品能力过期,等待补货刷新",
|
||||
due_at=now,
|
||||
)
|
||||
db.commit()
|
||||
queued = len(accounts)
|
||||
return {
|
||||
"scanned": len(accounts),
|
||||
"success": queued,
|
||||
"invalid": 0,
|
||||
"skipped": 0,
|
||||
"message": (
|
||||
f"本轮筛选 {len(accounts)} 个到期账号,新入队 {queued} 个"
|
||||
if accounts
|
||||
else "当前履约商品没有到期待刷新的账号能力"
|
||||
),
|
||||
"detail": {"queued": queued},
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""按账号供给目标缺口定向补货入队。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....models import AutomationJob
|
||||
from ...account_supply_service import enqueue_supply_replenishment
|
||||
|
||||
JOB_KEY = "account_supply_replenishment"
|
||||
|
||||
|
||||
def run_account_supply_replenishment(
|
||||
db: Session,
|
||||
job: AutomationJob,
|
||||
) -> dict[str, Any]:
|
||||
result = enqueue_supply_replenishment(
|
||||
db,
|
||||
max_per_target=max(1, int(job.batch_size or 50)),
|
||||
)
|
||||
return {
|
||||
"scanned": result.targets,
|
||||
"success": result.queued,
|
||||
"invalid": 0,
|
||||
"skipped": max(0, result.targets - result.shortages),
|
||||
"message": (
|
||||
f"评估 {result.targets} 个供给目标,"
|
||||
f"短缺 {result.shortages} 个,新排队 {result.queued} 个账号同步"
|
||||
if result.targets
|
||||
else "未配置账号供给目标"
|
||||
),
|
||||
"detail": {
|
||||
"shortages": result.shortages,
|
||||
"queued": result.queued,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["JOB_KEY", "run_account_supply_replenishment"]
|
||||
@@ -0,0 +1,47 @@
|
||||
"""账号内部同步队列作业。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....models import AccountWorkItem, AutomationJob
|
||||
from ...account_sync_runner import ACCOUNT_SYNC_WORK_TYPES, run_account_sync_queue
|
||||
|
||||
JOB_KEY = "account_sync_queue"
|
||||
|
||||
|
||||
def count_pending_work(db: Session, *, now: datetime | None = None) -> int:
|
||||
moment = now or datetime.now(UTC)
|
||||
return int(
|
||||
db.query(AccountWorkItem.id)
|
||||
.filter(
|
||||
AccountWorkItem.work_type.in_(ACCOUNT_SYNC_WORK_TYPES),
|
||||
AccountWorkItem.status == "pending",
|
||||
AccountWorkItem.due_at <= moment,
|
||||
)
|
||||
.count()
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
def run_account_sync_queue_job(
|
||||
db: Session,
|
||||
job: AutomationJob,
|
||||
) -> dict[str, Any]:
|
||||
result = run_account_sync_queue(
|
||||
db,
|
||||
batch_size=max(1, int(job.batch_size or 20)),
|
||||
)
|
||||
if result.get("scanned"):
|
||||
result["remaining"] = count_pending_work(db)
|
||||
return result
|
||||
|
||||
|
||||
__all__ = [
|
||||
"JOB_KEY",
|
||||
"count_pending_work",
|
||||
"run_account_sync_queue_job",
|
||||
]
|
||||
@@ -23,6 +23,7 @@ class TaskLane(StrEnum):
|
||||
BINDING_POLL = "binding_poll"
|
||||
LOGIN = "login"
|
||||
ACCOUNT_CHECK = "account_check"
|
||||
ACCOUNT_SYNC = "account_sync"
|
||||
AUTOMATION = "automation"
|
||||
OVERVIEW_QUERY = "overview_query"
|
||||
QUERY_BATCH = "query_batch"
|
||||
@@ -56,6 +57,7 @@ _DEFAULT_WORKERS: dict[TaskLane, int] = {
|
||||
TaskLane.BINDING_POLL: 2,
|
||||
TaskLane.LOGIN: 4,
|
||||
TaskLane.ACCOUNT_CHECK: 4,
|
||||
TaskLane.ACCOUNT_SYNC: 3,
|
||||
TaskLane.AUTOMATION: 2,
|
||||
TaskLane.OVERVIEW_QUERY: _env_int("DOUYU_MAX_ACTIVE_OVERVIEW_TASKS", 5),
|
||||
TaskLane.QUERY_BATCH: 3,
|
||||
|
||||
Reference in New Issue
Block a user