Files
live-hub-py/web/backend/services/platform_fulfillment/auto_allocator.py
T

296 lines
9.1 KiB
Python

"""自动为待处理直播账号履约任务选号;异常任务保留给人工处理。"""
from __future__ import annotations
import asyncio
import os
import threading
from collections.abc import Iterable
from datetime import UTC, datetime, timedelta
from typing import Any
from loguru import logger
from sqlalchemy import or_
from sqlalchemy.orm import Session
from ...database import WorkerSessionLocal
from ...models import PlatformFulfillmentTask
from . import service
DEFAULT_INTERVAL_SECONDS = 20
DEFAULT_BATCH_SIZE = 2
DEFAULT_RETRY_SECONDS = 45
_stop_event = threading.Event()
_thread: threading.Thread | None = None
_lock = threading.Lock()
_pending_lock = threading.Lock()
_pending_task_ids: set[int] = set()
_wake_event = threading.Event()
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, min(maximum, value))
def wake_auto_allocator(task_ids: Iterable[int] | None = None) -> None:
"""立即唤醒选号 worker;传入任务 ID 时优先处理这些新任务。"""
normalized = {int(task_id) for task_id in (task_ids or []) if int(task_id) > 0}
if normalized:
with _pending_lock:
_pending_task_ids.update(normalized)
_wake_event.set()
def _take_pending_task_ids(limit: int) -> list[int]:
with _pending_lock:
selected = sorted(_pending_task_ids)[:limit]
for task_id in selected:
_pending_task_ids.discard(task_id)
return selected
def _has_pending_task_ids() -> bool:
with _pending_lock:
return bool(_pending_task_ids)
def _schedule_retry(
db: Session,
*,
task_id: int,
moment: datetime,
retry_seconds: int,
) -> None:
task = db.get(PlatformFulfillmentTask, int(task_id))
if task is None or task.execution_status in {"completed", "cancelled"}:
return
task.next_allocate_at = moment + timedelta(seconds=retry_seconds)
db.commit()
def run_auto_allocation_once(
db: Session,
*,
limit: int | None = None,
now: datetime | None = None,
task_ids: list[int] | None = None,
) -> dict[str, int]:
"""领取已到期的 ready 任务;新任务不再等待固定 45 秒。"""
moment = now or datetime.now(UTC)
batch_size = limit or _env_int(
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_BATCH",
DEFAULT_BATCH_SIZE,
1,
20,
)
retry_seconds = _env_int(
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_RETRY_SECONDS",
DEFAULT_RETRY_SECONDS,
10,
3600,
)
# MySQL DATETIME 默认只保存到秒,写入时会按微秒四舍五入;
# 给 1 秒容差,避免刚写入的 next_allocate_at 被 worker 当成“未来任务”。
due_cutoff = moment + timedelta(seconds=1)
query = db.query(PlatformFulfillmentTask).filter(
PlatformFulfillmentTask.execution_status == "ready",
PlatformFulfillmentTask.account_id.is_(None),
PlatformFulfillmentTask.commodity_id != "",
or_(
PlatformFulfillmentTask.next_allocate_at.is_(None),
PlatformFulfillmentTask.next_allocate_at <= due_cutoff,
),
)
normalized_ids = [
int(task_id) for task_id in dict.fromkeys(task_ids or []) if int(task_id) > 0
]
if normalized_ids:
query = query.filter(PlatformFulfillmentTask.id.in_(normalized_ids))
query = service._apply_order_match_cutoff(
query,
service._order_match_started_at(db),
)
candidates = (
query.order_by(
PlatformFulfillmentTask.paid_at.asc(),
PlatformFulfillmentTask.id.asc(),
)
.limit(batch_size)
.all()
)
if not candidates:
return {"checked": 0, "allocated": 0, "failed": 0}
actor = service._integration_actor(db)
allocated = 0
failed = 0
for task in candidates:
candidate_task_id = int(task.id)
try:
detail: dict[str, Any] = service.allocate_task(
db,
task_id=candidate_task_id,
current=actor,
)
if detail.get("task", {}).get("account"):
allocated += 1
allocated_task = db.get(PlatformFulfillmentTask, candidate_task_id)
if allocated_task is not None:
allocated_task.next_allocate_at = None
db.commit()
try:
asyncio.run(
service.start_binding(
db,
task_id=candidate_task_id,
current=actor,
)
)
except Exception as exc: # noqa: BLE001 - allocation remains valid
db.rollback()
logger.warning(
"直播账号履约自动开始绑定失败 task_id={}: {}",
candidate_task_id,
exc,
)
else:
failed += 1
_schedule_retry(
db,
task_id=candidate_task_id,
moment=moment,
retry_seconds=retry_seconds,
)
except Exception as exc: # noqa: BLE001 - 单个任务异常不能中断整批
db.rollback()
failed += 1
_schedule_retry(
db,
task_id=candidate_task_id,
moment=moment,
retry_seconds=retry_seconds,
)
logger.warning(
"直播账号履约自动选号失败 task_id={}: {}",
candidate_task_id,
exc,
)
return {
"checked": len(candidates),
"allocated": allocated,
"failed": failed,
}
def start_allocated_bindings_once(
db: Session,
*,
limit: int = DEFAULT_BATCH_SIZE,
) -> int:
"""Start durable binding preparation for allocated, unstarted sessions."""
rows = (
db.query(PlatformFulfillmentTask)
.filter(
PlatformFulfillmentTask.execution_status == "allocated",
PlatformFulfillmentTask.binding_status == "not_started",
PlatformFulfillmentTask.account_id.isnot(None),
PlatformFulfillmentTask.lease_id.isnot(None),
)
.order_by(PlatformFulfillmentTask.updated_at.asc(), PlatformFulfillmentTask.id.asc())
.limit(max(1, limit))
.all()
)
if not rows:
return 0
actor = service._integration_actor(db)
started = 0
for task in rows:
try:
asyncio.run(
service.start_binding(
db,
task_id=int(task.id),
current=actor,
)
)
started += 1
except Exception as exc: # noqa: BLE001 - one session cannot block the rest
db.rollback()
logger.warning(
"直播账号履约补启动绑定失败 task_id={}: {}",
task.id,
exc,
)
return started
def _run() -> None:
interval = _env_int(
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_SECONDS",
DEFAULT_INTERVAL_SECONDS,
5,
3600,
)
while not _stop_event.is_set():
db = WorkerSessionLocal()
try:
pending_ids = _take_pending_task_ids(
_env_int(
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_BATCH",
DEFAULT_BATCH_SIZE,
1,
20,
)
)
result = run_auto_allocation_once(db, task_ids=pending_ids or None)
started = start_allocated_bindings_once(db)
if result["checked"]:
logger.info("直播账号履约自动选号: {}", result)
if started:
logger.info("直播账号履约补启动绑定: {} 条", started)
except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出
db.rollback()
logger.warning("直播账号履约自动选号轮次失败: {}", exc)
finally:
db.close()
if _stop_event.is_set():
break
if _has_pending_task_ids():
continue
_wake_event.wait(interval)
_wake_event.clear()
def start_platform_fulfillment_auto_allocator() -> bool:
global _thread
with _lock:
if _thread is not None and _thread.is_alive():
return False
_stop_event.clear()
_wake_event.clear()
with _pending_lock:
_pending_task_ids.clear()
_thread = threading.Thread(
target=_run,
name="platform-fulfillment-auto-allocator",
daemon=True,
)
_thread.start()
return True
def stop_platform_fulfillment_auto_allocator() -> None:
global _thread
_stop_event.set()
_wake_event.set()
with _lock:
thread = _thread
_thread = None
if thread is not None and thread.is_alive():
thread.join(timeout=3)