Files
live-hub-py/web/backend/services/payment_poll_scheduler.py
T

273 lines
7.7 KiB
Python

"""Database-backed scheduler for generated payment QR codes."""
from __future__ import annotations
import os
import threading
from concurrent.futures import Future
from datetime import UTC, datetime, timedelta
from loguru import logger
from sqlalchemy.orm import joinedload
from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .douyu_runner import DouyuBatchRunner
from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL
from .task_runtime import TaskLane, available_slots, try_submit
DEFAULT_INTERVAL_SECONDS = 1
DEFAULT_BATCH_SIZE = 30
CLAIM_SECONDS = 60
PAYMENT_TASK_TYPES = frozenset(
{
"create_elite_qr",
"create_esports_qr",
"create_gold_qr",
}
)
_stop_event = threading.Event()
_thread: threading.Thread | None = None
_lock = threading.Lock()
_in_flight: set[int] = set()
_in_flight_lock = threading.Lock()
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, min(maximum, value))
def _claim_due_payment_tasks(limit: int) -> list[int]:
db = WorkerSessionLocal()
now = datetime.now(UTC)
claimed: list[int] = []
try:
rows = (
db.query(DouyuTask)
.filter(
DouyuTask.task_type.in_(PAYMENT_TASK_TYPES),
DouyuTask.status == "running",
DouyuTask.queue_lane == "payment_poll",
DouyuTask.next_run_at.isnot(None),
DouyuTask.next_run_at <= now,
DouyuTask.run_deadline_at.isnot(None),
)
.order_by(DouyuTask.next_run_at.asc(), DouyuTask.id.asc())
.with_for_update(skip_locked=True)
.limit(limit)
.all()
)
for row in rows:
task_id = int(row.id)
with _in_flight_lock:
if task_id in _in_flight:
continue
_in_flight.add(task_id)
row.next_run_at = now + timedelta(seconds=CLAIM_SECONDS)
row.last_run_at = now
row.attempt_count = int(row.attempt_count or 0) + 1
claimed.append(task_id)
db.commit()
return claimed
except Exception:
db.rollback()
with _in_flight_lock:
for task_id in claimed:
_in_flight.discard(task_id)
raise
finally:
db.close()
def _reschedule_after_error(
db,
task: DouyuTask,
account,
exc: Exception,
) -> None:
result = dict(task.result or {})
result["payment_poll_error"] = str(exc)
runner = DouyuBatchRunner(
db,
task.batch_id,
task.task_type,
close_db=False,
)
runner._schedule_payment_poll(
db,
task,
result,
f"支付到账查询失败,稍后重试: {exc}",
delay_seconds=DOUYU_PAYMENT_POLL_INTERVAL,
)
if account is not None:
logger.warning(
"[douyu] 支付轮询失败 task_id={} account_id={}: {}",
task.id,
account.id,
exc,
)
def poll_payment_task_once(task_id: int) -> None:
db = WorkerSessionLocal()
task = None
account = None
try:
task = (
db.query(DouyuTask)
.options(joinedload(DouyuTask.account))
.filter(DouyuTask.id == int(task_id))
.first()
)
if (
task is None
or task.status != "running"
or task.task_type not in PAYMENT_TASK_TYPES
or task.queue_lane != "payment_poll"
):
return
account = task.account
if account is None:
DouyuBatchRunner(
db,
task.batch_id,
task.task_type,
close_db=False,
)._mark_task(
db,
task,
"failed",
"账号不存在,无法继续查询支付到账",
dict(task.result or {}),
)
return
runner = DouyuBatchRunner(
db,
task.batch_id,
task.task_type,
close_db=False,
)
now = datetime.now(UTC)
if task.run_deadline_at is not None and task.run_deadline_at <= now:
runner.mark_payment_poll_timeout(db, task)
return
config = runner._config_info(db)
runner.poll_payment_task_once(db, task, account, config)
except Exception as exc: # noqa: BLE001 - durable row is retried after backoff
db.rollback()
if task is not None:
try:
db.refresh(task)
if task.status == "running" and task.queue_lane == "payment_poll":
_reschedule_after_error(db, task, account, exc)
except Exception: # noqa: BLE001 - scheduler must survive one bad row
logger.exception("[douyu] 支付轮询失败后重排也失败 task_id={}", task_id)
finally:
db.close()
with _in_flight_lock:
_in_flight.discard(int(task_id))
def _release_callback(future: Future) -> None:
try:
future.result()
except Exception as exc: # noqa: BLE001 - worker already logs; keep scheduler alive
logger.warning("支付到账查询任务异常: {}", exc)
def run_payment_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
capacity = available_slots(TaskLane.PAYMENT_POLL)
if capacity <= 0:
return 0
claimed = _claim_due_payment_tasks(min(max(1, limit), capacity))
submitted = 0
for task_id in claimed:
future = try_submit(
TaskLane.PAYMENT_POLL,
poll_payment_task_once,
task_id,
stopped=_stop_event.is_set,
)
if future is None:
with _in_flight_lock:
_in_flight.discard(task_id)
break
future.add_done_callback(_release_callback)
submitted += 1
return submitted
def _run() -> None:
interval = _env_int(
"PAYMENT_POLL_DISPATCH_SECONDS",
DEFAULT_INTERVAL_SECONDS,
1,
30,
)
while not _stop_event.is_set():
try:
batch_size = _env_int(
"PAYMENT_POLL_DISPATCH_BATCH",
DEFAULT_BATCH_SIZE,
1,
200,
)
run_payment_poll_dispatch_once(batch_size)
except Exception as exc: # noqa: BLE001 - scheduler survives any one bad tick
logger.warning("支付到账调度轮次失败: {}", exc)
_stop_event.wait(interval)
def start_payment_poll_scheduler() -> bool:
global _thread
with _lock:
if _thread is not None and _thread.is_alive():
return False
_stop_event.clear()
_thread = threading.Thread(
target=_run,
name="payment-poll-scheduler",
daemon=True,
)
_thread.start()
logger.info("支付到账定时调度器已启动")
return True
def stop_payment_poll_scheduler() -> None:
global _thread
_stop_event.set()
with _lock:
thread = _thread
_thread = None
if thread is not None and thread.is_alive():
thread.join(timeout=3)
def scheduler_status() -> dict[str, object]:
with _lock:
running = _thread is not None and _thread.is_alive()
with _in_flight_lock:
in_flight = len(_in_flight)
return {
"running": running,
"in_flight": in_flight,
"claim_seconds": CLAIM_SECONDS,
}
__all__ = [
"PAYMENT_TASK_TYPES",
"poll_payment_task_once",
"run_payment_poll_dispatch_once",
"scheduler_status",
"start_payment_poll_scheduler",
"stop_payment_poll_scheduler",
]