270 lines
7.6 KiB
Python
270 lines
7.6 KiB
Python
"""Database-backed scheduler for generated payment QR codes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import threading
|
|
from concurrent.futures import Future
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from loguru import logger
|
|
from sqlalchemy.orm import joinedload
|
|
|
|
from ..database import WorkerSessionLocal
|
|
from ..models import DouyuTask
|
|
from .douyu_runner import DouyuBatchRunner
|
|
from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL
|
|
from .task_runtime import TaskLane, submit
|
|
|
|
DEFAULT_INTERVAL_SECONDS = 1
|
|
DEFAULT_BATCH_SIZE = 30
|
|
CLAIM_SECONDS = 60
|
|
PAYMENT_TASK_TYPES = frozenset(
|
|
{
|
|
"create_elite_qr",
|
|
"create_esports_qr",
|
|
"create_gold_qr",
|
|
}
|
|
)
|
|
|
|
_stop_event = threading.Event()
|
|
_thread: threading.Thread | None = None
|
|
_lock = threading.Lock()
|
|
_in_flight: set[int] = set()
|
|
_in_flight_lock = threading.Lock()
|
|
|
|
|
|
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
|
try:
|
|
value = int(os.getenv(name, str(default)))
|
|
except ValueError:
|
|
return default
|
|
return max(minimum, min(maximum, value))
|
|
|
|
|
|
def _claim_due_payment_tasks(limit: int) -> list[int]:
|
|
db = WorkerSessionLocal()
|
|
now = datetime.now(UTC)
|
|
claimed: list[int] = []
|
|
try:
|
|
rows = (
|
|
db.query(DouyuTask)
|
|
.filter(
|
|
DouyuTask.task_type.in_(PAYMENT_TASK_TYPES),
|
|
DouyuTask.status == "running",
|
|
DouyuTask.queue_lane == "payment_poll",
|
|
DouyuTask.next_run_at.isnot(None),
|
|
DouyuTask.next_run_at <= now,
|
|
DouyuTask.run_deadline_at.isnot(None),
|
|
)
|
|
.order_by(DouyuTask.next_run_at.asc(), DouyuTask.id.asc())
|
|
.with_for_update(skip_locked=True)
|
|
.limit(limit)
|
|
.all()
|
|
)
|
|
for row in rows:
|
|
task_id = int(row.id)
|
|
with _in_flight_lock:
|
|
if task_id in _in_flight:
|
|
continue
|
|
_in_flight.add(task_id)
|
|
row.next_run_at = now + timedelta(seconds=CLAIM_SECONDS)
|
|
row.last_run_at = now
|
|
row.attempt_count = int(row.attempt_count or 0) + 1
|
|
claimed.append(task_id)
|
|
db.commit()
|
|
return claimed
|
|
except Exception:
|
|
db.rollback()
|
|
with _in_flight_lock:
|
|
for task_id in claimed:
|
|
_in_flight.discard(task_id)
|
|
raise
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _reschedule_after_error(
|
|
db,
|
|
task: DouyuTask,
|
|
account,
|
|
exc: Exception,
|
|
) -> None:
|
|
result = dict(task.result or {})
|
|
result["payment_poll_error"] = str(exc)
|
|
runner = DouyuBatchRunner(
|
|
db,
|
|
task.batch_id,
|
|
task.task_type,
|
|
close_db=False,
|
|
)
|
|
runner._schedule_payment_poll(
|
|
db,
|
|
task,
|
|
result,
|
|
f"支付到账查询失败,稍后重试: {exc}",
|
|
delay_seconds=DOUYU_PAYMENT_POLL_INTERVAL,
|
|
)
|
|
if account is not None:
|
|
logger.warning(
|
|
"[douyu] 支付轮询失败 task_id={} account_id={}: {}",
|
|
task.id,
|
|
account.id,
|
|
exc,
|
|
)
|
|
|
|
|
|
def poll_payment_task_once(task_id: int) -> None:
|
|
db = WorkerSessionLocal()
|
|
task = None
|
|
account = None
|
|
try:
|
|
task = (
|
|
db.query(DouyuTask)
|
|
.options(joinedload(DouyuTask.account))
|
|
.filter(DouyuTask.id == int(task_id))
|
|
.first()
|
|
)
|
|
if (
|
|
task is None
|
|
or task.status != "running"
|
|
or task.task_type not in PAYMENT_TASK_TYPES
|
|
or task.queue_lane != "payment_poll"
|
|
):
|
|
return
|
|
account = task.account
|
|
if account is None:
|
|
DouyuBatchRunner(
|
|
db,
|
|
task.batch_id,
|
|
task.task_type,
|
|
close_db=False,
|
|
)._mark_task(
|
|
db,
|
|
task,
|
|
"failed",
|
|
"账号不存在,无法继续查询支付到账",
|
|
dict(task.result or {}),
|
|
)
|
|
return
|
|
runner = DouyuBatchRunner(
|
|
db,
|
|
task.batch_id,
|
|
task.task_type,
|
|
close_db=False,
|
|
)
|
|
now = datetime.now(UTC)
|
|
if task.run_deadline_at is not None and task.run_deadline_at <= now:
|
|
runner.mark_payment_poll_timeout(db, task)
|
|
return
|
|
config = runner._config_info(db)
|
|
runner.poll_payment_task_once(db, task, account, config)
|
|
except Exception as exc: # noqa: BLE001 - durable row is retried after backoff
|
|
db.rollback()
|
|
if task is not None:
|
|
try:
|
|
db.refresh(task)
|
|
if task.status == "running" and task.queue_lane == "payment_poll":
|
|
_reschedule_after_error(db, task, account, exc)
|
|
except Exception: # noqa: BLE001 - scheduler must survive one bad row
|
|
logger.exception("[douyu] 支付轮询失败后重排也失败 task_id={}", task_id)
|
|
finally:
|
|
db.close()
|
|
with _in_flight_lock:
|
|
_in_flight.discard(int(task_id))
|
|
|
|
|
|
def _release_callback(future: Future) -> None:
|
|
try:
|
|
future.result()
|
|
except Exception as exc: # noqa: BLE001 - worker already logs; keep scheduler alive
|
|
logger.warning("支付到账查询任务异常: {}", exc)
|
|
|
|
|
|
def run_payment_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
|
|
claimed = _claim_due_payment_tasks(max(1, limit))
|
|
submitted = 0
|
|
for task_id in claimed:
|
|
future = submit(
|
|
TaskLane.PAYMENT_POLL,
|
|
poll_payment_task_once,
|
|
task_id,
|
|
stopped=_stop_event.is_set,
|
|
)
|
|
if future is None:
|
|
with _in_flight_lock:
|
|
_in_flight.discard(task_id)
|
|
break
|
|
future.add_done_callback(_release_callback)
|
|
submitted += 1
|
|
return submitted
|
|
|
|
|
|
def _run() -> None:
|
|
interval = _env_int(
|
|
"PAYMENT_POLL_DISPATCH_SECONDS",
|
|
DEFAULT_INTERVAL_SECONDS,
|
|
1,
|
|
30,
|
|
)
|
|
while not _stop_event.is_set():
|
|
try:
|
|
batch_size = _env_int(
|
|
"PAYMENT_POLL_DISPATCH_BATCH",
|
|
DEFAULT_BATCH_SIZE,
|
|
1,
|
|
200,
|
|
)
|
|
run_payment_poll_dispatch_once(batch_size)
|
|
except Exception as exc: # noqa: BLE001 - scheduler survives any one bad tick
|
|
logger.warning("支付到账调度轮次失败: {}", exc)
|
|
_stop_event.wait(interval)
|
|
|
|
|
|
def start_payment_poll_scheduler() -> bool:
|
|
global _thread
|
|
with _lock:
|
|
if _thread is not None and _thread.is_alive():
|
|
return False
|
|
_stop_event.clear()
|
|
_thread = threading.Thread(
|
|
target=_run,
|
|
name="payment-poll-scheduler",
|
|
daemon=True,
|
|
)
|
|
_thread.start()
|
|
logger.info("支付到账定时调度器已启动")
|
|
return True
|
|
|
|
|
|
def stop_payment_poll_scheduler() -> None:
|
|
global _thread
|
|
_stop_event.set()
|
|
with _lock:
|
|
thread = _thread
|
|
_thread = None
|
|
if thread is not None and thread.is_alive():
|
|
thread.join(timeout=3)
|
|
|
|
|
|
def scheduler_status() -> dict[str, object]:
|
|
with _lock:
|
|
running = _thread is not None and _thread.is_alive()
|
|
with _in_flight_lock:
|
|
in_flight = len(_in_flight)
|
|
return {
|
|
"running": running,
|
|
"in_flight": in_flight,
|
|
"claim_seconds": CLAIM_SECONDS,
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"PAYMENT_TASK_TYPES",
|
|
"poll_payment_task_once",
|
|
"run_payment_poll_dispatch_once",
|
|
"scheduler_status",
|
|
"start_payment_poll_scheduler",
|
|
"stop_payment_poll_scheduler",
|
|
]
|