perf(douyu): 拆分任务槽并改造绑定扫码调度
This commit is contained in:
@@ -6,7 +6,7 @@ import io
|
||||
import threading
|
||||
import uuid
|
||||
import zipfile
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from concurrent.futures import as_completed
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
@@ -18,10 +18,11 @@ from core.douyu import DouyuLogin, create_login_api_strategy
|
||||
from core.douyu.login import AccountLike
|
||||
from core.douyu.proxy_fetcher import ProxyFetcher
|
||||
|
||||
from ..database import SessionLocal
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import ProxyConfig as ProxyConfigModel
|
||||
from .account_events_service import CATEGORY_AUTH, emit
|
||||
from .proxy_service import build_proxy_runtime
|
||||
from .task_runtime import TaskLane, submit
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
@@ -251,7 +252,7 @@ class AccountCheckRunner:
|
||||
status = str(item.get("status") or "")
|
||||
message = str(item.get("message") or "")
|
||||
try:
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
emit(
|
||||
db,
|
||||
@@ -422,21 +423,29 @@ class AccountCheckRunner:
|
||||
if self._shared_proxy_fetcher:
|
||||
self._shared_proxy_fetcher.warmup_whitelist()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.batch.concurrency) as executor:
|
||||
futures = []
|
||||
for index, account in enumerate(self.accounts):
|
||||
if self._stop.is_set():
|
||||
self._set_item(
|
||||
index,
|
||||
status="stopped",
|
||||
message="已停止",
|
||||
finished_at=_now(),
|
||||
)
|
||||
continue
|
||||
futures.append(executor.submit(self._run_one, index, account))
|
||||
futures = []
|
||||
for index, account in enumerate(self.accounts):
|
||||
if self._stop.is_set():
|
||||
self._set_item(
|
||||
index,
|
||||
status="stopped",
|
||||
message="已停止",
|
||||
finished_at=_now(),
|
||||
)
|
||||
continue
|
||||
future = submit(
|
||||
TaskLane.ACCOUNT_CHECK,
|
||||
self._run_one,
|
||||
index,
|
||||
account,
|
||||
stopped=self._stop.is_set,
|
||||
)
|
||||
if future is None:
|
||||
break
|
||||
futures.append(future)
|
||||
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
|
||||
self._set_batch(
|
||||
status="error", message=f"批次执行异常: {exc}", finished_at=_now()
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from concurrent.futures import as_completed
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
@@ -34,6 +34,7 @@ from ...douyu_account_probe import (
|
||||
probe_session,
|
||||
)
|
||||
from ...douyu_service import cookie_holding_filter, cookie_operable_filter
|
||||
from ...task_runtime import TaskLane, submit
|
||||
|
||||
JOB_KEY = "ck_health"
|
||||
FLUSH_EVERY = 25
|
||||
@@ -146,11 +147,14 @@ def run_ck_health(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
|
||||
results: dict[int, dict[str, Any]] = {}
|
||||
concurrency = max(min(int(job.concurrency or 1), 16), 1)
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = {pool.submit(_detect, account): account for account in accounts}
|
||||
for future in as_completed(futures):
|
||||
account = futures[future]
|
||||
results[account.id] = future.result()
|
||||
futures = {}
|
||||
for account in accounts:
|
||||
future = submit(TaskLane.ACCOUNT_CHECK, _detect, account)
|
||||
if future is not None:
|
||||
futures[future] = account
|
||||
for future in as_completed(futures):
|
||||
account = futures[future]
|
||||
results[account.id] = future.result()
|
||||
|
||||
# 只写检测结果三件套;cookie / status / attempt_* 一律不动。
|
||||
valid = invalid = errors = 0
|
||||
|
||||
@@ -20,7 +20,7 @@ from typing import Any
|
||||
from sqlalchemy import func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....database import SessionLocal
|
||||
from ....database import WorkerSessionLocal
|
||||
from ....models import Account, AutomationJob, LoginTask, User
|
||||
from ....models import ProxyConfig as ProxyConfigModel
|
||||
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
|
||||
@@ -148,7 +148,7 @@ def settle_fresh_results(
|
||||
退避时间,其中 2 条还被标成「极验」进了待人工队列。SQLite 的读一致性与 MySQL
|
||||
不同,这类问题在 SQLite 上复现不出来。
|
||||
"""
|
||||
session = SessionLocal()
|
||||
session = WorkerSessionLocal()
|
||||
try:
|
||||
fresh, order = _load_batch_state(session, batch_id)
|
||||
counters = settle_results(
|
||||
@@ -241,7 +241,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
_, max_total_time = get_relogin_limits()
|
||||
|
||||
# 批次用独立会话(runner 会在自己的线程/会话里写库),跑完再回来读结果。
|
||||
runner_db = SessionLocal()
|
||||
runner_db = WorkerSessionLocal()
|
||||
runner = LoginBatchRunner(
|
||||
db=runner_db,
|
||||
account_ids=account_ids,
|
||||
|
||||
@@ -24,7 +24,7 @@ from typing import Any
|
||||
from loguru import logger
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...database import SessionLocal
|
||||
from ...database import WorkerSessionLocal
|
||||
from ...models import AutomationJob, AutomationRun
|
||||
|
||||
# 调度轮询间隔,也是作业的运行节奏:每个 tick 跑一轮。
|
||||
@@ -71,7 +71,7 @@ _REGISTRY: dict[str, JobSpec] = {}
|
||||
|
||||
#: 同时最多几个作业在跑。作业之间互不依赖,并行是为了**慢作业不挡住快作业**
|
||||
#: (以前串行:一个作业跑 4 分钟,CK 巡检就得等 4 分钟)。
|
||||
_JOB_MAX_WORKERS = 4
|
||||
_JOB_MAX_WORKERS = 2
|
||||
|
||||
#: 运行间隔的下限:太短会变成忙轮询,没有意义
|
||||
MIN_RUN_INTERVAL_SECONDS = 30
|
||||
@@ -190,7 +190,7 @@ def _run_job_isolated(job_key: str) -> dict[str, Any] | None:
|
||||
多个作业共用一个 session 会互相踩事务(而他们现在是真的并行了)。
|
||||
重新按 key 取行,是因为调度线程里的那行可能已经过期。
|
||||
"""
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
job = db.query(AutomationJob).filter(AutomationJob.key == job_key).first()
|
||||
spec = _REGISTRY.get(job_key)
|
||||
@@ -337,7 +337,7 @@ def _run_one(db: Session, job: AutomationJob, spec: JobSpec) -> dict[str, Any]:
|
||||
db.rollback()
|
||||
logger.exception("自动化作业 {} 提交状态失败,尝试独立连接回写", job.key)
|
||||
try:
|
||||
rescue_db = SessionLocal()
|
||||
rescue_db = WorkerSessionLocal()
|
||||
try:
|
||||
r = (
|
||||
rescue_db.query(AutomationRun)
|
||||
@@ -412,7 +412,7 @@ def run_job_now(db: Session, key: str) -> dict[str, Any]:
|
||||
|
||||
|
||||
def _loop() -> None:
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
ensure_jobs(db)
|
||||
except Exception: # noqa: BLE001 初始化失败不应让线程静默退出
|
||||
@@ -424,7 +424,7 @@ def _loop() -> None:
|
||||
"自动化调度器已启动:心跳 {}s,作业周期 {}s", TICK_SECONDS, JOB_INTERVAL_SECONDS
|
||||
)
|
||||
while not _stop_event.wait(TICK_SECONDS):
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
# wait=False:派发完就返回 —— tick 不再被慢作业拖住
|
||||
run_due_jobs(db, wait=False)
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Database-backed scheduler for scan-wait binding tasks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from concurrent.futures import Future
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .douyu_runner import DouyuBatchRunner
|
||||
from .task_runtime import TaskLane, submit
|
||||
|
||||
DEFAULT_INTERVAL_SECONDS = 1
|
||||
DEFAULT_BATCH_SIZE = 20
|
||||
CLAIM_SECONDS = 60
|
||||
|
||||
_stop_event = threading.Event()
|
||||
_thread: threading.Thread | None = None
|
||||
_lock = threading.Lock()
|
||||
_in_flight: set[int] = set()
|
||||
_in_flight_lock = threading.Lock()
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return max(minimum, min(maximum, value))
|
||||
|
||||
|
||||
def _claim_due_binding_tasks(limit: int) -> list[int]:
|
||||
db = WorkerSessionLocal()
|
||||
now = datetime.now(UTC)
|
||||
claimed: list[int] = []
|
||||
try:
|
||||
rows = (
|
||||
db.query(DouyuTask)
|
||||
.filter(
|
||||
DouyuTask.task_type == "get_bind_qr",
|
||||
DouyuTask.status == "running",
|
||||
DouyuTask.queue_lane == "binding_poll",
|
||||
DouyuTask.next_run_at.isnot(None),
|
||||
DouyuTask.next_run_at <= now,
|
||||
)
|
||||
.order_by(DouyuTask.next_run_at.asc(), DouyuTask.id.asc())
|
||||
.with_for_update(skip_locked=True)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
for row in rows:
|
||||
task_id = int(row.id)
|
||||
with _in_flight_lock:
|
||||
if task_id in _in_flight:
|
||||
continue
|
||||
_in_flight.add(task_id)
|
||||
row.next_run_at = now + timedelta(seconds=CLAIM_SECONDS)
|
||||
claimed.append(task_id)
|
||||
db.commit()
|
||||
return claimed
|
||||
except Exception:
|
||||
db.rollback()
|
||||
with _in_flight_lock:
|
||||
for task_id in claimed:
|
||||
_in_flight.discard(task_id)
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def poll_binding_task_once(task_id: int) -> None:
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
task = (
|
||||
db.query(DouyuTask)
|
||||
.options(joinedload(DouyuTask.account))
|
||||
.filter(DouyuTask.id == int(task_id))
|
||||
.first()
|
||||
)
|
||||
if (
|
||||
task is None
|
||||
or task.status != "running"
|
||||
or task.task_type != "get_bind_qr"
|
||||
or task.queue_lane != "binding_poll"
|
||||
):
|
||||
return
|
||||
account = task.account
|
||||
if account is None or not (account.cookie or ""):
|
||||
task.queue_lane = ""
|
||||
task.next_run_at = None
|
||||
runner = DouyuBatchRunner(
|
||||
db,
|
||||
task.batch_id,
|
||||
"get_bind_qr",
|
||||
close_db=False,
|
||||
)
|
||||
runner._mark_task(db, task, "failed", "账号 Cookie 已失效,无法继续检测绑定")
|
||||
return
|
||||
runner = DouyuBatchRunner(
|
||||
db,
|
||||
task.batch_id,
|
||||
"get_bind_qr",
|
||||
close_db=False,
|
||||
)
|
||||
config = runner._config_info(db)
|
||||
runner.poll_bind_role_once(db, task, account, config)
|
||||
except Exception as exc: # noqa: BLE001 - the claimed row is retried later
|
||||
db.rollback()
|
||||
logger.warning("绑定扫码定时检测失败 task_id={}: {}", task_id, exc)
|
||||
finally:
|
||||
db.close()
|
||||
with _in_flight_lock:
|
||||
_in_flight.discard(int(task_id))
|
||||
|
||||
|
||||
def _release_callback(_future: Future) -> None:
|
||||
try:
|
||||
_future.result()
|
||||
except Exception as exc: # noqa: BLE001 - worker already logs; keep scheduler alive
|
||||
logger.warning("绑定扫码任务异常: {}", exc)
|
||||
|
||||
|
||||
def run_binding_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
|
||||
claimed = _claim_due_binding_tasks(max(1, limit))
|
||||
submitted = 0
|
||||
for task_id in claimed:
|
||||
future = submit(
|
||||
TaskLane.BINDING_POLL,
|
||||
poll_binding_task_once,
|
||||
task_id,
|
||||
stopped=_stop_event.is_set,
|
||||
)
|
||||
if future is None:
|
||||
with _in_flight_lock:
|
||||
_in_flight.discard(task_id)
|
||||
break
|
||||
future.add_done_callback(_release_callback)
|
||||
submitted += 1
|
||||
return submitted
|
||||
|
||||
|
||||
def _run() -> None:
|
||||
interval = _env_int(
|
||||
"BINDING_POLL_DISPATCH_SECONDS",
|
||||
DEFAULT_INTERVAL_SECONDS,
|
||||
1,
|
||||
30,
|
||||
)
|
||||
while not _stop_event.is_set():
|
||||
try:
|
||||
batch_size = _env_int(
|
||||
"BINDING_POLL_DISPATCH_BATCH",
|
||||
DEFAULT_BATCH_SIZE,
|
||||
1,
|
||||
200,
|
||||
)
|
||||
run_binding_poll_dispatch_once(batch_size)
|
||||
except Exception as exc: # noqa: BLE001 - scheduler must survive one bad tick
|
||||
logger.warning("绑定扫码调度轮次失败: {}", exc)
|
||||
_stop_event.wait(interval)
|
||||
|
||||
|
||||
def start_binding_poll_scheduler() -> bool:
|
||||
global _thread
|
||||
with _lock:
|
||||
if _thread is not None and _thread.is_alive():
|
||||
return False
|
||||
_stop_event.clear()
|
||||
_thread = threading.Thread(
|
||||
target=_run,
|
||||
name="binding-poll-scheduler",
|
||||
daemon=True,
|
||||
)
|
||||
_thread.start()
|
||||
logger.info("绑定扫码定时调度器已启动")
|
||||
return True
|
||||
|
||||
|
||||
def stop_binding_poll_scheduler() -> None:
|
||||
global _thread
|
||||
_stop_event.set()
|
||||
with _lock:
|
||||
thread = _thread
|
||||
_thread = None
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=3)
|
||||
|
||||
|
||||
def scheduler_status() -> dict[str, object]:
|
||||
with _lock:
|
||||
running = _thread is not None and _thread.is_alive()
|
||||
with _in_flight_lock:
|
||||
in_flight = len(_in_flight)
|
||||
return {
|
||||
"running": running,
|
||||
"in_flight": in_flight,
|
||||
"claim_seconds": CLAIM_SECONDS,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"poll_binding_task_once",
|
||||
"run_binding_poll_dispatch_once",
|
||||
"scheduler_status",
|
||||
"start_binding_poll_scheduler",
|
||||
"stop_binding_poll_scheduler",
|
||||
]
|
||||
@@ -9,7 +9,7 @@ import asyncio
|
||||
import os
|
||||
import threading
|
||||
|
||||
from web.backend.database import SessionLocal
|
||||
from web.backend.database import WorkerSessionLocal
|
||||
|
||||
from .douyu_concurrency import clamp_batch_task_concurrency
|
||||
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||
@@ -39,10 +39,15 @@ def start_douyu_batch(
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
log_queue: asyncio.Queue | None = None,
|
||||
) -> None:
|
||||
"""创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。"""
|
||||
loop = loop or asyncio.get_running_loop()
|
||||
log_queue = log_queue or asyncio.Queue()
|
||||
thread_db = SessionLocal()
|
||||
"""Create the batch coordinator; websocket logging is optional."""
|
||||
if loop is None:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
if log_queue is None and loop is not None:
|
||||
log_queue = asyncio.Queue()
|
||||
thread_db = WorkerSessionLocal()
|
||||
runner = DouyuBatchRunner(
|
||||
db=thread_db,
|
||||
batch_id=batch_id,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""斗鱼后台任务的进程级并发限制。
|
||||
"""斗鱼批次级并发限制。
|
||||
|
||||
任务线程和数据库连接池共享同一个 Uvicorn 进程。这里把「每批并发」和
|
||||
「全进程同时执行数」分开限制,避免多个批次叠加后把请求线程和连接池挤空。
|
||||
Process-wide execution is owned by ``task_runtime`` lanes; this module only
|
||||
keeps the per-batch cap and the overview sub-limit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -23,16 +23,11 @@ DOUYU_BATCH_TASK_CONCURRENCY = _env_positive_int(
|
||||
"DOUYU_BATCH_TASK_CONCURRENCY",
|
||||
5,
|
||||
)
|
||||
DOUYU_MAX_RUNNING_TASKS = _env_positive_int(
|
||||
"DOUYU_MAX_RUNNING_TASKS",
|
||||
12,
|
||||
)
|
||||
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS = _env_positive_int(
|
||||
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
|
||||
2,
|
||||
)
|
||||
|
||||
_task_slots = threading.BoundedSemaphore(DOUYU_MAX_RUNNING_TASKS)
|
||||
_overview_slots = threading.BoundedSemaphore(DOUYU_MAX_ACTIVE_OVERVIEW_TASKS)
|
||||
|
||||
|
||||
@@ -47,15 +42,6 @@ def _acquire(slots: threading.BoundedSemaphore, stopped: Callable[[], bool]) ->
|
||||
return False
|
||||
|
||||
|
||||
def acquire_running_task_slot(stopped: Callable[[], bool]) -> bool:
|
||||
"""等待一个全进程任务槽;停止时返回 False。"""
|
||||
return _acquire(_task_slots, stopped)
|
||||
|
||||
|
||||
def release_running_task_slot() -> None:
|
||||
_task_slots.release()
|
||||
|
||||
|
||||
def acquire_overview_slot(stopped: Callable[[], bool]) -> bool:
|
||||
"""等待一个一键查询槽;该限制跨所有批次共享。"""
|
||||
return _acquire(_overview_slots, stopped)
|
||||
@@ -68,10 +54,7 @@ def release_overview_slot() -> None:
|
||||
__all__ = [
|
||||
"DOUYU_BATCH_TASK_CONCURRENCY",
|
||||
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
|
||||
"DOUYU_MAX_RUNNING_TASKS",
|
||||
"acquire_overview_slot",
|
||||
"acquire_running_task_slot",
|
||||
"clamp_batch_task_concurrency",
|
||||
"release_overview_slot",
|
||||
"release_running_task_slot",
|
||||
]
|
||||
|
||||
@@ -2,21 +2,20 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import time
|
||||
from concurrent.futures import as_completed
|
||||
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from core.douyu import DouyuActivityError
|
||||
|
||||
from ..database import SessionLocal
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .cookie_check_service import check_douyu_cookie
|
||||
from .douyu_account_probe import persist_probe_result, probe_session
|
||||
from .douyu_concurrency import (
|
||||
acquire_overview_slot,
|
||||
acquire_running_task_slot,
|
||||
release_overview_slot,
|
||||
release_running_task_slot,
|
||||
)
|
||||
from .douyu_runner_bind import BindMixin
|
||||
from .douyu_runner_core import ( # noqa: F401 (douyu_batch_registry 供 routers 重导出)
|
||||
@@ -30,6 +29,11 @@ from .douyu_runner_manual import ManualMixin
|
||||
from .douyu_runner_overview import OverviewMixin
|
||||
from .douyu_runner_xpd import XpdMixin
|
||||
from .douyu_service import update_account_profile_from_cookie
|
||||
from .task_runtime import douyu_lane_for_task, submit
|
||||
|
||||
# Compatibility alias for tests and integrations that replace the worker
|
||||
# session factory at the runner boundary.
|
||||
SessionLocal = WorkerSessionLocal
|
||||
|
||||
|
||||
class DouyuBatchRunner(
|
||||
@@ -47,19 +51,13 @@ class DouyuBatchRunner(
|
||||
def _execute_one(self, task_id: int, config: dict, total: int):
|
||||
overview_task = self.task_type == "query_account_overview"
|
||||
overview_slot = False
|
||||
running_slot = False
|
||||
if overview_task:
|
||||
overview_slot = acquire_overview_slot(self._stop.is_set)
|
||||
if not overview_slot:
|
||||
return
|
||||
try:
|
||||
running_slot = acquire_running_task_slot(self._stop.is_set)
|
||||
if not running_slot:
|
||||
return
|
||||
self._execute_one_with_slot(task_id, config, total)
|
||||
finally:
|
||||
if running_slot:
|
||||
release_running_task_slot()
|
||||
if overview_slot:
|
||||
release_overview_slot()
|
||||
|
||||
@@ -221,19 +219,52 @@ class DouyuBatchRunner(
|
||||
self._push_task_event(task)
|
||||
|
||||
total = len(tasks)
|
||||
with ThreadPoolExecutor(max_workers=self.concurrency) as executor:
|
||||
futures = []
|
||||
for task in tasks:
|
||||
if self._stop.is_set():
|
||||
break
|
||||
futures.append(
|
||||
executor.submit(self._execute_one, task.id, config, total)
|
||||
lane = douyu_lane_for_task(self.task_type)
|
||||
pending: dict = {}
|
||||
next_index = 0
|
||||
while next_index < total or pending:
|
||||
while (
|
||||
not self._stop.is_set()
|
||||
and next_index < total
|
||||
and len(pending) < self.concurrency
|
||||
):
|
||||
task = tasks[next_index]
|
||||
future = submit(
|
||||
lane,
|
||||
self._execute_one,
|
||||
task.id,
|
||||
config,
|
||||
total,
|
||||
stopped=self._stop.is_set,
|
||||
)
|
||||
for future in as_completed(futures):
|
||||
if future is None:
|
||||
break
|
||||
pending[future] = (int(task.id), time.monotonic())
|
||||
next_index += 1
|
||||
|
||||
if not pending:
|
||||
break
|
||||
|
||||
try:
|
||||
completed = list(as_completed(pending, timeout=0.5))
|
||||
except TimeoutError:
|
||||
continue
|
||||
for future in completed:
|
||||
task_id, submitted_at = pending.pop(future)
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
|
||||
self._push_log("error", f"Worker 异常: {exc}")
|
||||
self._push_log(
|
||||
"error",
|
||||
f"Worker 异常 task_id={task_id}: {exc}",
|
||||
)
|
||||
else:
|
||||
waited = time.monotonic() - submitted_at
|
||||
if waited >= 5:
|
||||
self._push_log(
|
||||
"info",
|
||||
f"任务 task_id={task_id} 在 {lane.value} 槽排队 {waited:.1f}s",
|
||||
)
|
||||
|
||||
if self._stop.is_set():
|
||||
self._push_log("warning", f"斗鱼任务批次 {self.batch_id} 已停止")
|
||||
|
||||
@@ -632,6 +632,172 @@ class BindMixin:
|
||||
)
|
||||
return "", result
|
||||
|
||||
@staticmethod
|
||||
def _schedule_next_bind_poll(
|
||||
task: DouyuTask,
|
||||
*,
|
||||
now: datetime,
|
||||
interval_seconds: int,
|
||||
) -> None:
|
||||
task.queue_lane = "binding_poll"
|
||||
task.next_run_at = now + timedelta(seconds=max(1, interval_seconds))
|
||||
task.last_run_at = now
|
||||
task.attempt_count = int(task.attempt_count or 0) + 1
|
||||
|
||||
def poll_bind_role_once(
|
||||
self,
|
||||
db: Session,
|
||||
task: DouyuTask,
|
||||
account: Account,
|
||||
config: dict,
|
||||
) -> None:
|
||||
"""Run one scheduled binding poll and immediately release the worker."""
|
||||
now = datetime.now(UTC)
|
||||
task.last_run_at = now
|
||||
task.attempt_count = int(task.attempt_count or 0) + 1
|
||||
result = dict(task.result or {})
|
||||
query_aliases = [
|
||||
str(value)
|
||||
for value in (result.get("query_act_aliases") or [])
|
||||
if str(value or "").strip()
|
||||
]
|
||||
if not query_aliases:
|
||||
query_aliases = self._query_bind_act_aliases(config)
|
||||
deadline = task.run_deadline_at
|
||||
if deadline is not None and now >= deadline:
|
||||
result.update(
|
||||
{
|
||||
"bind_polling": False,
|
||||
"bind_ready_for_confirm": False,
|
||||
"bind_phase": "role_timeout",
|
||||
}
|
||||
)
|
||||
task.queue_lane = ""
|
||||
task.next_run_at = None
|
||||
self._mark_task(
|
||||
db,
|
||||
task,
|
||||
"failed",
|
||||
"绑定二维码已过期,未识别到新扫码角色",
|
||||
result,
|
||||
)
|
||||
return
|
||||
|
||||
client = self._client(str(account.cookie or ""))
|
||||
candidates = self._fetch_bind_info_candidates(client, query_aliases)
|
||||
if not candidates:
|
||||
attempts = max(1, int(task.attempt_count or 1))
|
||||
backoff = min(30, max(DOUYU_BIND_ROLE_POLL_INTERVAL, attempts * 2))
|
||||
result["bind_poll_error"] = "所有 actAlias 查询绑定信息失败"
|
||||
result["bind_polling"] = True
|
||||
result["bind_phase"] = "waiting_scan"
|
||||
self._schedule_next_bind_poll(task, now=now, interval_seconds=backoff)
|
||||
self._update_task_progress(
|
||||
db,
|
||||
task,
|
||||
"running",
|
||||
"等待扫码绑定:角色查询暂时失败",
|
||||
result,
|
||||
sync_platform=False,
|
||||
)
|
||||
return
|
||||
|
||||
baseline_role_name = str(result.get("baseline_role_name") or "")
|
||||
baseline_is_bound_act = bool(result.get("baseline_is_bound_act"))
|
||||
baseline_info = result.get("before_bind_info")
|
||||
if not isinstance(baseline_info, dict):
|
||||
baseline_info = None
|
||||
bind_info = (
|
||||
self._pick_bind_info(
|
||||
candidates,
|
||||
baseline_role_name=baseline_role_name,
|
||||
baseline_is_bound_act=baseline_is_bound_act,
|
||||
baseline_info=baseline_info,
|
||||
prefer_pending=True,
|
||||
prefer_aliases=query_aliases,
|
||||
)
|
||||
or candidates[0]
|
||||
)
|
||||
snapshot = self._bind_snapshot(bind_info)
|
||||
is_pending_role = self._is_pending_role(
|
||||
bind_info,
|
||||
baseline_role_name=baseline_role_name,
|
||||
baseline_is_bound_act=baseline_is_bound_act,
|
||||
baseline_info=baseline_info,
|
||||
)
|
||||
query_alias = str(bind_info.get("act_alias") or "")
|
||||
summary = (
|
||||
f"act={query_alias or '-'} "
|
||||
f"{self._format_bind_summary(bind_info, pending=is_pending_role)}"
|
||||
)
|
||||
|
||||
if is_pending_role:
|
||||
task.queue_lane = ""
|
||||
task.next_run_at = None
|
||||
result.update(
|
||||
{
|
||||
**snapshot,
|
||||
"act_alias": query_alias,
|
||||
"query_act_alias": query_alias,
|
||||
"bind_ready_for_confirm": True,
|
||||
"bind_confirmed": False,
|
||||
"bind_phase": "role_ready",
|
||||
"bind_polling": False,
|
||||
"poll_count": int(result.get("poll_count") or 0) + 1,
|
||||
"bind_summary": summary,
|
||||
}
|
||||
)
|
||||
account.updated_at = now
|
||||
self._mark_task(
|
||||
db,
|
||||
task,
|
||||
"success",
|
||||
f"已识别角色: {snapshot['role_name']},待确认绑定",
|
||||
result,
|
||||
)
|
||||
return
|
||||
|
||||
if snapshot["role_name"] and snapshot["is_bound_act"]:
|
||||
result["current_role_name"] = snapshot["role_name"]
|
||||
result["current_area_name"] = snapshot["area_name"]
|
||||
result["current_plat_name"] = snapshot["plat_name"]
|
||||
result.update(
|
||||
{
|
||||
"bind_info": bind_info,
|
||||
"act_alias": query_alias,
|
||||
"query_act_alias": query_alias,
|
||||
"is_bound_act": snapshot["is_bound_act"],
|
||||
"is_bound_role": snapshot["is_bound_role"],
|
||||
"is_bound_account": snapshot["is_bound_account"],
|
||||
"need_bind_act": snapshot["need_bind_act"],
|
||||
"need_bind_role": snapshot["need_bind_role"],
|
||||
"change_role_wait_time": snapshot["change_role_wait_time"],
|
||||
"can_change_role": snapshot["can_change_role"],
|
||||
"role_name": "",
|
||||
"area_name": "",
|
||||
"plat_name": "",
|
||||
"bind_ready_for_confirm": False,
|
||||
"bind_confirmed": False,
|
||||
"bind_phase": "waiting_scan",
|
||||
"bind_polling": True,
|
||||
"poll_count": int(result.get("poll_count") or 0) + 1,
|
||||
"bind_summary": summary,
|
||||
}
|
||||
)
|
||||
self._schedule_next_bind_poll(
|
||||
task,
|
||||
now=now,
|
||||
interval_seconds=DOUYU_BIND_ROLE_POLL_INTERVAL,
|
||||
)
|
||||
self._update_task_progress(
|
||||
db,
|
||||
task,
|
||||
"running",
|
||||
f"等待扫码绑定 ({summary})",
|
||||
result,
|
||||
sync_platform=False,
|
||||
)
|
||||
|
||||
def _execute_get_bind_qr(
|
||||
self, db: Session, task: DouyuTask, account: Account, cookie: str, config: dict
|
||||
):
|
||||
@@ -763,40 +929,26 @@ class BindMixin:
|
||||
"qr_expires_at": (
|
||||
qr_generated_at + timedelta(seconds=DOUYU_BIND_ROLE_POLL_SECONDS)
|
||||
).isoformat(),
|
||||
"baseline_role_name": current_role_name,
|
||||
"baseline_is_bound_act": before_snapshot["is_bound_act"],
|
||||
"poll_count": 0,
|
||||
}
|
||||
account.updated_at = datetime.now(UTC)
|
||||
# 关键:先把二维码 progress 出去,前端 running 期间即可弹窗扫码。
|
||||
task.queue_lane = "binding_poll"
|
||||
task.next_run_at = qr_generated_at + timedelta(
|
||||
seconds=DOUYU_BIND_ROLE_POLL_INTERVAL
|
||||
)
|
||||
task.run_deadline_at = qr_generated_at + timedelta(
|
||||
seconds=DOUYU_BIND_ROLE_POLL_SECONDS
|
||||
)
|
||||
task.last_run_at = qr_generated_at
|
||||
task.attempt_count = 0
|
||||
# Publish the QR immediately, then return without holding a worker while
|
||||
# the customer scans. The durable scheduler resumes role polling.
|
||||
self._update_task_progress(
|
||||
db, task, "running", "已生成绑定二维码,等待扫码绑定", result
|
||||
)
|
||||
|
||||
role_name, result = self._wait_bind_role_result(
|
||||
client,
|
||||
db,
|
||||
task,
|
||||
account,
|
||||
query_aliases,
|
||||
result,
|
||||
baseline_role_name=current_role_name,
|
||||
baseline_is_bound_act=before_snapshot["is_bound_act"],
|
||||
baseline_info=before,
|
||||
)
|
||||
if role_name:
|
||||
self._mark_task(
|
||||
db, task, "success", f"已识别角色: {role_name},待确认绑定", result
|
||||
)
|
||||
return
|
||||
if result.get("bind_phase") == "stopped":
|
||||
self._mark_task(db, task, "stopped", "任务已停止", result)
|
||||
return
|
||||
last_summary = str(result.get("bind_summary") or "")
|
||||
timeout_msg = "已生成绑定二维码,未检测到新扫码角色"
|
||||
if current_role_name:
|
||||
timeout_msg = f"{timeout_msg}(当前仍是 {current_role_name})"
|
||||
if last_summary:
|
||||
timeout_msg = f"{timeout_msg} | {last_summary}"
|
||||
self._mark_task(db, task, "success", timeout_msg, result)
|
||||
|
||||
def _execute_confirm_bind(
|
||||
self, db: Session, task: DouyuTask, account: Account, cookie: str, config: dict
|
||||
):
|
||||
|
||||
@@ -228,12 +228,12 @@ class DouyuBatchRunnerCore:
|
||||
if not enabled:
|
||||
return
|
||||
try:
|
||||
from ..database import SessionLocal
|
||||
from ..database import WorkerSessionLocal
|
||||
from .platform_fulfillment.service import sync_task_from_binding_runner
|
||||
|
||||
# 当前批次会话刚刚提交过任务结果。履约投影使用独立短会话,避免
|
||||
# get_task_detail 留下的只读事务在下一轮外部 HTTP 轮询期间占连接。
|
||||
sync_db = SessionLocal()
|
||||
sync_db = WorkerSessionLocal()
|
||||
try:
|
||||
sync_task_from_binding_runner(sync_db, douyu_task_id=int(douyu_task_id))
|
||||
finally:
|
||||
@@ -339,6 +339,8 @@ class DouyuBatchRunnerCore:
|
||||
status: str,
|
||||
message: str,
|
||||
result: dict | None = None,
|
||||
*,
|
||||
sync_platform: bool = True,
|
||||
) -> None:
|
||||
task_id = int(task.id)
|
||||
has_platform_task = task.platform_task_id is not None
|
||||
@@ -348,7 +350,8 @@ class DouyuBatchRunnerCore:
|
||||
task.result = slim_douyu_task_result_for_storage(result)
|
||||
db.commit()
|
||||
self._push_task_event(task)
|
||||
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
|
||||
if sync_platform:
|
||||
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
|
||||
|
||||
def _upsert_goods(self, db: Session, goods: list[dict]) -> None:
|
||||
upsert_goods_snapshot(db, goods)
|
||||
@@ -455,8 +458,8 @@ class DouyuBatchRegistry:
|
||||
def register(
|
||||
self,
|
||||
batch_id: str,
|
||||
log_queue: asyncio.Queue,
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
log_queue: asyncio.Queue | None,
|
||||
loop: asyncio.AbstractEventLoop | None,
|
||||
runner: DouyuBatchRunner,
|
||||
):
|
||||
self._batches[batch_id] = {
|
||||
|
||||
@@ -7,7 +7,7 @@ import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import and_, func, select
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.douyu.cookie_utils import cookie_value
|
||||
@@ -468,10 +468,22 @@ def cleanup_orphan_douyu_tasks(
|
||||
elif active_batch_ids is not None and active_batch_ids:
|
||||
query = query.filter(~DouyuTask.batch_id.in_(list(active_batch_ids)))
|
||||
|
||||
now = datetime.now(UTC)
|
||||
# Scan waits are durable. A restarted scheduler can resume them from the
|
||||
# database, so startup cleanup must not turn them into stopped tasks.
|
||||
query = query.filter(
|
||||
or_(
|
||||
DouyuTask.task_type != "get_bind_qr",
|
||||
DouyuTask.queue_lane != "binding_poll",
|
||||
DouyuTask.next_run_at.is_(None),
|
||||
DouyuTask.run_deadline_at.is_(None),
|
||||
DouyuTask.run_deadline_at <= now,
|
||||
)
|
||||
)
|
||||
|
||||
tasks = query.all()
|
||||
if not tasks:
|
||||
return 0
|
||||
now = datetime.now(UTC)
|
||||
account_ids = {int(task.account_id) for task in tasks if task.account_id}
|
||||
accounts = (
|
||||
{
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from concurrent.futures import as_completed
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
|
||||
@@ -13,7 +13,7 @@ from core.huya.auto_register import HuyaAutoRegisterResult, register_huya_with_s
|
||||
from core.huya.cookie_utils import normalize_huya_cookie
|
||||
from core.sms_provider import SmsLine
|
||||
|
||||
from ..database import SessionLocal
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import (
|
||||
HuyaRegisterBatch as HuyaRegisterBatchModel,
|
||||
)
|
||||
@@ -28,6 +28,7 @@ from ..models import (
|
||||
)
|
||||
from .huya_service import upsert_huya_cookie
|
||||
from .proxy_service import build_proxy_runtime
|
||||
from .task_runtime import TaskLane, submit
|
||||
|
||||
# 运行中状态:服务中断后视为未完成,可继续
|
||||
RUNNING_ITEM_STATUSES = frozenset({"sending", "waiting", "changing", "logging"})
|
||||
@@ -209,7 +210,7 @@ def load_batch_snapshot(
|
||||
batch_id: str, *, recover_interrupted: bool = True
|
||||
) -> dict | None:
|
||||
"""从数据库加载批次详情;若服务中断则标记为 interrupted。"""
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
batch_row = (
|
||||
db.query(HuyaRegisterBatchModel)
|
||||
@@ -258,7 +259,7 @@ def list_batch_summaries(
|
||||
) -> list[dict]:
|
||||
"""列出最近的注册批次摘要。live_batch_ids 中的 running 保持运行中。"""
|
||||
live = live_batch_ids or set()
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
rows = (
|
||||
db.query(HuyaRegisterBatchModel)
|
||||
@@ -335,7 +336,7 @@ def export_success_logs_text(
|
||||
limit: int = 5000,
|
||||
) -> str:
|
||||
"""从成功流水表导出 txt。"""
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
query = db.query(HuyaRegisterSuccessLog).order_by(
|
||||
HuyaRegisterSuccessLog.id.asc()
|
||||
@@ -368,7 +369,7 @@ def list_success_logs(
|
||||
limit: int = 200,
|
||||
) -> list[dict]:
|
||||
"""列出成功流水(含密码,供管理端展示/导出)。"""
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
query = db.query(HuyaRegisterSuccessLog).order_by(
|
||||
HuyaRegisterSuccessLog.id.desc()
|
||||
@@ -450,7 +451,7 @@ class HuyaRegisterRunner:
|
||||
"""把批次汇总写回数据库。"""
|
||||
if not self.batch.db_id:
|
||||
return
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
row = (
|
||||
db.query(HuyaRegisterBatchModel)
|
||||
@@ -493,7 +494,7 @@ class HuyaRegisterRunner:
|
||||
item = self.batch.items[index]
|
||||
if not item.db_id:
|
||||
return
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
row = (
|
||||
db.query(HuyaRegisterItemModel)
|
||||
@@ -536,7 +537,7 @@ class HuyaRegisterRunner:
|
||||
self, index: int, result: HuyaAutoRegisterResult
|
||||
) -> tuple[int | None, str, str]:
|
||||
"""成功时:写账号 + 成功流水(成功一个写一条,立即可导出)。"""
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
# upsert_huya_cookie 内部会 commit 一次
|
||||
account = upsert_huya_cookie(
|
||||
@@ -737,22 +738,30 @@ class HuyaRegisterRunner:
|
||||
self.batch.message = f"代理白名单预热失败: {msg}"
|
||||
self._persist_batch_meta()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.batch.concurrency) as executor:
|
||||
futures = []
|
||||
for index in indices:
|
||||
if self._stop.is_set():
|
||||
self._set_item(
|
||||
index,
|
||||
status="stopped",
|
||||
message="已停止",
|
||||
finished_at=_now(),
|
||||
)
|
||||
continue
|
||||
item = self.sms_lines[index]
|
||||
futures.append(executor.submit(self._run_one, index, item))
|
||||
futures = []
|
||||
for index in indices:
|
||||
if self._stop.is_set():
|
||||
self._set_item(
|
||||
index,
|
||||
status="stopped",
|
||||
message="已停止",
|
||||
finished_at=_now(),
|
||||
)
|
||||
continue
|
||||
item = self.sms_lines[index]
|
||||
future = submit(
|
||||
TaskLane.HUYA_REGISTER,
|
||||
self._run_one,
|
||||
index,
|
||||
item,
|
||||
stopped=self._stop.is_set,
|
||||
)
|
||||
if future is None:
|
||||
break
|
||||
futures.append(future)
|
||||
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
|
||||
with self._lock:
|
||||
self.batch.status = "error"
|
||||
@@ -811,7 +820,7 @@ class HuyaRegisterRegistry:
|
||||
for index, item in enumerate(sms_lines)
|
||||
]
|
||||
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
batch_row = HuyaRegisterBatchModel(
|
||||
batch_id=batch_id,
|
||||
@@ -871,7 +880,7 @@ class HuyaRegisterRegistry:
|
||||
started_at=_now(),
|
||||
)
|
||||
# DB 同步为 running,避免返回 pending 导致前端误判
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
row = (
|
||||
db.query(HuyaRegisterBatchModel)
|
||||
@@ -936,7 +945,7 @@ class HuyaRegisterRegistry:
|
||||
empty_msg = "没有可继续的未完成/失败条目"
|
||||
run_label = "续跑未完成"
|
||||
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
batch_row = (
|
||||
db.query(HuyaRegisterBatchModel)
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from concurrent.futures import as_completed
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from core.huya.cookie_utils import normalize_huya_cookie
|
||||
|
||||
from ..database import SessionLocal
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import HuyaAccount, HuyaTask
|
||||
from .huya_runner_bind import BindMixin
|
||||
from .huya_runner_core import ( # noqa: F401 (huya_batch_registry 供 routers 重导出)
|
||||
@@ -20,6 +20,7 @@ from .huya_runner_elite import EliteMixin
|
||||
from .huya_runner_goods import GoodsMixin
|
||||
from .huya_runner_recharge import RechargeMixin
|
||||
from .huya_service import HUYA_CONFIG_FIELDS, ensure_huya_config, huya_config_value
|
||||
from .task_runtime import TaskLane, submit
|
||||
|
||||
|
||||
class HuyaBatchRunner(
|
||||
@@ -34,7 +35,7 @@ class HuyaBatchRunner(
|
||||
def _execute_one(
|
||||
self, task_id: int, account_info: dict, config_info: dict, total: int
|
||||
):
|
||||
worker_db = SessionLocal()
|
||||
worker_db = WorkerSessionLocal()
|
||||
try:
|
||||
task = worker_db.query(HuyaTask).filter(HuyaTask.id == task_id).first()
|
||||
account = (
|
||||
@@ -193,27 +194,29 @@ class HuyaBatchRunner(
|
||||
self._push_log("result", "")
|
||||
return
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.concurrency) as executor:
|
||||
futures = []
|
||||
for item in task_infos:
|
||||
if self._stop.is_set():
|
||||
self._push_log("warning", "任务已停止,跳过剩余账号")
|
||||
break
|
||||
futures.append(
|
||||
executor.submit(
|
||||
self._execute_one,
|
||||
item["task_id"],
|
||||
item["account_info"],
|
||||
config_info,
|
||||
total,
|
||||
)
|
||||
)
|
||||
futures = []
|
||||
for item in task_infos:
|
||||
if self._stop.is_set():
|
||||
self._push_log("warning", "任务已停止,跳过剩余账号")
|
||||
break
|
||||
future = submit(
|
||||
TaskLane.HUYA_BATCH,
|
||||
self._execute_one,
|
||||
item["task_id"],
|
||||
item["account_info"],
|
||||
config_info,
|
||||
total,
|
||||
stopped=self._stop.is_set,
|
||||
)
|
||||
if future is None:
|
||||
break
|
||||
futures.append(future)
|
||||
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
|
||||
self._push_log("error", f"虎牙 Worker 异常: {exc}")
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
|
||||
self._push_log("error", f"虎牙 Worker 异常: {exc}")
|
||||
|
||||
self._push_log("info", f"虎牙批次 {self.batch_id} 完成")
|
||||
self._push_log("result", "")
|
||||
|
||||
@@ -90,10 +90,10 @@ class HuyaBatchRunnerCore:
|
||||
account_id = getattr(self._wss_sessions, "account_id", None)
|
||||
if not account_id:
|
||||
return
|
||||
from ..database import SessionLocal
|
||||
from ..database import WorkerSessionLocal
|
||||
from .huya_service import persist_huya_session_cookie
|
||||
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
if persist_huya_session_cookie(
|
||||
db, int(account_id), original_cookie, filled_cookie
|
||||
|
||||
@@ -14,9 +14,10 @@ from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...database import SessionLocal
|
||||
from ...database import WorkerSessionLocal
|
||||
from ...models import IntegrationOutbox
|
||||
from .security import build_webhook_signature
|
||||
|
||||
@@ -74,19 +75,29 @@ def enqueue_integration_event(
|
||||
if existing is not None:
|
||||
return False
|
||||
|
||||
db.add(
|
||||
IntegrationOutbox(
|
||||
event_id=event_id,
|
||||
destination=destination,
|
||||
event_type=event_type,
|
||||
dedupe_key=normalized_dedupe,
|
||||
payload_json=body,
|
||||
status="pending",
|
||||
attempts=0,
|
||||
next_attempt_at=datetime.now(UTC),
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(
|
||||
IntegrationOutbox(
|
||||
event_id=event_id,
|
||||
destination=destination,
|
||||
event_type=event_type,
|
||||
dedupe_key=normalized_dedupe,
|
||||
payload_json=body,
|
||||
status="pending",
|
||||
attempts=0,
|
||||
next_attempt_at=datetime.now(UTC),
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
except IntegrityError:
|
||||
# A concurrent transaction may have inserted the same stable dedupe key
|
||||
# after our pre-check. The nested transaction keeps the outer business
|
||||
# transaction usable and treats the event as already enqueued.
|
||||
if normalized_dedupe:
|
||||
return False
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
@@ -118,7 +129,7 @@ def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
|
||||
|
||||
|
||||
def _claim_due_events(limit: int) -> list[_ClaimedEvent]:
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
now = datetime.now(UTC)
|
||||
try:
|
||||
(
|
||||
@@ -202,7 +213,7 @@ def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None:
|
||||
|
||||
|
||||
def _mark_sent(event_id: str) -> None:
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
(
|
||||
db.query(IntegrationOutbox)
|
||||
@@ -229,7 +240,7 @@ def _mark_failed(event: _ClaimedEvent, error: Exception) -> None:
|
||||
)
|
||||
dead = attempts >= max_attempts
|
||||
message = str(error)[:2000]
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
(
|
||||
db.query(IntegrationOutbox)
|
||||
|
||||
@@ -5,7 +5,7 @@ import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from concurrent.futures import as_completed
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
@@ -32,6 +32,7 @@ from .douyu_account_probe import persist_probe_result, probe_session
|
||||
from .douyu_service import LOGIN_ATTEMPT_IN_FLIGHT_STATUSES
|
||||
from .login_failure import classify_failure, failure_kind_label, is_account_fault
|
||||
from .proxy_service import build_proxy_runtime
|
||||
from .task_runtime import TaskLane, submit
|
||||
|
||||
CHECK_STATUS_MESSAGES = {
|
||||
"account_cancelled": "账号已注销",
|
||||
@@ -296,7 +297,7 @@ def _persist_login_cookie_verification(
|
||||
|
||||
|
||||
class LoginBatchRunner:
|
||||
"""批量登录执行器,在线程中运行,通过 ThreadPoolExecutor 并发登录多个账号。"""
|
||||
"""批量登录执行器,通过进程级登录槽并发执行账号。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -357,7 +358,7 @@ class LoginBatchRunner:
|
||||
message: str = "重新登录已停止,旧 Cookie 已保留",
|
||||
) -> None:
|
||||
"""把未开始或已中断的重登尝试收敛为终态,旧 CK 原封不动。"""
|
||||
worker_db = SessionLocal()
|
||||
worker_db = WorkerSessionLocal()
|
||||
try:
|
||||
task = worker_db.query(LoginTask).filter(LoginTask.id == task_id).first()
|
||||
if task and _attempt_in_flight(task):
|
||||
@@ -376,7 +377,7 @@ class LoginBatchRunner:
|
||||
与 `_mark_relogin_stopped` 的区别只是入口:批次挂了但记录还没建出来时,
|
||||
这里是唯一能把「该账号的在途尝试」收干净的路径。
|
||||
"""
|
||||
worker_db = SessionLocal()
|
||||
worker_db = WorkerSessionLocal()
|
||||
try:
|
||||
tasks = (
|
||||
worker_db.query(LoginTask)
|
||||
@@ -432,7 +433,7 @@ class LoginBatchRunner:
|
||||
self._mark_relogin_stopped(task_id, "任务已停止,旧 Cookie 已保留")
|
||||
return
|
||||
|
||||
worker_db = SessionLocal()
|
||||
worker_db = WorkerSessionLocal()
|
||||
try:
|
||||
task = worker_db.query(LoginTask).filter(LoginTask.id == task_id).first()
|
||||
if not task:
|
||||
@@ -774,33 +775,33 @@ class LoginBatchRunner:
|
||||
self._push_log("result", "")
|
||||
return
|
||||
|
||||
# 并发执行登录,每个账号独立获取代理
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||||
futures = []
|
||||
for item_index, item in enumerate(task_infos):
|
||||
if self._stop.is_set():
|
||||
self._push_log("warning", "任务已停止,跳过剩余账号")
|
||||
# 所有模式都要把没轮到的尝试收敛掉,否则会一直停在
|
||||
# 「等待中」;已有 CK 的记录本身不受影响。
|
||||
for pending_item in task_infos[item_index:]:
|
||||
self._mark_relogin_stopped(
|
||||
pending_item["task_id"], "任务已停止,旧 Cookie 已保留"
|
||||
)
|
||||
break
|
||||
future = executor.submit(
|
||||
self._execute_one,
|
||||
item["task_id"],
|
||||
item["acc_info"],
|
||||
total,
|
||||
)
|
||||
futures.append(future)
|
||||
# 全局登录槽负责并发;批次只提交任务并收集结果。
|
||||
futures = []
|
||||
for item_index, item in enumerate(task_infos):
|
||||
if self._stop.is_set():
|
||||
self._push_log("warning", "任务已停止,跳过剩余账号")
|
||||
for pending_item in task_infos[item_index:]:
|
||||
self._mark_relogin_stopped(
|
||||
pending_item["task_id"], "任务已停止,旧 Cookie 已保留"
|
||||
)
|
||||
break
|
||||
future = submit(
|
||||
TaskLane.LOGIN,
|
||||
self._execute_one,
|
||||
item["task_id"],
|
||||
item["acc_info"],
|
||||
total,
|
||||
stopped=self._stop.is_set,
|
||||
)
|
||||
if future is None:
|
||||
break
|
||||
futures.append(future)
|
||||
|
||||
# 等待所有任务完成
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
|
||||
self._push_log("error", f"Worker 异常: {e}")
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
|
||||
self._push_log("error", f"Worker 异常: {e}")
|
||||
|
||||
self._push_log("info", f"批量{action_name}任务 {batch_id} 完成")
|
||||
self._push_log("result", "")
|
||||
@@ -902,5 +903,5 @@ def cleanup_orphan_login_tasks(
|
||||
return _converge_interrupted_attempts(db, message)
|
||||
|
||||
|
||||
# 在模块末尾导入 SessionLocal(避免循环导入)
|
||||
from ..database import SessionLocal
|
||||
# 在模块末尾导入 WorkerSessionLocal(避免循环导入)
|
||||
from ..database import WorkerSessionLocal
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Iterable
|
||||
@@ -12,7 +13,7 @@ from loguru import logger
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...database import SessionLocal
|
||||
from ...database import WorkerSessionLocal
|
||||
from ...models import PlatformFulfillmentTask
|
||||
from . import service
|
||||
|
||||
@@ -142,6 +143,21 @@ def run_auto_allocation_once(
|
||||
if allocated_task is not None:
|
||||
allocated_task.next_allocate_at = None
|
||||
db.commit()
|
||||
try:
|
||||
asyncio.run(
|
||||
service.start_binding(
|
||||
db,
|
||||
task_id=candidate_task_id,
|
||||
current=actor,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - allocation remains valid
|
||||
db.rollback()
|
||||
logger.warning(
|
||||
"直播账号履约自动开始绑定失败 task_id={}: {}",
|
||||
candidate_task_id,
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
failed += 1
|
||||
_schedule_retry(
|
||||
@@ -171,6 +187,48 @@ def run_auto_allocation_once(
|
||||
}
|
||||
|
||||
|
||||
def start_allocated_bindings_once(
|
||||
db: Session,
|
||||
*,
|
||||
limit: int = DEFAULT_BATCH_SIZE,
|
||||
) -> int:
|
||||
"""Start durable binding preparation for allocated, unstarted sessions."""
|
||||
rows = (
|
||||
db.query(PlatformFulfillmentTask)
|
||||
.filter(
|
||||
PlatformFulfillmentTask.execution_status == "allocated",
|
||||
PlatformFulfillmentTask.binding_status == "not_started",
|
||||
PlatformFulfillmentTask.account_id.isnot(None),
|
||||
PlatformFulfillmentTask.lease_id.isnot(None),
|
||||
)
|
||||
.order_by(PlatformFulfillmentTask.updated_at.asc(), PlatformFulfillmentTask.id.asc())
|
||||
.limit(max(1, limit))
|
||||
.all()
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
actor = service._integration_actor(db)
|
||||
started = 0
|
||||
for task in rows:
|
||||
try:
|
||||
asyncio.run(
|
||||
service.start_binding(
|
||||
db,
|
||||
task_id=int(task.id),
|
||||
current=actor,
|
||||
)
|
||||
)
|
||||
started += 1
|
||||
except Exception as exc: # noqa: BLE001 - one session cannot block the rest
|
||||
db.rollback()
|
||||
logger.warning(
|
||||
"直播账号履约补启动绑定失败 task_id={}: {}",
|
||||
task.id,
|
||||
exc,
|
||||
)
|
||||
return started
|
||||
|
||||
|
||||
def _run() -> None:
|
||||
interval = _env_int(
|
||||
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_SECONDS",
|
||||
@@ -179,7 +237,7 @@ def _run() -> None:
|
||||
3600,
|
||||
)
|
||||
while not _stop_event.is_set():
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
pending_ids = _take_pending_task_ids(
|
||||
_env_int(
|
||||
@@ -190,8 +248,11 @@ def _run() -> None:
|
||||
)
|
||||
)
|
||||
result = run_auto_allocation_once(db, task_ids=pending_ids or None)
|
||||
started = start_allocated_bindings_once(db)
|
||||
if result["checked"]:
|
||||
logger.info("直播账号履约自动选号: {}", result)
|
||||
if started:
|
||||
logger.info("直播账号履约补启动绑定: {} 条", started)
|
||||
except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出
|
||||
db.rollback()
|
||||
logger.warning("直播账号履约自动选号轮次失败: {}", exc)
|
||||
|
||||
@@ -29,6 +29,8 @@ def _state_dedupe_key(payload: dict[str, Any]) -> str:
|
||||
"platName": payload.get("platName"),
|
||||
"bindReadyForConfirm": payload.get("bindReadyForConfirm"),
|
||||
"bindConfirmed": payload.get("bindConfirmed"),
|
||||
"queueState": payload.get("queueState"),
|
||||
"queueLane": payload.get("queueLane"),
|
||||
"customerProfileReset": payload.get("customerProfileReset"),
|
||||
}
|
||||
digest = hashlib.sha256(
|
||||
|
||||
@@ -7,7 +7,7 @@ import threading
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from ...database import SessionLocal
|
||||
from ...database import WorkerSessionLocal
|
||||
|
||||
_stop_event = threading.Event()
|
||||
_thread: threading.Thread | None = None
|
||||
@@ -31,7 +31,7 @@ def _run() -> None:
|
||||
# 不抢占启动时的 Webhook;首轮延后一分钟。
|
||||
if _stop_event.wait(min(60, _interval_seconds())):
|
||||
break
|
||||
db = SessionLocal()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
queued = 0
|
||||
for _page in range(20):
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any
|
||||
from fastapi import HTTPException
|
||||
from loguru import logger
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...models import (
|
||||
@@ -763,6 +764,19 @@ def _load_task(db: Session, task_id: int) -> PlatformFulfillmentTask:
|
||||
return task
|
||||
|
||||
|
||||
def _load_task_for_update(db: Session, task_id: int) -> PlatformFulfillmentTask:
|
||||
task = (
|
||||
db.query(PlatformFulfillmentTask)
|
||||
.filter(PlatformFulfillmentTask.id == int(task_id))
|
||||
.populate_existing()
|
||||
.with_for_update()
|
||||
.first()
|
||||
)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail="直播账号履约任务不存在")
|
||||
return task
|
||||
|
||||
|
||||
def _retain_lease_for_active_collab(
|
||||
db: Session,
|
||||
*,
|
||||
@@ -1241,6 +1255,40 @@ def _binding_failure_issue(bind_task: DouyuTask, result: dict[str, Any]) -> str:
|
||||
return task_issue if messages.get(task_issue) else "binding_failed"
|
||||
|
||||
|
||||
def _customer_binding_queue_state(
|
||||
task: PlatformFulfillmentTask,
|
||||
bind_task: DouyuTask | None,
|
||||
*,
|
||||
lease_ready: bool,
|
||||
) -> tuple[str, int]:
|
||||
if task.execution_status == "completed":
|
||||
return "completed", 0
|
||||
if task.execution_status == "cancelled":
|
||||
return "cancelled", 0
|
||||
if task.issue_code == "mapping_missing":
|
||||
return "mapping_missing", 0
|
||||
if not task.account_id or not task.lease_id:
|
||||
return "allocation_queued", 2000
|
||||
if not lease_ready:
|
||||
return "lease_inactive", 0
|
||||
if bind_task is None:
|
||||
return "ready_to_start", 0
|
||||
result = bind_task.result or {}
|
||||
if bool(result.get("bind_confirmed")) or task.binding_status == "bound":
|
||||
return "confirmed", 0
|
||||
if pending_binding_role_from_result(result):
|
||||
return "role_ready", 0
|
||||
if bind_task.status in {"failed", "error", "stopped"}:
|
||||
return "failed", 0
|
||||
if task.binding_status == "confirming":
|
||||
return "confirming", 1000
|
||||
if bind_task.status in {"planned", "pending"}:
|
||||
return "qr_queued", 1000
|
||||
if result.get("url"):
|
||||
return "waiting_scan", 5000 if result.get("bind_polling") else 0
|
||||
return "preparing", 1000
|
||||
|
||||
|
||||
def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]:
|
||||
"""面向前端客户页的安全绑定快照,不包含账号、CK、积分和租约。"""
|
||||
task = _load_task(db, task_id)
|
||||
@@ -1259,6 +1307,11 @@ def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]:
|
||||
lease_ready = bool(task.account_id and lease and lease.status == "active")
|
||||
binding_failed = task.binding_status == "failed"
|
||||
binding_confirmed = task.binding_status == "bound"
|
||||
queue_state, retry_after_ms = _customer_binding_queue_state(
|
||||
task,
|
||||
bind_task,
|
||||
lease_ready=lease_ready,
|
||||
)
|
||||
return {
|
||||
"sessionId": task.id,
|
||||
"platformOrderId": task.platform_order_id,
|
||||
@@ -1282,6 +1335,10 @@ def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]:
|
||||
"platName": str(bind_result.get("plat_name") or ""),
|
||||
"bindReadyForConfirm": (bool(pending_role) and not binding_failed),
|
||||
"bindConfirmed": bool(bind_result.get("bind_confirmed")) and binding_confirmed,
|
||||
"queueState": queue_state,
|
||||
"retryAfterMs": retry_after_ms,
|
||||
"queueLane": bind_task.queue_lane if bind_task else "",
|
||||
"accepted": False,
|
||||
"stateVersion": int(task.state_version or 0),
|
||||
"completedAt": _jsonable(task.completed_at),
|
||||
"updatedAt": _jsonable(task.updated_at),
|
||||
@@ -1318,8 +1375,9 @@ def ensure_customer_session(
|
||||
if task.execution_status in {"pending_mapping", "ready", "manual_required"}:
|
||||
if task.issue_code == "mapping_missing":
|
||||
return customer_binding_out(db, task.id)
|
||||
actor = _integration_actor(db)
|
||||
allocate_task(db, task_id=task.id, current=actor)
|
||||
from .auto_allocator import wake_auto_allocator
|
||||
|
||||
wake_auto_allocator([int(task.id)])
|
||||
return customer_binding_out(db, task.id)
|
||||
|
||||
|
||||
@@ -1330,18 +1388,40 @@ async def customer_start_binding(
|
||||
force_refresh: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
task = _load_task(db, task_id)
|
||||
if task.execution_status in {"completed", "cancelled"}:
|
||||
return customer_binding_out(db, task.id)
|
||||
if task.issue_code == "mapping_missing":
|
||||
return customer_binding_out(db, task.id)
|
||||
|
||||
if not task.account_id or not task.lease_id:
|
||||
from .auto_allocator import wake_auto_allocator
|
||||
|
||||
wake_auto_allocator([int(task.id)])
|
||||
payload = customer_binding_out(db, task.id)
|
||||
payload["accepted"] = True
|
||||
payload["queueState"] = "allocation_queued"
|
||||
payload["retryAfterMs"] = 2000
|
||||
return payload
|
||||
|
||||
lease = db.query(AccountLease).filter(AccountLease.id == task.lease_id).first()
|
||||
if lease is None or lease.status != "active":
|
||||
return customer_binding_out(db, task.id)
|
||||
|
||||
previous_bind_task_id = task.bind_task_id
|
||||
actor = _integration_actor(db)
|
||||
if task.execution_status not in {"completed", "cancelled"} and (
|
||||
not task.account_id or not task.lease_id
|
||||
):
|
||||
allocate_task(db, task_id=task.id, current=actor)
|
||||
await start_binding(
|
||||
db,
|
||||
task_id=task.id,
|
||||
current=actor,
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
return customer_binding_out(db, task.id)
|
||||
payload = customer_binding_out(db, task.id)
|
||||
accepted = task.bind_task_id != previous_bind_task_id
|
||||
payload["accepted"] = accepted
|
||||
if accepted:
|
||||
payload["queueState"] = "qr_queued"
|
||||
payload["retryAfterMs"] = 1000
|
||||
return payload
|
||||
|
||||
|
||||
def customer_refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
|
||||
@@ -1881,7 +1961,7 @@ async def start_binding(
|
||||
current: User,
|
||||
force_refresh: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
task = _load_task(db, task_id)
|
||||
task = _load_task_for_update(db, task_id)
|
||||
if not task.account_id or not task.lease_id:
|
||||
raise HTTPException(status_code=409, detail="请先分配账号")
|
||||
lease = db.query(AccountLease).filter(AccountLease.id == task.lease_id).first()
|
||||
@@ -1924,9 +2004,16 @@ async def start_binding(
|
||||
message="等待生成绑定二维码",
|
||||
result={"payload": {}},
|
||||
created_by=current.id,
|
||||
platform_task_id=task.id,
|
||||
)
|
||||
db.add(bind_task)
|
||||
db.flush()
|
||||
try:
|
||||
db.flush()
|
||||
except IntegrityError:
|
||||
# The database uniqueness gate wins a race that started before either
|
||||
# request could acquire the fulfillment row lock.
|
||||
db.rollback()
|
||||
return get_task_detail(db, task_id)
|
||||
account = db.query(Account).filter(Account.id == task.account_id).first()
|
||||
if account is not None:
|
||||
record_douyu_task_started(
|
||||
@@ -2113,7 +2200,7 @@ def reconcile_binding_state_outbox(
|
||||
async def confirm_binding(
|
||||
db: Session, *, task_id: int, current: User
|
||||
) -> dict[str, Any]:
|
||||
task = _load_task(db, task_id)
|
||||
task = _load_task_for_update(db, task_id)
|
||||
task = _load_task(db, refresh_binding(db, task_id=task.id)["task"]["id"])
|
||||
if task.binding_status != "role_ready":
|
||||
raise HTTPException(status_code=409, detail="当前没有待确认的游戏角色")
|
||||
@@ -2146,6 +2233,7 @@ async def confirm_binding(
|
||||
"baseline_role_name": baseline_role_name,
|
||||
},
|
||||
created_by=current.id,
|
||||
platform_task_id=task.id,
|
||||
)
|
||||
db.add(confirm_task)
|
||||
db.flush()
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Bounded, process-local execution lanes for background work.
|
||||
|
||||
Each lane owns its workers and admission budget. A slow or saturated domain can
|
||||
therefore no longer consume every worker slot in the process. This runtime is
|
||||
intended for the current single-Uvicorn deployment; durable scheduling remains
|
||||
in the database.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
|
||||
class TaskLane(StrEnum):
|
||||
INTERACTIVE_BINDING = "interactive_binding"
|
||||
BINDING_POLL = "binding_poll"
|
||||
LOGIN = "login"
|
||||
ACCOUNT_CHECK = "account_check"
|
||||
AUTOMATION = "automation"
|
||||
QUERY_BATCH = "query_batch"
|
||||
COLLAB_RECOMMEND = "collab_recommend"
|
||||
PAYMENT_WAIT = "payment_wait"
|
||||
OPS_WRITE = "ops_write"
|
||||
HUYA_BATCH = "huya_batch"
|
||||
HUYA_REGISTER = "huya_register"
|
||||
INTEGRATION = "integration"
|
||||
MAINTENANCE = "maintenance"
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, minimum: int = 1) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LaneConfig:
|
||||
workers: int
|
||||
max_in_flight: int
|
||||
|
||||
|
||||
_DEFAULT_WORKERS: dict[TaskLane, int] = {
|
||||
TaskLane.INTERACTIVE_BINDING: 3,
|
||||
TaskLane.BINDING_POLL: 2,
|
||||
TaskLane.LOGIN: 4,
|
||||
TaskLane.ACCOUNT_CHECK: 4,
|
||||
TaskLane.AUTOMATION: 2,
|
||||
TaskLane.QUERY_BATCH: 3,
|
||||
TaskLane.COLLAB_RECOMMEND: 2,
|
||||
TaskLane.PAYMENT_WAIT: 2,
|
||||
TaskLane.OPS_WRITE: 2,
|
||||
TaskLane.HUYA_BATCH: 2,
|
||||
TaskLane.HUYA_REGISTER: 2,
|
||||
TaskLane.INTEGRATION: 2,
|
||||
TaskLane.MAINTENANCE: 1,
|
||||
}
|
||||
|
||||
|
||||
def _lane_config(lane: TaskLane) -> LaneConfig:
|
||||
key = lane.value.upper()
|
||||
workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane])
|
||||
max_in_flight = _env_int(
|
||||
f"TASK_LANE_{key}_MAX_IN_FLIGHT",
|
||||
max(workers * 32, workers),
|
||||
)
|
||||
return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers))
|
||||
|
||||
|
||||
@dataclass
|
||||
class _LaneRuntime:
|
||||
config: LaneConfig
|
||||
executor: ThreadPoolExecutor
|
||||
admission: threading.BoundedSemaphore
|
||||
|
||||
|
||||
_LOCK = threading.RLock()
|
||||
_RUNTIMES: dict[TaskLane, _LaneRuntime] = {}
|
||||
|
||||
|
||||
def _runtime(lane: TaskLane) -> _LaneRuntime:
|
||||
with _LOCK:
|
||||
runtime = _RUNTIMES.get(lane)
|
||||
if runtime is not None:
|
||||
return runtime
|
||||
config = _lane_config(lane)
|
||||
runtime = _LaneRuntime(
|
||||
config=config,
|
||||
executor=ThreadPoolExecutor(
|
||||
max_workers=config.workers,
|
||||
thread_name_prefix=f"lane-{lane.value}",
|
||||
),
|
||||
admission=threading.BoundedSemaphore(config.max_in_flight),
|
||||
)
|
||||
_RUNTIMES[lane] = runtime
|
||||
return runtime
|
||||
|
||||
|
||||
def submit[T](
|
||||
lane: TaskLane,
|
||||
fn: Callable[..., T],
|
||||
/,
|
||||
*args: Any,
|
||||
stopped: Callable[[], bool] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Future[T] | None:
|
||||
"""Submit work to a lane, waiting for admission when the lane is full.
|
||||
|
||||
Batch coordinators are cheap threads and are allowed to block here. Worker
|
||||
threads themselves must not recursively submit to the same lane.
|
||||
"""
|
||||
runtime = _runtime(lane)
|
||||
while True:
|
||||
if stopped is not None and stopped():
|
||||
return None
|
||||
if runtime.admission.acquire(timeout=0.2):
|
||||
break
|
||||
|
||||
def run() -> T:
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
runtime.admission.release()
|
||||
|
||||
return runtime.executor.submit(run)
|
||||
|
||||
|
||||
def lane_stats() -> dict[str, dict[str, int]]:
|
||||
with _LOCK:
|
||||
runtimes = list(_RUNTIMES.items())
|
||||
stats: dict[str, dict[str, int]] = {}
|
||||
for lane, runtime in runtimes:
|
||||
stats[lane.value] = {
|
||||
"workers": runtime.config.workers,
|
||||
"max_in_flight": runtime.config.max_in_flight,
|
||||
"admission_value": runtime.admission._value,
|
||||
}
|
||||
return stats
|
||||
|
||||
|
||||
def shutdown(wait: bool = False) -> None:
|
||||
with _LOCK:
|
||||
runtimes = list(_RUNTIMES.values())
|
||||
_RUNTIMES.clear()
|
||||
for runtime in runtimes:
|
||||
runtime.executor.shutdown(wait=wait, cancel_futures=False)
|
||||
|
||||
|
||||
def douyu_lane_for_task(task_type: str) -> TaskLane:
|
||||
if task_type in {
|
||||
"get_bind_qr",
|
||||
"confirm_bind",
|
||||
"prepare_esports_bind",
|
||||
"get_esports_bind_qr",
|
||||
"confirm_esports_bind",
|
||||
"get_xpd_bind_qr",
|
||||
"query_xpd_bind_info",
|
||||
"confirm_xpd_bind",
|
||||
"query_xpd_role",
|
||||
}:
|
||||
return TaskLane.INTERACTIVE_BINDING
|
||||
if task_type in {
|
||||
"create_elite_qr",
|
||||
"create_esports_qr",
|
||||
"create_gold_qr",
|
||||
}:
|
||||
return TaskLane.PAYMENT_WAIT
|
||||
if task_type in {
|
||||
"lock_goods",
|
||||
"pay_locked_order",
|
||||
"exchange_goods",
|
||||
"exchange_esports_goods",
|
||||
"exchange_xpd_goods",
|
||||
"donate_elite_gift",
|
||||
"donate_esports_chicken_gift",
|
||||
"donate_esports_firework_gift",
|
||||
}:
|
||||
return TaskLane.OPS_WRITE
|
||||
return TaskLane.QUERY_BATCH
|
||||
|
||||
|
||||
__all__ = [
|
||||
"TaskLane",
|
||||
"douyu_lane_for_task",
|
||||
"lane_stats",
|
||||
"shutdown",
|
||||
"submit",
|
||||
]
|
||||
Reference in New Issue
Block a user