perf(douyu): 拆分任务槽并改造绑定扫码调度

This commit is contained in:
yml
2026-11-20 22:03:20 +08:00
parent 8869e427d2
commit 071d45ad1d
40 changed files with 1635 additions and 245 deletions
+50 -19
View File
@@ -2,21 +2,20 @@
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor, as_completed
import time
from concurrent.futures import as_completed
from sqlalchemy.orm import joinedload
from core.douyu import DouyuActivityError
from ..database import SessionLocal
from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .cookie_check_service import check_douyu_cookie
from .douyu_account_probe import persist_probe_result, probe_session
from .douyu_concurrency import (
acquire_overview_slot,
acquire_running_task_slot,
release_overview_slot,
release_running_task_slot,
)
from .douyu_runner_bind import BindMixin
from .douyu_runner_core import ( # noqa: F401 (douyu_batch_registry 供 routers 重导出)
@@ -30,6 +29,11 @@ from .douyu_runner_manual import ManualMixin
from .douyu_runner_overview import OverviewMixin
from .douyu_runner_xpd import XpdMixin
from .douyu_service import update_account_profile_from_cookie
from .task_runtime import douyu_lane_for_task, submit
# Compatibility alias for tests and integrations that replace the worker
# session factory at the runner boundary.
SessionLocal = WorkerSessionLocal
class DouyuBatchRunner(
@@ -47,19 +51,13 @@ class DouyuBatchRunner(
def _execute_one(self, task_id: int, config: dict, total: int):
overview_task = self.task_type == "query_account_overview"
overview_slot = False
running_slot = False
if overview_task:
overview_slot = acquire_overview_slot(self._stop.is_set)
if not overview_slot:
return
try:
running_slot = acquire_running_task_slot(self._stop.is_set)
if not running_slot:
return
self._execute_one_with_slot(task_id, config, total)
finally:
if running_slot:
release_running_task_slot()
if overview_slot:
release_overview_slot()
@@ -221,19 +219,52 @@ class DouyuBatchRunner(
self._push_task_event(task)
total = len(tasks)
with ThreadPoolExecutor(max_workers=self.concurrency) as executor:
futures = []
for task in tasks:
if self._stop.is_set():
break
futures.append(
executor.submit(self._execute_one, task.id, config, total)
lane = douyu_lane_for_task(self.task_type)
pending: dict = {}
next_index = 0
while next_index < total or pending:
while (
not self._stop.is_set()
and next_index < total
and len(pending) < self.concurrency
):
task = tasks[next_index]
future = submit(
lane,
self._execute_one,
task.id,
config,
total,
stopped=self._stop.is_set,
)
for future in as_completed(futures):
if future is None:
break
pending[future] = (int(task.id), time.monotonic())
next_index += 1
if not pending:
break
try:
completed = list(as_completed(pending, timeout=0.5))
except TimeoutError:
continue
for future in completed:
task_id, submitted_at = pending.pop(future)
try:
future.result()
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
self._push_log("error", f"Worker 异常: {exc}")
self._push_log(
"error",
f"Worker 异常 task_id={task_id}: {exc}",
)
else:
waited = time.monotonic() - submitted_at
if waited >= 5:
self._push_log(
"info",
f"任务 task_id={task_id} 在 {lane.value} 槽排队 {waited:.1f}s",
)
if self._stop.is_set():
self._push_log("warning", f"斗鱼任务批次 {self.batch_id} 已停止")