perf(douyu): 拆分任务槽并改造绑定扫码调度
This commit is contained in:
@@ -2,21 +2,20 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import time
|
||||
from concurrent.futures import as_completed
|
||||
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from core.douyu import DouyuActivityError
|
||||
|
||||
from ..database import SessionLocal
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .cookie_check_service import check_douyu_cookie
|
||||
from .douyu_account_probe import persist_probe_result, probe_session
|
||||
from .douyu_concurrency import (
|
||||
acquire_overview_slot,
|
||||
acquire_running_task_slot,
|
||||
release_overview_slot,
|
||||
release_running_task_slot,
|
||||
)
|
||||
from .douyu_runner_bind import BindMixin
|
||||
from .douyu_runner_core import ( # noqa: F401 (douyu_batch_registry 供 routers 重导出)
|
||||
@@ -30,6 +29,11 @@ from .douyu_runner_manual import ManualMixin
|
||||
from .douyu_runner_overview import OverviewMixin
|
||||
from .douyu_runner_xpd import XpdMixin
|
||||
from .douyu_service import update_account_profile_from_cookie
|
||||
from .task_runtime import douyu_lane_for_task, submit
|
||||
|
||||
# Compatibility alias for tests and integrations that replace the worker
|
||||
# session factory at the runner boundary.
|
||||
SessionLocal = WorkerSessionLocal
|
||||
|
||||
|
||||
class DouyuBatchRunner(
|
||||
@@ -47,19 +51,13 @@ class DouyuBatchRunner(
|
||||
def _execute_one(self, task_id: int, config: dict, total: int):
|
||||
overview_task = self.task_type == "query_account_overview"
|
||||
overview_slot = False
|
||||
running_slot = False
|
||||
if overview_task:
|
||||
overview_slot = acquire_overview_slot(self._stop.is_set)
|
||||
if not overview_slot:
|
||||
return
|
||||
try:
|
||||
running_slot = acquire_running_task_slot(self._stop.is_set)
|
||||
if not running_slot:
|
||||
return
|
||||
self._execute_one_with_slot(task_id, config, total)
|
||||
finally:
|
||||
if running_slot:
|
||||
release_running_task_slot()
|
||||
if overview_slot:
|
||||
release_overview_slot()
|
||||
|
||||
@@ -221,19 +219,52 @@ class DouyuBatchRunner(
|
||||
self._push_task_event(task)
|
||||
|
||||
total = len(tasks)
|
||||
with ThreadPoolExecutor(max_workers=self.concurrency) as executor:
|
||||
futures = []
|
||||
for task in tasks:
|
||||
if self._stop.is_set():
|
||||
break
|
||||
futures.append(
|
||||
executor.submit(self._execute_one, task.id, config, total)
|
||||
lane = douyu_lane_for_task(self.task_type)
|
||||
pending: dict = {}
|
||||
next_index = 0
|
||||
while next_index < total or pending:
|
||||
while (
|
||||
not self._stop.is_set()
|
||||
and next_index < total
|
||||
and len(pending) < self.concurrency
|
||||
):
|
||||
task = tasks[next_index]
|
||||
future = submit(
|
||||
lane,
|
||||
self._execute_one,
|
||||
task.id,
|
||||
config,
|
||||
total,
|
||||
stopped=self._stop.is_set,
|
||||
)
|
||||
for future in as_completed(futures):
|
||||
if future is None:
|
||||
break
|
||||
pending[future] = (int(task.id), time.monotonic())
|
||||
next_index += 1
|
||||
|
||||
if not pending:
|
||||
break
|
||||
|
||||
try:
|
||||
completed = list(as_completed(pending, timeout=0.5))
|
||||
except TimeoutError:
|
||||
continue
|
||||
for future in completed:
|
||||
task_id, submitted_at = pending.pop(future)
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
|
||||
self._push_log("error", f"Worker 异常: {exc}")
|
||||
self._push_log(
|
||||
"error",
|
||||
f"Worker 异常 task_id={task_id}: {exc}",
|
||||
)
|
||||
else:
|
||||
waited = time.monotonic() - submitted_at
|
||||
if waited >= 5:
|
||||
self._push_log(
|
||||
"info",
|
||||
f"任务 task_id={task_id} 在 {lane.value} 槽排队 {waited:.1f}s",
|
||||
)
|
||||
|
||||
if self._stop.is_set():
|
||||
self._push_log("warning", f"斗鱼任务批次 {self.batch_id} 已停止")
|
||||
|
||||
Reference in New Issue
Block a user