perf(douyu): 拆分任务槽并改造绑定扫码调度

This commit is contained in:
yml
2026-11-20 22:03:20 +08:00
parent 8869e427d2
commit 071d45ad1d
40 changed files with 1635 additions and 245 deletions
+195
View File
@@ -0,0 +1,195 @@
"""Bounded, process-local execution lanes for background work.
Each lane owns its workers and admission budget. A slow or saturated domain can
therefore no longer consume every worker slot in the process. This runtime is
intended for the current single-Uvicorn deployment; durable scheduling remains
in the database.
"""
from __future__ import annotations
import os
import threading
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from enum import StrEnum
from typing import Any
class TaskLane(StrEnum):
INTERACTIVE_BINDING = "interactive_binding"
BINDING_POLL = "binding_poll"
LOGIN = "login"
ACCOUNT_CHECK = "account_check"
AUTOMATION = "automation"
QUERY_BATCH = "query_batch"
COLLAB_RECOMMEND = "collab_recommend"
PAYMENT_WAIT = "payment_wait"
OPS_WRITE = "ops_write"
HUYA_BATCH = "huya_batch"
HUYA_REGISTER = "huya_register"
INTEGRATION = "integration"
MAINTENANCE = "maintenance"
def _env_int(name: str, default: int, minimum: int = 1) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, value)
@dataclass(frozen=True)
class LaneConfig:
workers: int
max_in_flight: int
_DEFAULT_WORKERS: dict[TaskLane, int] = {
TaskLane.INTERACTIVE_BINDING: 3,
TaskLane.BINDING_POLL: 2,
TaskLane.LOGIN: 4,
TaskLane.ACCOUNT_CHECK: 4,
TaskLane.AUTOMATION: 2,
TaskLane.QUERY_BATCH: 3,
TaskLane.COLLAB_RECOMMEND: 2,
TaskLane.PAYMENT_WAIT: 2,
TaskLane.OPS_WRITE: 2,
TaskLane.HUYA_BATCH: 2,
TaskLane.HUYA_REGISTER: 2,
TaskLane.INTEGRATION: 2,
TaskLane.MAINTENANCE: 1,
}
def _lane_config(lane: TaskLane) -> LaneConfig:
key = lane.value.upper()
workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane])
max_in_flight = _env_int(
f"TASK_LANE_{key}_MAX_IN_FLIGHT",
max(workers * 32, workers),
)
return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers))
@dataclass
class _LaneRuntime:
config: LaneConfig
executor: ThreadPoolExecutor
admission: threading.BoundedSemaphore
_LOCK = threading.RLock()
_RUNTIMES: dict[TaskLane, _LaneRuntime] = {}
def _runtime(lane: TaskLane) -> _LaneRuntime:
with _LOCK:
runtime = _RUNTIMES.get(lane)
if runtime is not None:
return runtime
config = _lane_config(lane)
runtime = _LaneRuntime(
config=config,
executor=ThreadPoolExecutor(
max_workers=config.workers,
thread_name_prefix=f"lane-{lane.value}",
),
admission=threading.BoundedSemaphore(config.max_in_flight),
)
_RUNTIMES[lane] = runtime
return runtime
def submit[T](
lane: TaskLane,
fn: Callable[..., T],
/,
*args: Any,
stopped: Callable[[], bool] | None = None,
**kwargs: Any,
) -> Future[T] | None:
"""Submit work to a lane, waiting for admission when the lane is full.
Batch coordinators are cheap threads and are allowed to block here. Worker
threads themselves must not recursively submit to the same lane.
"""
runtime = _runtime(lane)
while True:
if stopped is not None and stopped():
return None
if runtime.admission.acquire(timeout=0.2):
break
def run() -> T:
try:
return fn(*args, **kwargs)
finally:
runtime.admission.release()
return runtime.executor.submit(run)
def lane_stats() -> dict[str, dict[str, int]]:
with _LOCK:
runtimes = list(_RUNTIMES.items())
stats: dict[str, dict[str, int]] = {}
for lane, runtime in runtimes:
stats[lane.value] = {
"workers": runtime.config.workers,
"max_in_flight": runtime.config.max_in_flight,
"admission_value": runtime.admission._value,
}
return stats
def shutdown(wait: bool = False) -> None:
with _LOCK:
runtimes = list(_RUNTIMES.values())
_RUNTIMES.clear()
for runtime in runtimes:
runtime.executor.shutdown(wait=wait, cancel_futures=False)
def douyu_lane_for_task(task_type: str) -> TaskLane:
if task_type in {
"get_bind_qr",
"confirm_bind",
"prepare_esports_bind",
"get_esports_bind_qr",
"confirm_esports_bind",
"get_xpd_bind_qr",
"query_xpd_bind_info",
"confirm_xpd_bind",
"query_xpd_role",
}:
return TaskLane.INTERACTIVE_BINDING
if task_type in {
"create_elite_qr",
"create_esports_qr",
"create_gold_qr",
}:
return TaskLane.PAYMENT_WAIT
if task_type in {
"lock_goods",
"pay_locked_order",
"exchange_goods",
"exchange_esports_goods",
"exchange_xpd_goods",
"donate_elite_gift",
"donate_esports_chicken_gift",
"donate_esports_firework_gift",
}:
return TaskLane.OPS_WRITE
return TaskLane.QUERY_BATCH
__all__ = [
"TaskLane",
"douyu_lane_for_task",
"lane_stats",
"shutdown",
"submit",
]