perf(douyu): 消除任务锁竞争并增加创建背压

This commit is contained in:
yml
2026-11-19 22:01:21 +08:00
parent 62cb0a41ec
commit 09cdf3f2eb
11 changed files with 555 additions and 95 deletions
+21 -3
View File
@@ -6,6 +6,7 @@
from __future__ import annotations
import asyncio
import os
import threading
from web.backend.database import SessionLocal
@@ -13,16 +14,33 @@ from web.backend.database import SessionLocal
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
def _env_positive_int(name: str, default: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return value if value > 0 else default
DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
"DOUYU_BATCH_CREATE_CONCURRENCY",
2,
)
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
def start_douyu_batch(
*,
batch_id: str,
task_type: str,
payload: dict,
concurrency: int,
loop: asyncio.AbstractEventLoop | None = None,
log_queue: asyncio.Queue | None = None,
) -> None:
"""必须在运行中的 asyncio 事件循环里调用。"""
log_queue = asyncio.Queue()
loop = asyncio.get_running_loop()
"""创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。"""
loop = loop or asyncio.get_running_loop()
log_queue = log_queue or asyncio.Queue()
thread_db = SessionLocal()
runner = DouyuBatchRunner(
db=thread_db,