perf(douyu): 消除任务锁竞争并增加创建背压
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
|
||||
from web.backend.database import SessionLocal
|
||||
@@ -13,16 +14,33 @@ from web.backend.database import SessionLocal
|
||||
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||
|
||||
|
||||
def _env_positive_int(name: str, default: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
|
||||
"DOUYU_BATCH_CREATE_CONCURRENCY",
|
||||
2,
|
||||
)
|
||||
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
|
||||
|
||||
|
||||
def start_douyu_batch(
|
||||
*,
|
||||
batch_id: str,
|
||||
task_type: str,
|
||||
payload: dict,
|
||||
concurrency: int,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
log_queue: asyncio.Queue | None = None,
|
||||
) -> None:
|
||||
"""必须在运行中的 asyncio 事件循环里调用。"""
|
||||
log_queue = asyncio.Queue()
|
||||
loop = asyncio.get_running_loop()
|
||||
"""创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。"""
|
||||
loop = loop or asyncio.get_running_loop()
|
||||
log_queue = log_queue or asyncio.Queue()
|
||||
thread_db = SessionLocal()
|
||||
runner = DouyuBatchRunner(
|
||||
db=thread_db,
|
||||
|
||||
Reference in New Issue
Block a user