Files
live-hub-py/web/backend/services/douyu_concurrency.py
T

78 lines
2.0 KiB
Python

"""斗鱼后台任务的进程级并发限制。
任务线程和数据库连接池共享同一个 Uvicorn 进程。这里把「每批并发」和
「全进程同时执行数」分开限制,避免多个批次叠加后把请求线程和连接池挤空。
"""
from __future__ import annotations
import os
import threading
from collections.abc import Callable
def _env_positive_int(name: str, default: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return value if value > 0 else default
DOUYU_BATCH_TASK_CONCURRENCY = _env_positive_int(
"DOUYU_BATCH_TASK_CONCURRENCY",
5,
)
DOUYU_MAX_RUNNING_TASKS = _env_positive_int(
"DOUYU_MAX_RUNNING_TASKS",
12,
)
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS = _env_positive_int(
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
2,
)
_task_slots = threading.BoundedSemaphore(DOUYU_MAX_RUNNING_TASKS)
_overview_slots = threading.BoundedSemaphore(DOUYU_MAX_ACTIVE_OVERVIEW_TASKS)
def clamp_batch_task_concurrency(value: int) -> int:
return max(1, min(int(value), DOUYU_BATCH_TASK_CONCURRENCY))
def _acquire(slots: threading.BoundedSemaphore, stopped: Callable[[], bool]) -> bool:
while not stopped():
if slots.acquire(timeout=0.2):
return True
return False
def acquire_running_task_slot(stopped: Callable[[], bool]) -> bool:
"""等待一个全进程任务槽;停止时返回 False。"""
return _acquire(_task_slots, stopped)
def release_running_task_slot() -> None:
_task_slots.release()
def acquire_overview_slot(stopped: Callable[[], bool]) -> bool:
"""等待一个一键查询槽;该限制跨所有批次共享。"""
return _acquire(_overview_slots, stopped)
def release_overview_slot() -> None:
_overview_slots.release()
__all__ = [
"DOUYU_BATCH_TASK_CONCURRENCY",
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
"DOUYU_MAX_RUNNING_TASKS",
"acquire_overview_slot",
"acquire_running_task_slot",
"clamp_batch_task_concurrency",
"release_overview_slot",
"release_running_task_slot",
]