78 lines
2.0 KiB
Python
78 lines
2.0 KiB
Python
"""斗鱼后台任务的进程级并发限制。
|
|
|
|
任务线程和数据库连接池共享同一个 Uvicorn 进程。这里把「每批并发」和
|
|
「全进程同时执行数」分开限制,避免多个批次叠加后把请求线程和连接池挤空。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import threading
|
|
from collections.abc import Callable
|
|
|
|
|
|
def _env_positive_int(name: str, default: int) -> int:
|
|
try:
|
|
value = int(os.getenv(name, str(default)))
|
|
except ValueError:
|
|
return default
|
|
return value if value > 0 else default
|
|
|
|
|
|
DOUYU_BATCH_TASK_CONCURRENCY = _env_positive_int(
|
|
"DOUYU_BATCH_TASK_CONCURRENCY",
|
|
5,
|
|
)
|
|
DOUYU_MAX_RUNNING_TASKS = _env_positive_int(
|
|
"DOUYU_MAX_RUNNING_TASKS",
|
|
12,
|
|
)
|
|
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS = _env_positive_int(
|
|
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
|
|
2,
|
|
)
|
|
|
|
_task_slots = threading.BoundedSemaphore(DOUYU_MAX_RUNNING_TASKS)
|
|
_overview_slots = threading.BoundedSemaphore(DOUYU_MAX_ACTIVE_OVERVIEW_TASKS)
|
|
|
|
|
|
def clamp_batch_task_concurrency(value: int) -> int:
|
|
return max(1, min(int(value), DOUYU_BATCH_TASK_CONCURRENCY))
|
|
|
|
|
|
def _acquire(slots: threading.BoundedSemaphore, stopped: Callable[[], bool]) -> bool:
|
|
while not stopped():
|
|
if slots.acquire(timeout=0.2):
|
|
return True
|
|
return False
|
|
|
|
|
|
def acquire_running_task_slot(stopped: Callable[[], bool]) -> bool:
|
|
"""等待一个全进程任务槽;停止时返回 False。"""
|
|
return _acquire(_task_slots, stopped)
|
|
|
|
|
|
def release_running_task_slot() -> None:
|
|
_task_slots.release()
|
|
|
|
|
|
def acquire_overview_slot(stopped: Callable[[], bool]) -> bool:
|
|
"""等待一个一键查询槽;该限制跨所有批次共享。"""
|
|
return _acquire(_overview_slots, stopped)
|
|
|
|
|
|
def release_overview_slot() -> None:
|
|
_overview_slots.release()
|
|
|
|
|
|
__all__ = [
|
|
"DOUYU_BATCH_TASK_CONCURRENCY",
|
|
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
|
|
"DOUYU_MAX_RUNNING_TASKS",
|
|
"acquire_overview_slot",
|
|
"acquire_running_task_slot",
|
|
"clamp_batch_task_concurrency",
|
|
"release_overview_slot",
|
|
"release_running_task_slot",
|
|
]
|