perf(douyu): 限制任务并发并释放长事务连接
This commit is contained in:
@@ -11,6 +11,7 @@ import threading
|
||||
|
||||
from web.backend.database import SessionLocal
|
||||
|
||||
from .douyu_concurrency import clamp_batch_task_concurrency
|
||||
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||
|
||||
|
||||
@@ -26,7 +27,7 @@ DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
|
||||
"DOUYU_BATCH_CREATE_CONCURRENCY",
|
||||
2,
|
||||
)
|
||||
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
|
||||
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 10)
|
||||
|
||||
|
||||
def start_douyu_batch(
|
||||
@@ -49,7 +50,7 @@ def start_douyu_batch(
|
||||
payload=payload,
|
||||
log_queue=log_queue,
|
||||
loop=loop,
|
||||
concurrency=max(1, min(int(concurrency), 10)),
|
||||
concurrency=clamp_batch_task_concurrency(concurrency),
|
||||
)
|
||||
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
|
||||
threading.Thread(target=runner.run, daemon=True).start()
|
||||
|
||||
@@ -483,13 +483,21 @@ def lookup_account(
|
||||
"message": "账号不存在,请先到「账号管理」导入并登录",
|
||||
"account_username": _text(username, max_length=128),
|
||||
}
|
||||
manual_id = str(config.manual_id or "")
|
||||
rid = str(config.rid or "")
|
||||
act_alias = str(config.bind_act_alias or "")
|
||||
confirm_act_alias = str(config.confirm_act_alias or "")
|
||||
legacy_act_alias = str(config.legacy_act_alias or "")
|
||||
# 脱离 ORM 后提交,网络期间不持有连接,也不会因 commit 过期而重新查库。
|
||||
db.expunge(account)
|
||||
db.commit()
|
||||
probe = _probe_with_client(
|
||||
account,
|
||||
manual_id=str(config.manual_id or ""),
|
||||
rid=str(config.rid or ""),
|
||||
act_alias=str(config.bind_act_alias or ""),
|
||||
confirm_act_alias=str(config.confirm_act_alias or ""),
|
||||
legacy_act_alias=str(config.legacy_act_alias or ""),
|
||||
manual_id=manual_id,
|
||||
rid=rid,
|
||||
act_alias=act_alias,
|
||||
confirm_act_alias=confirm_act_alias,
|
||||
legacy_act_alias=legacy_act_alias,
|
||||
client=client,
|
||||
with_goods=with_goods,
|
||||
)
|
||||
@@ -1540,6 +1548,15 @@ def create_order(
|
||||
if exists is not None:
|
||||
raise CollabError("订单号已存在", status_code=409, code="order_no_exists")
|
||||
|
||||
owner_user_id = int(owner_id) if owner_id is not None else int(current.id)
|
||||
owner = db.get(User, owner_user_id)
|
||||
if owner is None or not owner.is_active:
|
||||
raise CollabError(
|
||||
"负责人不存在或已停用",
|
||||
status_code=400,
|
||||
code="owner_invalid",
|
||||
)
|
||||
|
||||
images = normalize_images(home_images)
|
||||
explicit_points = _to_int(required_points)
|
||||
name, cid, points = _resolve_gift(
|
||||
@@ -1551,7 +1568,7 @@ def create_order(
|
||||
order = DouyuExternalOrder(
|
||||
order_no=order_text,
|
||||
shop=_text(shop, max_length=64),
|
||||
owner_id=int(owner_id) if owner_id is not None else int(current.id),
|
||||
owner_id=owner_user_id,
|
||||
account_username=_text(account_username, max_length=128),
|
||||
gift_name=name,
|
||||
commodity_id=cid,
|
||||
@@ -1576,13 +1593,22 @@ def create_order(
|
||||
)
|
||||
if account is not None and not draft:
|
||||
config = resolve_config(db)
|
||||
manual_id = str(config.manual_id or "")
|
||||
rid = str(config.rid or "")
|
||||
act_alias = str(config.bind_act_alias or "")
|
||||
confirm_act_alias = str(config.confirm_act_alias or "")
|
||||
legacy_act_alias = str(config.legacy_act_alias or "")
|
||||
# order 还没有加入 Session。账号脱离 ORM 后提交,探针期间不占连接。
|
||||
db.expunge(account)
|
||||
db.commit()
|
||||
db.add(account)
|
||||
probe = _probe_with_client(
|
||||
account,
|
||||
manual_id=str(config.manual_id or ""),
|
||||
rid=str(config.rid or ""),
|
||||
act_alias=str(config.bind_act_alias or ""),
|
||||
confirm_act_alias=str(config.confirm_act_alias or ""),
|
||||
legacy_act_alias=str(config.legacy_act_alias or ""),
|
||||
manual_id=manual_id,
|
||||
rid=rid,
|
||||
act_alias=act_alias,
|
||||
confirm_act_alias=confirm_act_alias,
|
||||
legacy_act_alias=legacy_act_alias,
|
||||
client=client,
|
||||
with_goods=True,
|
||||
)
|
||||
@@ -3150,10 +3176,14 @@ def refresh_activity_goods(
|
||||
if account is None:
|
||||
raise CollabError("没有带 CK 的斗鱼账号,无法刷新活动商品", code="no_account")
|
||||
active_client = client or _client_for_account(account)
|
||||
config_info = config_mapping(config)
|
||||
db.expunge(account)
|
||||
db.commit()
|
||||
db.add(account)
|
||||
try:
|
||||
probe = probe_account(
|
||||
account,
|
||||
config,
|
||||
config_info,
|
||||
client=active_client,
|
||||
session_checker=None,
|
||||
with_elite=False,
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""斗鱼后台任务的进程级并发限制。
|
||||
|
||||
任务线程和数据库连接池共享同一个 Uvicorn 进程。这里把「每批并发」和
|
||||
「全进程同时执行数」分开限制,避免多个批次叠加后把请求线程和连接池挤空。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
def _env_positive_int(name: str, default: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
DOUYU_BATCH_TASK_CONCURRENCY = _env_positive_int(
|
||||
"DOUYU_BATCH_TASK_CONCURRENCY",
|
||||
5,
|
||||
)
|
||||
DOUYU_MAX_RUNNING_TASKS = _env_positive_int(
|
||||
"DOUYU_MAX_RUNNING_TASKS",
|
||||
12,
|
||||
)
|
||||
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS = _env_positive_int(
|
||||
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
|
||||
2,
|
||||
)
|
||||
|
||||
_task_slots = threading.BoundedSemaphore(DOUYU_MAX_RUNNING_TASKS)
|
||||
_overview_slots = threading.BoundedSemaphore(DOUYU_MAX_ACTIVE_OVERVIEW_TASKS)
|
||||
|
||||
|
||||
def clamp_batch_task_concurrency(value: int) -> int:
|
||||
return max(1, min(int(value), DOUYU_BATCH_TASK_CONCURRENCY))
|
||||
|
||||
|
||||
def _acquire(slots: threading.BoundedSemaphore, stopped: Callable[[], bool]) -> bool:
|
||||
while not stopped():
|
||||
if slots.acquire(timeout=0.2):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def acquire_running_task_slot(stopped: Callable[[], bool]) -> bool:
|
||||
"""等待一个全进程任务槽;停止时返回 False。"""
|
||||
return _acquire(_task_slots, stopped)
|
||||
|
||||
|
||||
def release_running_task_slot() -> None:
|
||||
_task_slots.release()
|
||||
|
||||
|
||||
def acquire_overview_slot(stopped: Callable[[], bool]) -> bool:
|
||||
"""等待一个一键查询槽;该限制跨所有批次共享。"""
|
||||
return _acquire(_overview_slots, stopped)
|
||||
|
||||
|
||||
def release_overview_slot() -> None:
|
||||
_overview_slots.release()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DOUYU_BATCH_TASK_CONCURRENCY",
|
||||
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
|
||||
"DOUYU_MAX_RUNNING_TASKS",
|
||||
"acquire_overview_slot",
|
||||
"acquire_running_task_slot",
|
||||
"clamp_batch_task_concurrency",
|
||||
"release_overview_slot",
|
||||
"release_running_task_slot",
|
||||
]
|
||||
@@ -12,6 +12,12 @@ from ..database import SessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .cookie_check_service import check_douyu_cookie
|
||||
from .douyu_account_probe import persist_probe_result, probe_session
|
||||
from .douyu_concurrency import (
|
||||
acquire_overview_slot,
|
||||
acquire_running_task_slot,
|
||||
release_overview_slot,
|
||||
release_running_task_slot,
|
||||
)
|
||||
from .douyu_runner_bind import BindMixin
|
||||
from .douyu_runner_core import ( # noqa: F401 (douyu_batch_registry 供 routers 重导出)
|
||||
DouyuBatchRunnerCore,
|
||||
@@ -39,6 +45,25 @@ class DouyuBatchRunner(
|
||||
"""批量执行斗鱼活动任务(功能域 Mixin 聚合 + 批次调度)。"""
|
||||
|
||||
def _execute_one(self, task_id: int, config: dict, total: int):
|
||||
overview_task = self.task_type == "query_account_overview"
|
||||
overview_slot = False
|
||||
running_slot = False
|
||||
if overview_task:
|
||||
overview_slot = acquire_overview_slot(self._stop.is_set)
|
||||
if not overview_slot:
|
||||
return
|
||||
try:
|
||||
running_slot = acquire_running_task_slot(self._stop.is_set)
|
||||
if not running_slot:
|
||||
return
|
||||
self._execute_one_with_slot(task_id, config, total)
|
||||
finally:
|
||||
if running_slot:
|
||||
release_running_task_slot()
|
||||
if overview_slot:
|
||||
release_overview_slot()
|
||||
|
||||
def _execute_one_with_slot(self, task_id: int, config: dict, total: int):
|
||||
worker_db = SessionLocal()
|
||||
try:
|
||||
task = (
|
||||
|
||||
@@ -24,6 +24,7 @@ from ..models import (
|
||||
ProxyConfig as ProxyConfigModel,
|
||||
)
|
||||
from .douyu_account_probe import activity_client
|
||||
from .douyu_concurrency import clamp_batch_task_concurrency
|
||||
from .douyu_goods_service import upsert_goods_snapshot
|
||||
from .douyu_service import (
|
||||
DOUYU_CONFIG_FIELDS,
|
||||
@@ -113,7 +114,7 @@ class DouyuBatchRunnerCore:
|
||||
self.payload = payload or {}
|
||||
self.log_queue = log_queue
|
||||
self.loop = loop
|
||||
self.concurrency = max(1, min(concurrency, 10))
|
||||
self.concurrency = clamp_batch_task_concurrency(concurrency)
|
||||
self._stop = threading.Event()
|
||||
self._counter_lock = threading.Lock()
|
||||
self._started = 0
|
||||
@@ -222,14 +223,25 @@ class DouyuBatchRunnerCore:
|
||||
}
|
||||
asyncio.run_coroutine_threadsafe(self.log_queue.put(event), self.loop)
|
||||
|
||||
def _sync_platform_fulfillment(self, db: Session, task: DouyuTask) -> None:
|
||||
def _sync_platform_fulfillment(self, douyu_task_id: int, *, enabled: bool) -> None:
|
||||
"""绑定/确认任务完成后立即刷新履约投影并写 Outbox。"""
|
||||
if not enabled:
|
||||
return
|
||||
try:
|
||||
from ..database import SessionLocal
|
||||
from .platform_fulfillment.service import sync_task_from_binding_runner
|
||||
|
||||
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
|
||||
# 当前批次会话刚刚提交过任务结果。履约投影使用独立短会话,避免
|
||||
# get_task_detail 留下的只读事务在下一轮外部 HTTP 轮询期间占连接。
|
||||
sync_db = SessionLocal()
|
||||
try:
|
||||
sync_task_from_binding_runner(sync_db, douyu_task_id=int(douyu_task_id))
|
||||
finally:
|
||||
sync_db.close()
|
||||
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
|
||||
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
|
||||
logger.exception(
|
||||
"[douyu] 同步直播账号履约状态失败: task_id={}", douyu_task_id
|
||||
)
|
||||
|
||||
def _sync_platform_fulfillment_operation(
|
||||
self,
|
||||
@@ -261,6 +273,8 @@ class DouyuBatchRunnerCore:
|
||||
message: str,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
task_id = int(task.id)
|
||||
has_platform_task = task.platform_task_id is not None
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
@@ -316,7 +330,7 @@ class DouyuBatchRunnerCore:
|
||||
logger.exception(
|
||||
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
|
||||
)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
|
||||
|
||||
def _update_task_progress(
|
||||
self,
|
||||
@@ -326,13 +340,15 @@ class DouyuBatchRunnerCore:
|
||||
message: str,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
task_id = int(task.id)
|
||||
has_platform_task = task.platform_task_id is not None
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
task.result = slim_douyu_task_result_for_storage(result)
|
||||
db.commit()
|
||||
self._push_task_event(task)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
|
||||
|
||||
def _upsert_goods(self, db: Session, goods: list[dict]) -> None:
|
||||
upsert_goods_snapshot(db, goods)
|
||||
|
||||
@@ -463,6 +463,9 @@ class OverviewMixin:
|
||||
source=f"douyu_task.query_account_overview.{scope}",
|
||||
record_event=False,
|
||||
)
|
||||
# 后面还要查询其他作用域和官方兑换记录。先提交角色快照,
|
||||
# 避免数据库事务跨过下一轮外部 HTTP 请求。
|
||||
db.commit()
|
||||
|
||||
# ---- 官方兑换记录(**三个作用域都拉**)----
|
||||
#
|
||||
@@ -490,6 +493,8 @@ class OverviewMixin:
|
||||
overview[f"exchange_record_count_{scope}"] = len(rows)
|
||||
|
||||
overview["exchange_record_count"] = len(all_records)
|
||||
# 兑换记录开始前结束前一步可能留下的角色快照事务。
|
||||
db.commit()
|
||||
# 把官方记录**补成账号事件**(带真实兑换时间、按作用域去重)。
|
||||
overview["exchange_events_added"] = _record_official_exchanges(
|
||||
db, account, all_records
|
||||
|
||||
Reference in New Issue
Block a user