perf(douyu): 限制任务并发并释放长事务连接

This commit is contained in:
yml
2026-11-20 20:16:53 +08:00
parent 5d44ab4ca1
commit 18fd9840f7
16 changed files with 331 additions and 27 deletions
@@ -0,0 +1,40 @@
"""add douyu_tasks creator/scope/id index
普通用户的任务列表按 created_by + handbook_scope 过滤并按 id 倒序分页。
已有 (created_by, id) 只能定位创建人,再向后扫描数百行才能凑够目标工作台
的任务;新增三列复合索引后可以直接按目标范围倒序取页。
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261120_0100"
down_revision: str | None = "20261119_0099"
branch_labels: str | None = None
depends_on: str | None = None
INDEX_NAME = "ix_douyu_tasks_created_by_scope_id"
def _indexes() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("douyu_tasks"):
return set()
return {index["name"] for index in inspector.get_indexes("douyu_tasks")}
def upgrade() -> None:
if INDEX_NAME not in _indexes():
op.create_index(
INDEX_NAME,
"douyu_tasks",
["created_by", "handbook_scope", "id"],
unique=False,
)
def downgrade() -> None:
if INDEX_NAME in _indexes():
op.drop_index(INDEX_NAME, table_name="douyu_tasks")
+6
View File
@@ -382,6 +382,12 @@ class DouyuTask(Base):
"handbook_scope",
"id",
),
Index(
"ix_douyu_tasks_created_by_scope_id",
"created_by",
"handbook_scope",
"id",
),
)
account: Mapped[Account] = relationship("Account", back_populates="douyu_tasks")
@@ -10,6 +10,7 @@ from typing import Any
from fastapi import APIRouter, Depends, Header, HTTPException, Request
from sqlalchemy.orm import Session
from starlette.requests import ClientDisconnect
from ..database import get_db
from ..models import IntegrationInbox
@@ -50,7 +51,11 @@ async def sync_order_event(
x_integration_signature: str = Header("", alias="X-Integration-Signature"),
) -> dict[str, Any]:
require_integration_key(x_integration_key)
raw_body = await request.body()
try:
raw_body = await request.body()
except ClientDisconnect:
# Webhook 发送方取消不是应用异常,不要把正常断连写成 ASGI traceback。
return {"fetched": 0, "created": 0, "updated": 0, "disconnected": True}
if not verify_webhook_signature(
secret=x_integration_key,
timestamp=x_integration_timestamp,
+1 -1
View File
@@ -1610,7 +1610,7 @@ class DouyuTaskBatchRequest(BaseModel):
account_ids: list[int]
task_type: str
handbook_scope: str = Field(..., pattern="^(elite|esports|peace)$")
concurrency: int = Field(3, ge=1, le=10)
concurrency: int = Field(3, ge=1, le=5)
payload: dict[str, Any] = Field(default_factory=dict)
platform_task_id: int | None = Field(None, gt=0)
+3 -2
View File
@@ -11,6 +11,7 @@ import threading
from web.backend.database import SessionLocal
from .douyu_concurrency import clamp_batch_task_concurrency
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
@@ -26,7 +27,7 @@ DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
"DOUYU_BATCH_CREATE_CONCURRENCY",
2,
)
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 10)
def start_douyu_batch(
@@ -49,7 +50,7 @@ def start_douyu_batch(
payload=payload,
log_queue=log_queue,
loop=loop,
concurrency=max(1, min(int(concurrency), 10)),
concurrency=clamp_batch_task_concurrency(concurrency),
)
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
threading.Thread(target=runner.run, daemon=True).start()
+42 -12
View File
@@ -483,13 +483,21 @@ def lookup_account(
"message": "账号不存在,请先到「账号管理」导入并登录",
"account_username": _text(username, max_length=128),
}
manual_id = str(config.manual_id or "")
rid = str(config.rid or "")
act_alias = str(config.bind_act_alias or "")
confirm_act_alias = str(config.confirm_act_alias or "")
legacy_act_alias = str(config.legacy_act_alias or "")
# 脱离 ORM 后提交,网络期间不持有连接,也不会因 commit 过期而重新查库。
db.expunge(account)
db.commit()
probe = _probe_with_client(
account,
manual_id=str(config.manual_id or ""),
rid=str(config.rid or ""),
act_alias=str(config.bind_act_alias or ""),
confirm_act_alias=str(config.confirm_act_alias or ""),
legacy_act_alias=str(config.legacy_act_alias or ""),
manual_id=manual_id,
rid=rid,
act_alias=act_alias,
confirm_act_alias=confirm_act_alias,
legacy_act_alias=legacy_act_alias,
client=client,
with_goods=with_goods,
)
@@ -1540,6 +1548,15 @@ def create_order(
if exists is not None:
raise CollabError("订单号已存在", status_code=409, code="order_no_exists")
owner_user_id = int(owner_id) if owner_id is not None else int(current.id)
owner = db.get(User, owner_user_id)
if owner is None or not owner.is_active:
raise CollabError(
"负责人不存在或已停用",
status_code=400,
code="owner_invalid",
)
images = normalize_images(home_images)
explicit_points = _to_int(required_points)
name, cid, points = _resolve_gift(
@@ -1551,7 +1568,7 @@ def create_order(
order = DouyuExternalOrder(
order_no=order_text,
shop=_text(shop, max_length=64),
owner_id=int(owner_id) if owner_id is not None else int(current.id),
owner_id=owner_user_id,
account_username=_text(account_username, max_length=128),
gift_name=name,
commodity_id=cid,
@@ -1576,13 +1593,22 @@ def create_order(
)
if account is not None and not draft:
config = resolve_config(db)
manual_id = str(config.manual_id or "")
rid = str(config.rid or "")
act_alias = str(config.bind_act_alias or "")
confirm_act_alias = str(config.confirm_act_alias or "")
legacy_act_alias = str(config.legacy_act_alias or "")
# order 还没有加入 Session。账号脱离 ORM 后提交,探针期间不占连接。
db.expunge(account)
db.commit()
db.add(account)
probe = _probe_with_client(
account,
manual_id=str(config.manual_id or ""),
rid=str(config.rid or ""),
act_alias=str(config.bind_act_alias or ""),
confirm_act_alias=str(config.confirm_act_alias or ""),
legacy_act_alias=str(config.legacy_act_alias or ""),
manual_id=manual_id,
rid=rid,
act_alias=act_alias,
confirm_act_alias=confirm_act_alias,
legacy_act_alias=legacy_act_alias,
client=client,
with_goods=True,
)
@@ -3150,10 +3176,14 @@ def refresh_activity_goods(
if account is None:
raise CollabError("没有带 CK 的斗鱼账号,无法刷新活动商品", code="no_account")
active_client = client or _client_for_account(account)
config_info = config_mapping(config)
db.expunge(account)
db.commit()
db.add(account)
try:
probe = probe_account(
account,
config,
config_info,
client=active_client,
session_checker=None,
with_elite=False,
+77
View File
@@ -0,0 +1,77 @@
"""斗鱼后台任务的进程级并发限制。
任务线程和数据库连接池共享同一个 Uvicorn 进程。这里把「每批并发」和
「全进程同时执行数」分开限制,避免多个批次叠加后把请求线程和连接池挤空。
"""
from __future__ import annotations
import os
import threading
from collections.abc import Callable
def _env_positive_int(name: str, default: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return value if value > 0 else default
DOUYU_BATCH_TASK_CONCURRENCY = _env_positive_int(
"DOUYU_BATCH_TASK_CONCURRENCY",
5,
)
DOUYU_MAX_RUNNING_TASKS = _env_positive_int(
"DOUYU_MAX_RUNNING_TASKS",
12,
)
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS = _env_positive_int(
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
2,
)
_task_slots = threading.BoundedSemaphore(DOUYU_MAX_RUNNING_TASKS)
_overview_slots = threading.BoundedSemaphore(DOUYU_MAX_ACTIVE_OVERVIEW_TASKS)
def clamp_batch_task_concurrency(value: int) -> int:
return max(1, min(int(value), DOUYU_BATCH_TASK_CONCURRENCY))
def _acquire(slots: threading.BoundedSemaphore, stopped: Callable[[], bool]) -> bool:
while not stopped():
if slots.acquire(timeout=0.2):
return True
return False
def acquire_running_task_slot(stopped: Callable[[], bool]) -> bool:
"""等待一个全进程任务槽;停止时返回 False。"""
return _acquire(_task_slots, stopped)
def release_running_task_slot() -> None:
_task_slots.release()
def acquire_overview_slot(stopped: Callable[[], bool]) -> bool:
"""等待一个一键查询槽;该限制跨所有批次共享。"""
return _acquire(_overview_slots, stopped)
def release_overview_slot() -> None:
_overview_slots.release()
__all__ = [
"DOUYU_BATCH_TASK_CONCURRENCY",
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
"DOUYU_MAX_RUNNING_TASKS",
"acquire_overview_slot",
"acquire_running_task_slot",
"clamp_batch_task_concurrency",
"release_overview_slot",
"release_running_task_slot",
]
+25
View File
@@ -12,6 +12,12 @@ from ..database import SessionLocal
from ..models import DouyuTask
from .cookie_check_service import check_douyu_cookie
from .douyu_account_probe import persist_probe_result, probe_session
from .douyu_concurrency import (
acquire_overview_slot,
acquire_running_task_slot,
release_overview_slot,
release_running_task_slot,
)
from .douyu_runner_bind import BindMixin
from .douyu_runner_core import ( # noqa: F401 (douyu_batch_registry 供 routers 重导出)
DouyuBatchRunnerCore,
@@ -39,6 +45,25 @@ class DouyuBatchRunner(
"""批量执行斗鱼活动任务(功能域 Mixin 聚合 + 批次调度)。"""
def _execute_one(self, task_id: int, config: dict, total: int):
overview_task = self.task_type == "query_account_overview"
overview_slot = False
running_slot = False
if overview_task:
overview_slot = acquire_overview_slot(self._stop.is_set)
if not overview_slot:
return
try:
running_slot = acquire_running_task_slot(self._stop.is_set)
if not running_slot:
return
self._execute_one_with_slot(task_id, config, total)
finally:
if running_slot:
release_running_task_slot()
if overview_slot:
release_overview_slot()
def _execute_one_with_slot(self, task_id: int, config: dict, total: int):
worker_db = SessionLocal()
try:
task = (
+22 -6
View File
@@ -24,6 +24,7 @@ from ..models import (
ProxyConfig as ProxyConfigModel,
)
from .douyu_account_probe import activity_client
from .douyu_concurrency import clamp_batch_task_concurrency
from .douyu_goods_service import upsert_goods_snapshot
from .douyu_service import (
DOUYU_CONFIG_FIELDS,
@@ -113,7 +114,7 @@ class DouyuBatchRunnerCore:
self.payload = payload or {}
self.log_queue = log_queue
self.loop = loop
self.concurrency = max(1, min(concurrency, 10))
self.concurrency = clamp_batch_task_concurrency(concurrency)
self._stop = threading.Event()
self._counter_lock = threading.Lock()
self._started = 0
@@ -222,14 +223,25 @@ class DouyuBatchRunnerCore:
}
asyncio.run_coroutine_threadsafe(self.log_queue.put(event), self.loop)
def _sync_platform_fulfillment(self, db: Session, task: DouyuTask) -> None:
def _sync_platform_fulfillment(self, douyu_task_id: int, *, enabled: bool) -> None:
"""绑定/确认任务完成后立即刷新履约投影并写 Outbox。"""
if not enabled:
return
try:
from ..database import SessionLocal
from .platform_fulfillment.service import sync_task_from_binding_runner
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
# 当前批次会话刚刚提交过任务结果。履约投影使用独立短会话,避免
# get_task_detail 留下的只读事务在下一轮外部 HTTP 轮询期间占连接。
sync_db = SessionLocal()
try:
sync_task_from_binding_runner(sync_db, douyu_task_id=int(douyu_task_id))
finally:
sync_db.close()
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
logger.exception(
"[douyu] 同步直播账号履约状态失败: task_id={}", douyu_task_id
)
def _sync_platform_fulfillment_operation(
self,
@@ -261,6 +273,8 @@ class DouyuBatchRunnerCore:
message: str,
result: dict | None = None,
) -> None:
task_id = int(task.id)
has_platform_task = task.platform_task_id is not None
task.status = status
task.message = message[:512]
if result is not None:
@@ -316,7 +330,7 @@ class DouyuBatchRunnerCore:
logger.exception(
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
)
self._sync_platform_fulfillment(db, task)
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
def _update_task_progress(
self,
@@ -326,13 +340,15 @@ class DouyuBatchRunnerCore:
message: str,
result: dict | None = None,
) -> None:
task_id = int(task.id)
has_platform_task = task.platform_task_id is not None
task.status = status
task.message = message[:512]
if result is not None:
task.result = slim_douyu_task_result_for_storage(result)
db.commit()
self._push_task_event(task)
self._sync_platform_fulfillment(db, task)
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
def _upsert_goods(self, db: Session, goods: list[dict]) -> None:
upsert_goods_snapshot(db, goods)
@@ -463,6 +463,9 @@ class OverviewMixin:
source=f"douyu_task.query_account_overview.{scope}",
record_event=False,
)
# 后面还要查询其他作用域和官方兑换记录。先提交角色快照,
# 避免数据库事务跨过下一轮外部 HTTP 请求。
db.commit()
# ---- 官方兑换记录(**三个作用域都拉**)----
#
@@ -490,6 +493,8 @@ class OverviewMixin:
overview[f"exchange_record_count_{scope}"] = len(rows)
overview["exchange_record_count"] = len(all_records)
# 兑换记录开始前结束前一步可能留下的角色快照事务。
db.commit()
# 把官方记录**补成账号事件**(带真实兑换时间、按作用域去重)。
overview["exchange_events_added"] = _record_official_exchanges(
db, account, all_records