perf(douyu): 限制任务并发并释放长事务连接
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
"""add douyu_tasks creator/scope/id index
|
||||
|
||||
普通用户的任务列表按 created_by + handbook_scope 过滤并按 id 倒序分页。
|
||||
已有 (created_by, id) 只能定位创建人,再向后扫描数百行才能凑够目标工作台
|
||||
的任务;新增三列复合索引后可以直接按目标范围倒序取页。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20261120_0100"
|
||||
down_revision: str | None = "20261119_0099"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
INDEX_NAME = "ix_douyu_tasks_created_by_scope_id"
|
||||
|
||||
|
||||
def _indexes() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("douyu_tasks"):
|
||||
return set()
|
||||
return {index["name"] for index in inspector.get_indexes("douyu_tasks")}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if INDEX_NAME not in _indexes():
|
||||
op.create_index(
|
||||
INDEX_NAME,
|
||||
"douyu_tasks",
|
||||
["created_by", "handbook_scope", "id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if INDEX_NAME in _indexes():
|
||||
op.drop_index(INDEX_NAME, table_name="douyu_tasks")
|
||||
@@ -382,6 +382,12 @@ class DouyuTask(Base):
|
||||
"handbook_scope",
|
||||
"id",
|
||||
),
|
||||
Index(
|
||||
"ix_douyu_tasks_created_by_scope_id",
|
||||
"created_by",
|
||||
"handbook_scope",
|
||||
"id",
|
||||
),
|
||||
)
|
||||
|
||||
account: Mapped[Account] = relationship("Account", back_populates="douyu_tasks")
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import IntegrationInbox
|
||||
@@ -50,7 +51,11 @@ async def sync_order_event(
|
||||
x_integration_signature: str = Header("", alias="X-Integration-Signature"),
|
||||
) -> dict[str, Any]:
|
||||
require_integration_key(x_integration_key)
|
||||
raw_body = await request.body()
|
||||
try:
|
||||
raw_body = await request.body()
|
||||
except ClientDisconnect:
|
||||
# Webhook 发送方取消不是应用异常,不要把正常断连写成 ASGI traceback。
|
||||
return {"fetched": 0, "created": 0, "updated": 0, "disconnected": True}
|
||||
if not verify_webhook_signature(
|
||||
secret=x_integration_key,
|
||||
timestamp=x_integration_timestamp,
|
||||
|
||||
@@ -1610,7 +1610,7 @@ class DouyuTaskBatchRequest(BaseModel):
|
||||
account_ids: list[int]
|
||||
task_type: str
|
||||
handbook_scope: str = Field(..., pattern="^(elite|esports|peace)$")
|
||||
concurrency: int = Field(3, ge=1, le=10)
|
||||
concurrency: int = Field(3, ge=1, le=5)
|
||||
payload: dict[str, Any] = Field(default_factory=dict)
|
||||
platform_task_id: int | None = Field(None, gt=0)
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import threading
|
||||
|
||||
from web.backend.database import SessionLocal
|
||||
|
||||
from .douyu_concurrency import clamp_batch_task_concurrency
|
||||
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||
|
||||
|
||||
@@ -26,7 +27,7 @@ DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
|
||||
"DOUYU_BATCH_CREATE_CONCURRENCY",
|
||||
2,
|
||||
)
|
||||
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
|
||||
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 10)
|
||||
|
||||
|
||||
def start_douyu_batch(
|
||||
@@ -49,7 +50,7 @@ def start_douyu_batch(
|
||||
payload=payload,
|
||||
log_queue=log_queue,
|
||||
loop=loop,
|
||||
concurrency=max(1, min(int(concurrency), 10)),
|
||||
concurrency=clamp_batch_task_concurrency(concurrency),
|
||||
)
|
||||
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
|
||||
threading.Thread(target=runner.run, daemon=True).start()
|
||||
|
||||
@@ -483,13 +483,21 @@ def lookup_account(
|
||||
"message": "账号不存在,请先到「账号管理」导入并登录",
|
||||
"account_username": _text(username, max_length=128),
|
||||
}
|
||||
manual_id = str(config.manual_id or "")
|
||||
rid = str(config.rid or "")
|
||||
act_alias = str(config.bind_act_alias or "")
|
||||
confirm_act_alias = str(config.confirm_act_alias or "")
|
||||
legacy_act_alias = str(config.legacy_act_alias or "")
|
||||
# 脱离 ORM 后提交,网络期间不持有连接,也不会因 commit 过期而重新查库。
|
||||
db.expunge(account)
|
||||
db.commit()
|
||||
probe = _probe_with_client(
|
||||
account,
|
||||
manual_id=str(config.manual_id or ""),
|
||||
rid=str(config.rid or ""),
|
||||
act_alias=str(config.bind_act_alias or ""),
|
||||
confirm_act_alias=str(config.confirm_act_alias or ""),
|
||||
legacy_act_alias=str(config.legacy_act_alias or ""),
|
||||
manual_id=manual_id,
|
||||
rid=rid,
|
||||
act_alias=act_alias,
|
||||
confirm_act_alias=confirm_act_alias,
|
||||
legacy_act_alias=legacy_act_alias,
|
||||
client=client,
|
||||
with_goods=with_goods,
|
||||
)
|
||||
@@ -1540,6 +1548,15 @@ def create_order(
|
||||
if exists is not None:
|
||||
raise CollabError("订单号已存在", status_code=409, code="order_no_exists")
|
||||
|
||||
owner_user_id = int(owner_id) if owner_id is not None else int(current.id)
|
||||
owner = db.get(User, owner_user_id)
|
||||
if owner is None or not owner.is_active:
|
||||
raise CollabError(
|
||||
"负责人不存在或已停用",
|
||||
status_code=400,
|
||||
code="owner_invalid",
|
||||
)
|
||||
|
||||
images = normalize_images(home_images)
|
||||
explicit_points = _to_int(required_points)
|
||||
name, cid, points = _resolve_gift(
|
||||
@@ -1551,7 +1568,7 @@ def create_order(
|
||||
order = DouyuExternalOrder(
|
||||
order_no=order_text,
|
||||
shop=_text(shop, max_length=64),
|
||||
owner_id=int(owner_id) if owner_id is not None else int(current.id),
|
||||
owner_id=owner_user_id,
|
||||
account_username=_text(account_username, max_length=128),
|
||||
gift_name=name,
|
||||
commodity_id=cid,
|
||||
@@ -1576,13 +1593,22 @@ def create_order(
|
||||
)
|
||||
if account is not None and not draft:
|
||||
config = resolve_config(db)
|
||||
manual_id = str(config.manual_id or "")
|
||||
rid = str(config.rid or "")
|
||||
act_alias = str(config.bind_act_alias or "")
|
||||
confirm_act_alias = str(config.confirm_act_alias or "")
|
||||
legacy_act_alias = str(config.legacy_act_alias or "")
|
||||
# order 还没有加入 Session。账号脱离 ORM 后提交,探针期间不占连接。
|
||||
db.expunge(account)
|
||||
db.commit()
|
||||
db.add(account)
|
||||
probe = _probe_with_client(
|
||||
account,
|
||||
manual_id=str(config.manual_id or ""),
|
||||
rid=str(config.rid or ""),
|
||||
act_alias=str(config.bind_act_alias or ""),
|
||||
confirm_act_alias=str(config.confirm_act_alias or ""),
|
||||
legacy_act_alias=str(config.legacy_act_alias or ""),
|
||||
manual_id=manual_id,
|
||||
rid=rid,
|
||||
act_alias=act_alias,
|
||||
confirm_act_alias=confirm_act_alias,
|
||||
legacy_act_alias=legacy_act_alias,
|
||||
client=client,
|
||||
with_goods=True,
|
||||
)
|
||||
@@ -3150,10 +3176,14 @@ def refresh_activity_goods(
|
||||
if account is None:
|
||||
raise CollabError("没有带 CK 的斗鱼账号,无法刷新活动商品", code="no_account")
|
||||
active_client = client or _client_for_account(account)
|
||||
config_info = config_mapping(config)
|
||||
db.expunge(account)
|
||||
db.commit()
|
||||
db.add(account)
|
||||
try:
|
||||
probe = probe_account(
|
||||
account,
|
||||
config,
|
||||
config_info,
|
||||
client=active_client,
|
||||
session_checker=None,
|
||||
with_elite=False,
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""斗鱼后台任务的进程级并发限制。
|
||||
|
||||
任务线程和数据库连接池共享同一个 Uvicorn 进程。这里把「每批并发」和
|
||||
「全进程同时执行数」分开限制,避免多个批次叠加后把请求线程和连接池挤空。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
def _env_positive_int(name: str, default: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
DOUYU_BATCH_TASK_CONCURRENCY = _env_positive_int(
|
||||
"DOUYU_BATCH_TASK_CONCURRENCY",
|
||||
5,
|
||||
)
|
||||
DOUYU_MAX_RUNNING_TASKS = _env_positive_int(
|
||||
"DOUYU_MAX_RUNNING_TASKS",
|
||||
12,
|
||||
)
|
||||
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS = _env_positive_int(
|
||||
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
|
||||
2,
|
||||
)
|
||||
|
||||
_task_slots = threading.BoundedSemaphore(DOUYU_MAX_RUNNING_TASKS)
|
||||
_overview_slots = threading.BoundedSemaphore(DOUYU_MAX_ACTIVE_OVERVIEW_TASKS)
|
||||
|
||||
|
||||
def clamp_batch_task_concurrency(value: int) -> int:
|
||||
return max(1, min(int(value), DOUYU_BATCH_TASK_CONCURRENCY))
|
||||
|
||||
|
||||
def _acquire(slots: threading.BoundedSemaphore, stopped: Callable[[], bool]) -> bool:
|
||||
while not stopped():
|
||||
if slots.acquire(timeout=0.2):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def acquire_running_task_slot(stopped: Callable[[], bool]) -> bool:
|
||||
"""等待一个全进程任务槽;停止时返回 False。"""
|
||||
return _acquire(_task_slots, stopped)
|
||||
|
||||
|
||||
def release_running_task_slot() -> None:
|
||||
_task_slots.release()
|
||||
|
||||
|
||||
def acquire_overview_slot(stopped: Callable[[], bool]) -> bool:
|
||||
"""等待一个一键查询槽;该限制跨所有批次共享。"""
|
||||
return _acquire(_overview_slots, stopped)
|
||||
|
||||
|
||||
def release_overview_slot() -> None:
|
||||
_overview_slots.release()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DOUYU_BATCH_TASK_CONCURRENCY",
|
||||
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
|
||||
"DOUYU_MAX_RUNNING_TASKS",
|
||||
"acquire_overview_slot",
|
||||
"acquire_running_task_slot",
|
||||
"clamp_batch_task_concurrency",
|
||||
"release_overview_slot",
|
||||
"release_running_task_slot",
|
||||
]
|
||||
@@ -12,6 +12,12 @@ from ..database import SessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .cookie_check_service import check_douyu_cookie
|
||||
from .douyu_account_probe import persist_probe_result, probe_session
|
||||
from .douyu_concurrency import (
|
||||
acquire_overview_slot,
|
||||
acquire_running_task_slot,
|
||||
release_overview_slot,
|
||||
release_running_task_slot,
|
||||
)
|
||||
from .douyu_runner_bind import BindMixin
|
||||
from .douyu_runner_core import ( # noqa: F401 (douyu_batch_registry 供 routers 重导出)
|
||||
DouyuBatchRunnerCore,
|
||||
@@ -39,6 +45,25 @@ class DouyuBatchRunner(
|
||||
"""批量执行斗鱼活动任务(功能域 Mixin 聚合 + 批次调度)。"""
|
||||
|
||||
def _execute_one(self, task_id: int, config: dict, total: int):
|
||||
overview_task = self.task_type == "query_account_overview"
|
||||
overview_slot = False
|
||||
running_slot = False
|
||||
if overview_task:
|
||||
overview_slot = acquire_overview_slot(self._stop.is_set)
|
||||
if not overview_slot:
|
||||
return
|
||||
try:
|
||||
running_slot = acquire_running_task_slot(self._stop.is_set)
|
||||
if not running_slot:
|
||||
return
|
||||
self._execute_one_with_slot(task_id, config, total)
|
||||
finally:
|
||||
if running_slot:
|
||||
release_running_task_slot()
|
||||
if overview_slot:
|
||||
release_overview_slot()
|
||||
|
||||
def _execute_one_with_slot(self, task_id: int, config: dict, total: int):
|
||||
worker_db = SessionLocal()
|
||||
try:
|
||||
task = (
|
||||
|
||||
@@ -24,6 +24,7 @@ from ..models import (
|
||||
ProxyConfig as ProxyConfigModel,
|
||||
)
|
||||
from .douyu_account_probe import activity_client
|
||||
from .douyu_concurrency import clamp_batch_task_concurrency
|
||||
from .douyu_goods_service import upsert_goods_snapshot
|
||||
from .douyu_service import (
|
||||
DOUYU_CONFIG_FIELDS,
|
||||
@@ -113,7 +114,7 @@ class DouyuBatchRunnerCore:
|
||||
self.payload = payload or {}
|
||||
self.log_queue = log_queue
|
||||
self.loop = loop
|
||||
self.concurrency = max(1, min(concurrency, 10))
|
||||
self.concurrency = clamp_batch_task_concurrency(concurrency)
|
||||
self._stop = threading.Event()
|
||||
self._counter_lock = threading.Lock()
|
||||
self._started = 0
|
||||
@@ -222,14 +223,25 @@ class DouyuBatchRunnerCore:
|
||||
}
|
||||
asyncio.run_coroutine_threadsafe(self.log_queue.put(event), self.loop)
|
||||
|
||||
def _sync_platform_fulfillment(self, db: Session, task: DouyuTask) -> None:
|
||||
def _sync_platform_fulfillment(self, douyu_task_id: int, *, enabled: bool) -> None:
|
||||
"""绑定/确认任务完成后立即刷新履约投影并写 Outbox。"""
|
||||
if not enabled:
|
||||
return
|
||||
try:
|
||||
from ..database import SessionLocal
|
||||
from .platform_fulfillment.service import sync_task_from_binding_runner
|
||||
|
||||
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
|
||||
# 当前批次会话刚刚提交过任务结果。履约投影使用独立短会话,避免
|
||||
# get_task_detail 留下的只读事务在下一轮外部 HTTP 轮询期间占连接。
|
||||
sync_db = SessionLocal()
|
||||
try:
|
||||
sync_task_from_binding_runner(sync_db, douyu_task_id=int(douyu_task_id))
|
||||
finally:
|
||||
sync_db.close()
|
||||
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
|
||||
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
|
||||
logger.exception(
|
||||
"[douyu] 同步直播账号履约状态失败: task_id={}", douyu_task_id
|
||||
)
|
||||
|
||||
def _sync_platform_fulfillment_operation(
|
||||
self,
|
||||
@@ -261,6 +273,8 @@ class DouyuBatchRunnerCore:
|
||||
message: str,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
task_id = int(task.id)
|
||||
has_platform_task = task.platform_task_id is not None
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
@@ -316,7 +330,7 @@ class DouyuBatchRunnerCore:
|
||||
logger.exception(
|
||||
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
|
||||
)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
|
||||
|
||||
def _update_task_progress(
|
||||
self,
|
||||
@@ -326,13 +340,15 @@ class DouyuBatchRunnerCore:
|
||||
message: str,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
task_id = int(task.id)
|
||||
has_platform_task = task.platform_task_id is not None
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
task.result = slim_douyu_task_result_for_storage(result)
|
||||
db.commit()
|
||||
self._push_task_event(task)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
|
||||
|
||||
def _upsert_goods(self, db: Session, goods: list[dict]) -> None:
|
||||
upsert_goods_snapshot(db, goods)
|
||||
|
||||
@@ -463,6 +463,9 @@ class OverviewMixin:
|
||||
source=f"douyu_task.query_account_overview.{scope}",
|
||||
record_event=False,
|
||||
)
|
||||
# 后面还要查询其他作用域和官方兑换记录。先提交角色快照,
|
||||
# 避免数据库事务跨过下一轮外部 HTTP 请求。
|
||||
db.commit()
|
||||
|
||||
# ---- 官方兑换记录(**三个作用域都拉**)----
|
||||
#
|
||||
@@ -490,6 +493,8 @@ class OverviewMixin:
|
||||
overview[f"exchange_record_count_{scope}"] = len(rows)
|
||||
|
||||
overview["exchange_record_count"] = len(all_records)
|
||||
# 兑换记录开始前结束前一步可能留下的角色快照事务。
|
||||
db.commit()
|
||||
# 把官方记录**补成账号事件**(带真实兑换时间、按作用域去重)。
|
||||
overview["exchange_events_added"] = _record_official_exchanges(
|
||||
db, account, all_records
|
||||
|
||||
Reference in New Issue
Block a user