perf(douyu): 消除任务锁竞争并增加创建背压

This commit is contained in:
yml
2026-11-19 22:01:21 +08:00
parent 62cb0a41ec
commit 09cdf3f2eb
11 changed files with 555 additions and 95 deletions
+13
View File
@@ -82,6 +82,19 @@ if DATABASE_URL.startswith("sqlite"):
cursor.execute("PRAGMA busy_timeout=30000")
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
else:
@event.listens_for(engine, "connect")
def _set_mysql_session_guards(dbapi_connection, connection_record):
"""让行锁等待快速失败,避免单个事务把连接池堵住几十秒。"""
cursor = dbapi_connection.cursor()
try:
cursor.execute(
"SET SESSION innodb_lock_wait_timeout = "
f"{_env_int('DB_LOCK_WAIT_TIMEOUT', 10)}"
)
finally:
cursor.close()
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)
+1 -1
View File
@@ -201,7 +201,7 @@ app.include_router(files.router)
@app.get("/api/health")
def health():
async def health():
return {"status": "ok"}
+77 -15
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
from datetime import UTC, datetime
from typing import Any, cast
from weakref import WeakKeyDictionary
from fastapi import (
APIRouter,
@@ -18,6 +19,7 @@ from fastapi import (
from loguru import logger
from sqlalchemy import case, func, or_, select
from sqlalchemy.orm import Session, joinedload
from starlette.concurrency import run_in_threadpool
from core.douyu import (
DouyuActivityError,
@@ -56,15 +58,20 @@ from ..services.account_binding import bind_payload
from ..services.audit_service import record_audit
from ..services.business_events import record_recharge
from ..services.douyu_activity_query import activity_client, query_activity_goods
from ..services.douyu_batch_launcher import start_douyu_batch
from ..services.douyu_batch_launcher import (
DOUYU_BATCH_CREATE_CONCURRENCY,
DOUYU_MAX_ACTIVE_BATCHES,
start_douyu_batch,
)
from ..services.douyu_goods_service import serialize_goods
from ..services.douyu_runner import DouyuBatchRunner, douyu_batch_registry
from ..services.douyu_service import (
DOUYU_ACTIVE_TASK_STATUSES,
DOUYU_CONFIG_FIELDS,
DOUYU_HANDBOOK_TASK_TYPES,
DOUYU_ACTIVE_TASK_STATUSES,
SUPPORTED_DOUYU_TASK_TYPES,
DouyuActivePaymentError,
DouyuTaskBackpressureError,
apply_douyu_config_defaults,
cleanup_orphan_douyu_tasks,
cookie_holding_filter,
@@ -77,6 +84,33 @@ from ..services.platform_fulfillment import service as platform_fulfillment_serv
router = APIRouter(prefix="/api/douyu", tags=["斗鱼活动"])
_BATCH_CREATE_SEMAPHORES: WeakKeyDictionary[
asyncio.AbstractEventLoop,
asyncio.Semaphore,
] = WeakKeyDictionary()
def _batch_create_semaphore() -> asyncio.Semaphore:
"""按事件循环复用创建锁,避免测试和开发热重载跨 loop 复用 Semaphore。"""
loop = asyncio.get_running_loop()
semaphore = _BATCH_CREATE_SEMAPHORES.get(loop)
if semaphore is None:
semaphore = asyncio.Semaphore(DOUYU_BATCH_CREATE_CONCURRENCY)
_BATCH_CREATE_SEMAPHORES[loop] = semaphore
return semaphore
def _ensure_active_batch_budget(active_count: int) -> None:
if active_count >= DOUYU_MAX_ACTIVE_BATCHES:
raise HTTPException(
status_code=429,
detail=(
f"斗鱼任务繁忙:当前有 {active_count} 个批次运行中,"
f"上限 {DOUYU_MAX_ACTIVE_BATCHES} 个,请稍后重试"
),
headers={"Retry-After": "5"},
)
def _supplier_response_value(payload: dict, *keys: str):
"""兼容供应商把订单字段放在根节点、data 或 result 中。"""
@@ -782,13 +816,12 @@ def list_xpd_goods(
return rows
@router.post("/tasks/batch")
async def create_task_batch(
def _prepare_task_batch(
req: DouyuTaskBatchRequest,
db: Session = Depends(get_db),
current: User = Depends(require_permission("douyu:task")),
):
"""创建斗鱼任务记录并启动后台执行器。"""
db: Session,
current: User,
) -> tuple[str, int]:
"""在线程池中完成全部同步 DB 工作,避免阻塞 Uvicorn 事件循环。"""
if not req.account_ids:
raise HTTPException(status_code=400, detail="请选择斗鱼账号")
_require_task_account_access(
@@ -818,12 +851,14 @@ async def create_task_batch(
)
platform_task = db.get(PlatformFulfillmentTask, int(platform_task.id))
active_batch_ids = douyu_batch_registry.active_ids()
cleanup_orphan_douyu_tasks(
db,
active_batch_ids=douyu_batch_registry.active_ids(),
active_batch_ids=active_batch_ids,
statuses=DOUYU_ACTIVE_TASK_STATUSES,
message="任务已中断(无执行器接管)",
)
_ensure_active_batch_budget(len(active_batch_ids))
try:
batch_id, count = create_douyu_planned_tasks(
@@ -837,6 +872,12 @@ async def create_task_batch(
)
except DouyuActivePaymentError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except DouyuTaskBackpressureError as exc:
raise HTTPException(
status_code=429,
detail=str(exc),
headers={"Retry-After": "5"},
) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
if count == 0:
@@ -911,12 +952,33 @@ async def create_task_batch(
)
db.commit()
start_douyu_batch(
batch_id=batch_id,
task_type=req.task_type,
payload=req.payload,
concurrency=req.concurrency,
)
return batch_id, count
@router.post("/tasks/batch")
async def create_task_batch(
req: DouyuTaskBatchRequest,
db: Session = Depends(get_db),
current: User = Depends(require_permission("douyu:task")),
):
"""创建斗鱼任务记录并启动后台执行器。"""
# 同步 SQLAlchemy 段必须在线程池执行;事件循环只负责最后启动后台线程。
async with _batch_create_semaphore():
batch_id, count = await run_in_threadpool(
_prepare_task_batch,
req,
db,
current,
)
await run_in_threadpool(
start_douyu_batch,
batch_id=batch_id,
task_type=req.task_type,
payload=req.payload,
concurrency=req.concurrency,
loop=asyncio.get_running_loop(),
log_queue=asyncio.Queue(),
)
return {"batch_id": batch_id, "count": count, "success": True}
+21 -3
View File
@@ -6,6 +6,7 @@
from __future__ import annotations
import asyncio
import os
import threading
from web.backend.database import SessionLocal
@@ -13,16 +14,33 @@ from web.backend.database import SessionLocal
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
def _env_positive_int(name: str, default: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return value if value > 0 else default
DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
"DOUYU_BATCH_CREATE_CONCURRENCY",
2,
)
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
def start_douyu_batch(
*,
batch_id: str,
task_type: str,
payload: dict,
concurrency: int,
loop: asyncio.AbstractEventLoop | None = None,
log_queue: asyncio.Queue | None = None,
) -> None:
"""必须在运行中的 asyncio 事件循环里调用。"""
log_queue = asyncio.Queue()
loop = asyncio.get_running_loop()
"""创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。"""
loop = loop or asyncio.get_running_loop()
log_queue = log_queue or asyncio.Queue()
thread_db = SessionLocal()
runner = DouyuBatchRunner(
db=thread_db,
+79 -31
View File
@@ -6,7 +6,7 @@ import asyncio
import threading
import time
from datetime import UTC, datetime
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from loguru import logger
from sqlalchemy.orm import Session
@@ -59,6 +59,38 @@ DOUYU_PROXY_TASK_TYPES = {
}
_DOUYU_RESULT_DEBUG_KEYS = {
"raw",
"query_params",
"qq_app_scheme",
"role_param_url",
"game_role_bind_url",
}
def slim_douyu_task_result_for_storage(result: dict | None) -> dict | None:
"""去掉任务结果里的调试大字段,避免每次轮询重复写大 JSON。
运行器在内存中仍使用完整 result;这里只影响落库和 WS 展示。二维码 URL、
角色状态、积分、订单号等业务字段全部保留。
"""
if result is None:
return None
def clean(value: Any) -> Any:
if isinstance(value, dict):
return {
key: clean(item)
for key, item in value.items()
if key not in _DOUYU_RESULT_DEBUG_KEYS
}
if isinstance(value, list):
return [clean(item) for item in value]
return value
return clean(result)
class DouyuBatchRunnerCore:
"""斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。"""
@@ -196,9 +228,7 @@ class DouyuBatchRunnerCore:
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
logger.exception(
"[douyu] 同步直播账号履约状态失败: task_id={}", task.id
)
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
def _sync_platform_fulfillment_operation(
self,
@@ -220,9 +250,7 @@ class DouyuBatchRunnerCore:
message=message,
)
except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果
logger.exception(
"[douyu] 写入履约操作时间线失败: task_id={}", task.id
)
logger.exception("[douyu] 写入履约操作时间线失败: task_id={}", task.id)
def _mark_task(
self,
@@ -235,30 +263,28 @@ class DouyuBatchRunnerCore:
task.status = status
task.message = message[:512]
if result is not None:
task.result = result
task.result = slim_douyu_task_result_for_storage(result)
task.finished_at = datetime.now(UTC)
if task.account_id and status in {"success", "failed", "error", "stopped"}:
from .business_events import record_douyu_task_result
account = (
db.query(Account).filter(Account.id == task.account_id).first()
)
if account is not None:
record_douyu_task_result(
db,
account,
task_id=int(task.id),
task_type=str(task.task_type or ""),
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
str(task.task_type or ""),
str(task.task_type or "斗鱼任务"),
),
scope=str(task.handbook_scope or ""),
status=status,
message=message,
occurred_at=task.finished_at,
actor_id=int(task.created_by),
)
account = (
db.query(Account).filter(Account.id == task.account_id).first()
if task.account_id and status in {"success", "failed", "error", "stopped"}
else None
)
event_context = (
{
"task_id": int(task.id),
"task_type": str(task.task_type or ""),
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
str(task.task_type or ""),
str(task.task_type or "斗鱼任务"),
),
"scope": str(task.handbook_scope or ""),
"finished_at": task.finished_at,
"created_by": int(task.created_by),
}
if account is not None
else None
)
self._sync_platform_fulfillment_operation(
db,
task,
@@ -267,6 +293,28 @@ class DouyuBatchRunnerCore:
)
db.commit()
self._push_task_event(task)
if account is not None and event_context is not None:
try:
from .business_events import record_douyu_task_result
record_douyu_task_result(
db,
account,
task_id=event_context["task_id"],
task_type=event_context["task_type"],
task_label=event_context["task_label"],
scope=event_context["scope"],
status=status,
message=message,
occurred_at=event_context["finished_at"],
actor_id=event_context["created_by"],
)
db.commit()
except Exception: # noqa: BLE001 - 时间线旁路不能覆盖已提交的任务结果
db.rollback()
logger.exception(
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
)
self._sync_platform_fulfillment(db, task)
def _update_task_progress(
@@ -280,7 +328,7 @@ class DouyuBatchRunnerCore:
task.status = status
task.message = message[:512]
if result is not None:
task.result = result
task.result = slim_douyu_task_result_for_storage(result)
db.commit()
self._push_task_event(task)
self._sync_platform_fulfillment(db, task)
+118 -43
View File
@@ -2,9 +2,11 @@
from __future__ import annotations
import os
import uuid
from datetime import UTC, datetime
from loguru import logger
from sqlalchemy import and_, func, select
from sqlalchemy.orm import Session
@@ -146,10 +148,42 @@ DOUYU_ACTIVE_PAYMENT_TASK_TYPES = (
)
def _env_positive_int(name: str, default: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return value if value > 0 else default
DOUYU_MAX_ACTIVE_TASKS = _env_positive_int("DOUYU_MAX_ACTIVE_TASKS", 250)
class DouyuActivePaymentError(ValueError):
"""同一账号已有未结束的充值任务。"""
class DouyuTaskBackpressureError(ValueError):
"""全局活跃斗鱼任务超过安全上限。"""
def _ensure_active_task_budget(db: Session, additional: int) -> None:
"""为任务创建提供全局背压,避免无限线程和连接同时挤入数据库。"""
if additional <= 0:
return
active = (
db.query(func.count(DouyuTask.id))
.filter(DouyuTask.status.in_(DOUYU_ACTIVE_TASK_STATUSES))
.scalar()
or 0
)
if active + additional > DOUYU_MAX_ACTIVE_TASKS:
raise DouyuTaskBackpressureError(
f"斗鱼任务繁忙:当前活跃 {active} 个,"
f"上限 {DOUYU_MAX_ACTIVE_TASKS} 个,请稍后重试"
)
def douyu_config_value(field: str, value):
"""读取配置值;空值自动回退到默认值。"""
default = DOUYU_CONFIG_DEFAULTS[field]
@@ -348,6 +382,7 @@ def create_douyu_planned_tasks(
raise ValueError("该任务不属于当前工作台")
accounts = visible_douyu_task_accounts(db, account_ids)
_ensure_active_task_budget(db, len(accounts))
_lock_payment_accounts(db, accounts, task_type)
if (
task_type in {"refresh_goods", "refresh_esports_goods", "refresh_xpd_goods"}
@@ -375,33 +410,39 @@ def create_douyu_planned_tasks(
db.add(task)
task_rows.append((task, account))
db.flush()
events: list[dict] = []
if task_rows:
label = SUPPORTED_DOUYU_TASK_TYPES[task_type]
emit_bulk(
db,
[
{
"account_id": account.id,
"event_type": EVENT_TASK_STARTED,
"title": f"{label}:任务已创建",
"category": CATEGORY_ACTIVITY,
"detail": {
"task_id": task.id,
"task_type": task_type,
"scope": handbook_scope,
"status": "planned",
},
"source": "douyu_task",
"subject_type": "task",
"subject_key": str(task.id),
"actor_type": "user",
"actor_id": created_by,
"source_ref": f"douyu_task_started:{task.id}",
}
for task, account in task_rows
],
)
events = [
{
"account_id": account.id,
"event_type": EVENT_TASK_STARTED,
"title": f"{label}:任务已创建",
"category": CATEGORY_ACTIVITY,
"detail": {
"task_id": task.id,
"task_type": task_type,
"scope": handbook_scope,
"status": "planned",
},
"source": "douyu_task",
"subject_type": "task",
"subject_key": str(task.id),
"actor_type": "user",
"actor_id": created_by,
"source_ref": f"douyu_task_started:{task.id}",
}
for task, account in task_rows
]
# 任务行必须先独立提交。账号事件是旁路,不能在持有任务行锁时批量插入。
db.commit()
if events:
try:
emit_bulk(db, events)
db.commit()
except Exception: # noqa: BLE001 - 时间线不能回滚已经提交的任务
db.rollback()
logger.exception("批量写入斗鱼任务创建事件失败")
return batch_id, len(accounts)
@@ -424,39 +465,73 @@ def cleanup_orphan_douyu_tasks(
if not tasks:
return 0
now = datetime.now(UTC)
from .business_events import record_douyu_task_result
account_ids = {int(task.account_id) for task in tasks if task.account_id}
accounts = (
{
account.id: account
for account in db.query(Account).filter(Account.id.in_(account_ids)).all()
}
if account_ids
else {}
)
event_contexts: list[tuple[dict, Account]] = []
platform_task_ids: list[int] = []
for task in tasks:
task.status = "stopped"
task.message = message
task.finished_at = now
if task.account_id:
account = db.query(Account).filter(Account.id == task.account_id).first()
if account is not None:
record_douyu_task_result(
db,
account = accounts.get(int(task.account_id)) if task.account_id else None
if account is not None:
event_contexts.append(
(
{
"task_id": int(task.id),
"task_type": str(task.task_type or ""),
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
str(task.task_type or ""),
str(task.task_type or "斗鱼任务"),
),
"scope": str(task.handbook_scope or ""),
},
account,
task_id=int(task.id),
task_type=str(task.task_type or ""),
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
str(task.task_type or ""),
str(task.task_type or "斗鱼任务"),
),
scope=str(task.handbook_scope or ""),
status="stopped",
message=message,
occurred_at=now,
)
)
if task.platform_task_id:
from .platform_fulfillment.service import record_douyu_operation_result
platform_task_ids.append(int(task.id))
# 先提交任务终态,再写账号时间线,避免 account_events 与 douyu_tasks
# 在并发创建和结束任务时形成反向锁依赖。
db.commit()
from .business_events import record_douyu_task_result
for context, account in event_contexts:
record_douyu_task_result(
db,
account,
task_id=context["task_id"],
task_type=context["task_type"],
task_label=context["task_label"],
scope=context["scope"],
status="stopped",
message=message,
occurred_at=now,
)
db.commit()
if platform_task_ids:
from .platform_fulfillment.service import record_douyu_operation_result
for task_id in platform_task_ids:
task = db.get(DouyuTask, task_id)
if task is None:
continue
record_douyu_operation_result(
db,
task,
status="stopped",
message=message,
)
db.commit()
db.commit()
return len(tasks)
+4 -1
View File
@@ -138,7 +138,10 @@ def _mark_attempt_failed(
失败重试会新建记录或新 attempt,因此是新的一条 —— 这是对的,那是新的事实。
"""
completed_at = datetime.now(UTC)
attempt_at = task.attempt_at or completed_at
# MySQL DATETIME(0) 会按毫秒四舍五入到整秒。若 source_ref 使用内存里
# 带微秒的原值,第一次写 13:57:13.999 会落成 13:57:14;第二次重读后
# 幂等键变成下一秒,唯一索引就无法拦住重复事件。先截断到整秒再统一使用。
attempt_at = task.attempt_at or completed_at.replace(microsecond=0)
attempt_utc = (
attempt_at.replace(tzinfo=UTC)
if attempt_at.tzinfo is None