perf(douyu): 消除任务锁竞争并增加创建背压
This commit is contained in:
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, cast
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from fastapi import (
|
||||
APIRouter,
|
||||
@@ -18,6 +19,7 @@ from fastapi import (
|
||||
from loguru import logger
|
||||
from sqlalchemy import case, func, or_, select
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from core.douyu import (
|
||||
DouyuActivityError,
|
||||
@@ -56,15 +58,20 @@ from ..services.account_binding import bind_payload
|
||||
from ..services.audit_service import record_audit
|
||||
from ..services.business_events import record_recharge
|
||||
from ..services.douyu_activity_query import activity_client, query_activity_goods
|
||||
from ..services.douyu_batch_launcher import start_douyu_batch
|
||||
from ..services.douyu_batch_launcher import (
|
||||
DOUYU_BATCH_CREATE_CONCURRENCY,
|
||||
DOUYU_MAX_ACTIVE_BATCHES,
|
||||
start_douyu_batch,
|
||||
)
|
||||
from ..services.douyu_goods_service import serialize_goods
|
||||
from ..services.douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||
from ..services.douyu_service import (
|
||||
DOUYU_ACTIVE_TASK_STATUSES,
|
||||
DOUYU_CONFIG_FIELDS,
|
||||
DOUYU_HANDBOOK_TASK_TYPES,
|
||||
DOUYU_ACTIVE_TASK_STATUSES,
|
||||
SUPPORTED_DOUYU_TASK_TYPES,
|
||||
DouyuActivePaymentError,
|
||||
DouyuTaskBackpressureError,
|
||||
apply_douyu_config_defaults,
|
||||
cleanup_orphan_douyu_tasks,
|
||||
cookie_holding_filter,
|
||||
@@ -77,6 +84,33 @@ from ..services.platform_fulfillment import service as platform_fulfillment_serv
|
||||
|
||||
router = APIRouter(prefix="/api/douyu", tags=["斗鱼活动"])
|
||||
|
||||
_BATCH_CREATE_SEMAPHORES: WeakKeyDictionary[
|
||||
asyncio.AbstractEventLoop,
|
||||
asyncio.Semaphore,
|
||||
] = WeakKeyDictionary()
|
||||
|
||||
|
||||
def _batch_create_semaphore() -> asyncio.Semaphore:
|
||||
"""按事件循环复用创建锁,避免测试和开发热重载跨 loop 复用 Semaphore。"""
|
||||
loop = asyncio.get_running_loop()
|
||||
semaphore = _BATCH_CREATE_SEMAPHORES.get(loop)
|
||||
if semaphore is None:
|
||||
semaphore = asyncio.Semaphore(DOUYU_BATCH_CREATE_CONCURRENCY)
|
||||
_BATCH_CREATE_SEMAPHORES[loop] = semaphore
|
||||
return semaphore
|
||||
|
||||
|
||||
def _ensure_active_batch_budget(active_count: int) -> None:
|
||||
if active_count >= DOUYU_MAX_ACTIVE_BATCHES:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=(
|
||||
f"斗鱼任务繁忙:当前有 {active_count} 个批次运行中,"
|
||||
f"上限 {DOUYU_MAX_ACTIVE_BATCHES} 个,请稍后重试"
|
||||
),
|
||||
headers={"Retry-After": "5"},
|
||||
)
|
||||
|
||||
|
||||
def _supplier_response_value(payload: dict, *keys: str):
|
||||
"""兼容供应商把订单字段放在根节点、data 或 result 中。"""
|
||||
@@ -782,13 +816,12 @@ def list_xpd_goods(
|
||||
return rows
|
||||
|
||||
|
||||
@router.post("/tasks/batch")
|
||||
async def create_task_batch(
|
||||
def _prepare_task_batch(
|
||||
req: DouyuTaskBatchRequest,
|
||||
db: Session = Depends(get_db),
|
||||
current: User = Depends(require_permission("douyu:task")),
|
||||
):
|
||||
"""创建斗鱼任务记录并启动后台执行器。"""
|
||||
db: Session,
|
||||
current: User,
|
||||
) -> tuple[str, int]:
|
||||
"""在线程池中完成全部同步 DB 工作,避免阻塞 Uvicorn 事件循环。"""
|
||||
if not req.account_ids:
|
||||
raise HTTPException(status_code=400, detail="请选择斗鱼账号")
|
||||
_require_task_account_access(
|
||||
@@ -818,12 +851,14 @@ async def create_task_batch(
|
||||
)
|
||||
platform_task = db.get(PlatformFulfillmentTask, int(platform_task.id))
|
||||
|
||||
active_batch_ids = douyu_batch_registry.active_ids()
|
||||
cleanup_orphan_douyu_tasks(
|
||||
db,
|
||||
active_batch_ids=douyu_batch_registry.active_ids(),
|
||||
active_batch_ids=active_batch_ids,
|
||||
statuses=DOUYU_ACTIVE_TASK_STATUSES,
|
||||
message="任务已中断(无执行器接管)",
|
||||
)
|
||||
_ensure_active_batch_budget(len(active_batch_ids))
|
||||
|
||||
try:
|
||||
batch_id, count = create_douyu_planned_tasks(
|
||||
@@ -837,6 +872,12 @@ async def create_task_batch(
|
||||
)
|
||||
except DouyuActivePaymentError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except DouyuTaskBackpressureError as exc:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=str(exc),
|
||||
headers={"Retry-After": "5"},
|
||||
) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
if count == 0:
|
||||
@@ -911,12 +952,33 @@ async def create_task_batch(
|
||||
)
|
||||
db.commit()
|
||||
|
||||
start_douyu_batch(
|
||||
batch_id=batch_id,
|
||||
task_type=req.task_type,
|
||||
payload=req.payload,
|
||||
concurrency=req.concurrency,
|
||||
)
|
||||
return batch_id, count
|
||||
|
||||
|
||||
@router.post("/tasks/batch")
|
||||
async def create_task_batch(
|
||||
req: DouyuTaskBatchRequest,
|
||||
db: Session = Depends(get_db),
|
||||
current: User = Depends(require_permission("douyu:task")),
|
||||
):
|
||||
"""创建斗鱼任务记录并启动后台执行器。"""
|
||||
# 同步 SQLAlchemy 段必须在线程池执行;事件循环只负责最后启动后台线程。
|
||||
async with _batch_create_semaphore():
|
||||
batch_id, count = await run_in_threadpool(
|
||||
_prepare_task_batch,
|
||||
req,
|
||||
db,
|
||||
current,
|
||||
)
|
||||
await run_in_threadpool(
|
||||
start_douyu_batch,
|
||||
batch_id=batch_id,
|
||||
task_type=req.task_type,
|
||||
payload=req.payload,
|
||||
concurrency=req.concurrency,
|
||||
loop=asyncio.get_running_loop(),
|
||||
log_queue=asyncio.Queue(),
|
||||
)
|
||||
|
||||
return {"batch_id": batch_id, "count": count, "success": True}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user