perf(douyu): 消除任务锁竞争并增加创建背压

This commit is contained in:
yml
2026-11-19 22:01:21 +08:00
parent 62cb0a41ec
commit 09cdf3f2eb
11 changed files with 555 additions and 95 deletions
+6
View File
@@ -68,6 +68,12 @@ DB_POOL_SIZE=20
DB_MAX_OVERFLOW=20 DB_MAX_OVERFLOW=20
DB_POOL_TIMEOUT=30 DB_POOL_TIMEOUT=30
DB_POOL_RECYCLE=1800 DB_POOL_RECYCLE=1800
# 行锁 10 秒仍未拿到就失败,避免把连接池和请求线程长期堵住。
DB_LOCK_WAIT_TIMEOUT=10
# 斗鱼批次创建并发与全局活跃量背压。
DOUYU_BATCH_CREATE_CONCURRENCY=2
DOUYU_MAX_ACTIVE_BATCHES=20
DOUYU_MAX_ACTIVE_TASKS=250
# Cookie 安全标志(生产环境 HTTPS 部署时设为 true) # Cookie 安全标志(生产环境 HTTPS 部署时设为 true)
COOKIE_SECURE=false COOKIE_SECURE=false
+4
View File
@@ -40,6 +40,10 @@ services:
- DB_MAX_OVERFLOW=${DB_MAX_OVERFLOW:-20} - DB_MAX_OVERFLOW=${DB_MAX_OVERFLOW:-20}
- DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30} - DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30}
- DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-1800} - DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-1800}
- DB_LOCK_WAIT_TIMEOUT=${DB_LOCK_WAIT_TIMEOUT:-10}
- DOUYU_BATCH_CREATE_CONCURRENCY=${DOUYU_BATCH_CREATE_CONCURRENCY:-2}
- DOUYU_MAX_ACTIVE_BATCHES=${DOUYU_MAX_ACTIVE_BATCHES:-20}
- DOUYU_MAX_ACTIVE_TASKS=${DOUYU_MAX_ACTIVE_TASKS:-250}
# JWT 密钥(生产环境务必修改,可用 python -c "import secrets; print(secrets.token_urlsafe(32))" 生成) # JWT 密钥(生产环境务必修改,可用 python -c "import secrets; print(secrets.token_urlsafe(32))" 生成)
- JWT_SECRET_KEY=${JWT_SECRET_KEY:-} - JWT_SECRET_KEY=${JWT_SECRET_KEY:-}
# 后台登录滑动会话:24h 空闲过期,7d 绝对上限。 # 后台登录滑动会话:24h 空闲过期,7d 绝对上限。
+227
View File
@@ -0,0 +1,227 @@
import inspect
import pytest
from fastapi import HTTPException
from sqlalchemy import text
from tests.support.db import make_test_session
from web.backend.database import engine as app_engine
from web.backend.main import health
from web.backend.models import Account, DouyuTask, User
from web.backend.routers import douyu as douyu_router
from web.backend.services import business_events, douyu_service
from web.backend.services.douyu_runner_core import (
DouyuBatchRunnerCore,
slim_douyu_task_result_for_storage,
)
def _fixtures(db, *, status: str = "planned") -> tuple[User, Account, DouyuTask]:
user = User(username="concurrency-user", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="concurrency-account",
password="password",
email="concurrency@example.com",
email_password="mail-password",
cookie="acf_uid=1",
)
db.add(account)
db.flush()
task = DouyuTask(
batch_id="concurrency-batch",
account_id=account.id,
account_assigned_to=user.id,
task_type="get_bind_qr",
handbook_scope="elite",
status=status,
created_by=user.id,
)
db.add(task)
db.commit()
return user, account, task
def test_task_rows_commit_before_started_events(db, monkeypatch):
user = User(username="event-order", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="event-order-account",
password="password",
email="event-order@example.com",
email_password="mail-password",
cookie="acf_uid=2",
)
db.add(account)
db.commit()
observed: list[int] = []
def fake_emit_bulk_read_committed(_db, events):
other = make_test_session()
try:
observed.append(
other.query(DouyuTask).filter(DouyuTask.status == "planned").count()
)
finally:
other.close()
return len(events)
monkeypatch.setattr(douyu_service, "emit_bulk", fake_emit_bulk_read_committed)
batch_id, count = douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_points",
"elite",
user.id,
)
assert batch_id
assert count == 1
assert observed == [1]
def test_mark_task_commits_result_before_account_event(db, monkeypatch):
_, account, task = _fixtures(db, status="running")
task.account = account
runner = DouyuBatchRunnerCore(
db,
task.batch_id,
task.task_type,
close_db=False,
)
observed: list[tuple[str, object]] = []
def fake_record(_db, _account, **kwargs):
other = make_test_session()
try:
row = other.get(DouyuTask, task.id)
observed.append((row.status, row.result))
finally:
other.close()
return True
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
runner._mark_task(
db,
task,
"success",
"done",
{
"url": "https://example.test/qr",
"raw": {"large": "payload"},
"before_bind_info": {
"role_name": "role-a",
"raw": {"large": "nested"},
},
},
)
assert observed == [
(
"success",
{
"url": "https://example.test/qr",
"before_bind_info": {"role_name": "role-a"},
},
)
]
def test_cleanup_commits_stopped_status_before_account_event(db, monkeypatch):
_, _, task = _fixtures(db, status="running")
observed: list[str] = []
def fake_record(_db, _account, **kwargs):
other = make_test_session()
try:
observed.append(other.get(DouyuTask, task.id).status)
finally:
other.close()
return True
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
cleaned = douyu_service.cleanup_orphan_douyu_tasks(
db,
active_batch_ids=set(),
statuses=("running",),
)
assert cleaned == 1
assert observed == ["stopped"]
def test_active_task_budget_rejects_new_batch(db, monkeypatch):
user = User(username="budget-user", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="budget-account",
password="password",
email="budget@example.com",
email_password="mail-password",
cookie="acf_uid=3",
)
db.add(account)
db.commit()
monkeypatch.setattr(douyu_service, "DOUYU_MAX_ACTIVE_TASKS", 1)
douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_points",
"elite",
user.id,
)
with pytest.raises(douyu_service.DouyuTaskBackpressureError, match="上限 1"):
douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_game_name",
"elite",
user.id,
)
def test_active_batch_budget_returns_429(monkeypatch):
monkeypatch.setattr(douyu_router, "DOUYU_MAX_ACTIVE_BATCHES", 1)
with pytest.raises(HTTPException) as exc_info:
douyu_router._ensure_active_batch_budget(1)
assert exc_info.value.status_code == 429
def test_slim_task_result_removes_debug_payloads():
result = slim_douyu_task_result_for_storage(
{
"url": "https://example.test/qr",
"raw": {"debug": True},
"query_params": {"token": "secret"},
"before_bind_info": {
"role_name": "role-a",
"raw": {"debug": True},
},
"items": [{"name": "kept", "raw": {"debug": True}}],
}
)
assert result == {
"url": "https://example.test/qr",
"before_bind_info": {"role_name": "role-a"},
"items": [{"name": "kept"}],
}
def test_health_endpoint_does_not_use_db_threadpool():
assert inspect.iscoroutinefunction(health)
def test_mysql_lock_wait_timeout_is_bounded():
with app_engine.connect() as connection:
value = connection.execute(
text("SELECT @@SESSION.innodb_lock_wait_timeout")
).scalar()
assert value is not None
assert int(value) <= 10
+5 -1
View File
@@ -142,7 +142,11 @@ class TestLoginFailureEvents(_Base):
self.session.commit() self.session.commit()
_mark_attempt_failed(task, "极验验证失败: slide") _mark_attempt_failed(task, "极验验证失败: slide")
self.session.commit() self.session.commit()
assert len(self.events(account.id)) == 1 events = self.events(account.id)
assert len(events) == 1, [
(event.source_ref, event.occurred_at) for event in events
]
assert events[0].occurred_at.microsecond == 0
def test_event_does_not_break_when_task_has_no_session(self): def test_event_does_not_break_when_task_has_no_session(self):
"""纯内存构造的任务(没有 session)不能因为写事件而炸掉主流程。""" """纯内存构造的任务(没有 session)不能因为写事件而炸掉主流程。"""
+13
View File
@@ -82,6 +82,19 @@ if DATABASE_URL.startswith("sqlite"):
cursor.execute("PRAGMA busy_timeout=30000") cursor.execute("PRAGMA busy_timeout=30000")
cursor.execute("PRAGMA foreign_keys=ON") cursor.execute("PRAGMA foreign_keys=ON")
cursor.close() cursor.close()
else:
@event.listens_for(engine, "connect")
def _set_mysql_session_guards(dbapi_connection, connection_record):
"""让行锁等待快速失败,避免单个事务把连接池堵住几十秒。"""
cursor = dbapi_connection.cursor()
try:
cursor.execute(
"SET SESSION innodb_lock_wait_timeout = "
f"{_env_int('DB_LOCK_WAIT_TIMEOUT', 10)}"
)
finally:
cursor.close()
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False) SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)
+1 -1
View File
@@ -201,7 +201,7 @@ app.include_router(files.router)
@app.get("/api/health") @app.get("/api/health")
def health(): async def health():
return {"status": "ok"} return {"status": "ok"}
+77 -15
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio import asyncio
from datetime import UTC, datetime from datetime import UTC, datetime
from typing import Any, cast from typing import Any, cast
from weakref import WeakKeyDictionary
from fastapi import ( from fastapi import (
APIRouter, APIRouter,
@@ -18,6 +19,7 @@ from fastapi import (
from loguru import logger from loguru import logger
from sqlalchemy import case, func, or_, select from sqlalchemy import case, func, or_, select
from sqlalchemy.orm import Session, joinedload from sqlalchemy.orm import Session, joinedload
from starlette.concurrency import run_in_threadpool
from core.douyu import ( from core.douyu import (
DouyuActivityError, DouyuActivityError,
@@ -56,15 +58,20 @@ from ..services.account_binding import bind_payload
from ..services.audit_service import record_audit from ..services.audit_service import record_audit
from ..services.business_events import record_recharge from ..services.business_events import record_recharge
from ..services.douyu_activity_query import activity_client, query_activity_goods from ..services.douyu_activity_query import activity_client, query_activity_goods
from ..services.douyu_batch_launcher import start_douyu_batch from ..services.douyu_batch_launcher import (
DOUYU_BATCH_CREATE_CONCURRENCY,
DOUYU_MAX_ACTIVE_BATCHES,
start_douyu_batch,
)
from ..services.douyu_goods_service import serialize_goods from ..services.douyu_goods_service import serialize_goods
from ..services.douyu_runner import DouyuBatchRunner, douyu_batch_registry from ..services.douyu_runner import DouyuBatchRunner, douyu_batch_registry
from ..services.douyu_service import ( from ..services.douyu_service import (
DOUYU_ACTIVE_TASK_STATUSES,
DOUYU_CONFIG_FIELDS, DOUYU_CONFIG_FIELDS,
DOUYU_HANDBOOK_TASK_TYPES, DOUYU_HANDBOOK_TASK_TYPES,
DOUYU_ACTIVE_TASK_STATUSES,
SUPPORTED_DOUYU_TASK_TYPES, SUPPORTED_DOUYU_TASK_TYPES,
DouyuActivePaymentError, DouyuActivePaymentError,
DouyuTaskBackpressureError,
apply_douyu_config_defaults, apply_douyu_config_defaults,
cleanup_orphan_douyu_tasks, cleanup_orphan_douyu_tasks,
cookie_holding_filter, cookie_holding_filter,
@@ -77,6 +84,33 @@ from ..services.platform_fulfillment import service as platform_fulfillment_serv
router = APIRouter(prefix="/api/douyu", tags=["斗鱼活动"]) router = APIRouter(prefix="/api/douyu", tags=["斗鱼活动"])
_BATCH_CREATE_SEMAPHORES: WeakKeyDictionary[
asyncio.AbstractEventLoop,
asyncio.Semaphore,
] = WeakKeyDictionary()
def _batch_create_semaphore() -> asyncio.Semaphore:
"""按事件循环复用创建锁,避免测试和开发热重载跨 loop 复用 Semaphore。"""
loop = asyncio.get_running_loop()
semaphore = _BATCH_CREATE_SEMAPHORES.get(loop)
if semaphore is None:
semaphore = asyncio.Semaphore(DOUYU_BATCH_CREATE_CONCURRENCY)
_BATCH_CREATE_SEMAPHORES[loop] = semaphore
return semaphore
def _ensure_active_batch_budget(active_count: int) -> None:
if active_count >= DOUYU_MAX_ACTIVE_BATCHES:
raise HTTPException(
status_code=429,
detail=(
f"斗鱼任务繁忙:当前有 {active_count} 个批次运行中,"
f"上限 {DOUYU_MAX_ACTIVE_BATCHES} 个,请稍后重试"
),
headers={"Retry-After": "5"},
)
def _supplier_response_value(payload: dict, *keys: str): def _supplier_response_value(payload: dict, *keys: str):
"""兼容供应商把订单字段放在根节点、data 或 result 中。""" """兼容供应商把订单字段放在根节点、data 或 result 中。"""
@@ -782,13 +816,12 @@ def list_xpd_goods(
return rows return rows
@router.post("/tasks/batch") def _prepare_task_batch(
async def create_task_batch(
req: DouyuTaskBatchRequest, req: DouyuTaskBatchRequest,
db: Session = Depends(get_db), db: Session,
current: User = Depends(require_permission("douyu:task")), current: User,
): ) -> tuple[str, int]:
"""创建斗鱼任务记录并启动后台执行器。""" """在线程池中完成全部同步 DB 工作,避免阻塞 Uvicorn 事件循环。"""
if not req.account_ids: if not req.account_ids:
raise HTTPException(status_code=400, detail="请选择斗鱼账号") raise HTTPException(status_code=400, detail="请选择斗鱼账号")
_require_task_account_access( _require_task_account_access(
@@ -818,12 +851,14 @@ async def create_task_batch(
) )
platform_task = db.get(PlatformFulfillmentTask, int(platform_task.id)) platform_task = db.get(PlatformFulfillmentTask, int(platform_task.id))
active_batch_ids = douyu_batch_registry.active_ids()
cleanup_orphan_douyu_tasks( cleanup_orphan_douyu_tasks(
db, db,
active_batch_ids=douyu_batch_registry.active_ids(), active_batch_ids=active_batch_ids,
statuses=DOUYU_ACTIVE_TASK_STATUSES, statuses=DOUYU_ACTIVE_TASK_STATUSES,
message="任务已中断(无执行器接管)", message="任务已中断(无执行器接管)",
) )
_ensure_active_batch_budget(len(active_batch_ids))
try: try:
batch_id, count = create_douyu_planned_tasks( batch_id, count = create_douyu_planned_tasks(
@@ -837,6 +872,12 @@ async def create_task_batch(
) )
except DouyuActivePaymentError as exc: except DouyuActivePaymentError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc raise HTTPException(status_code=409, detail=str(exc)) from exc
except DouyuTaskBackpressureError as exc:
raise HTTPException(
status_code=429,
detail=str(exc),
headers={"Retry-After": "5"},
) from exc
except ValueError as exc: except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc raise HTTPException(status_code=400, detail=str(exc)) from exc
if count == 0: if count == 0:
@@ -911,12 +952,33 @@ async def create_task_batch(
) )
db.commit() db.commit()
start_douyu_batch( return batch_id, count
batch_id=batch_id,
task_type=req.task_type,
payload=req.payload, @router.post("/tasks/batch")
concurrency=req.concurrency, async def create_task_batch(
) req: DouyuTaskBatchRequest,
db: Session = Depends(get_db),
current: User = Depends(require_permission("douyu:task")),
):
"""创建斗鱼任务记录并启动后台执行器。"""
# 同步 SQLAlchemy 段必须在线程池执行;事件循环只负责最后启动后台线程。
async with _batch_create_semaphore():
batch_id, count = await run_in_threadpool(
_prepare_task_batch,
req,
db,
current,
)
await run_in_threadpool(
start_douyu_batch,
batch_id=batch_id,
task_type=req.task_type,
payload=req.payload,
concurrency=req.concurrency,
loop=asyncio.get_running_loop(),
log_queue=asyncio.Queue(),
)
return {"batch_id": batch_id, "count": count, "success": True} return {"batch_id": batch_id, "count": count, "success": True}
+21 -3
View File
@@ -6,6 +6,7 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import os
import threading import threading
from web.backend.database import SessionLocal from web.backend.database import SessionLocal
@@ -13,16 +14,33 @@ from web.backend.database import SessionLocal
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
def _env_positive_int(name: str, default: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return value if value > 0 else default
DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
"DOUYU_BATCH_CREATE_CONCURRENCY",
2,
)
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
def start_douyu_batch( def start_douyu_batch(
*, *,
batch_id: str, batch_id: str,
task_type: str, task_type: str,
payload: dict, payload: dict,
concurrency: int, concurrency: int,
loop: asyncio.AbstractEventLoop | None = None,
log_queue: asyncio.Queue | None = None,
) -> None: ) -> None:
"""必须在运行中的 asyncio 事件循环里调用。""" """创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。"""
log_queue = asyncio.Queue() loop = loop or asyncio.get_running_loop()
loop = asyncio.get_running_loop() log_queue = log_queue or asyncio.Queue()
thread_db = SessionLocal() thread_db = SessionLocal()
runner = DouyuBatchRunner( runner = DouyuBatchRunner(
db=thread_db, db=thread_db,
+79 -31
View File
@@ -6,7 +6,7 @@ import asyncio
import threading import threading
import time import time
from datetime import UTC, datetime from datetime import UTC, datetime
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, Any
from loguru import logger from loguru import logger
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -59,6 +59,38 @@ DOUYU_PROXY_TASK_TYPES = {
} }
_DOUYU_RESULT_DEBUG_KEYS = {
"raw",
"query_params",
"qq_app_scheme",
"role_param_url",
"game_role_bind_url",
}
def slim_douyu_task_result_for_storage(result: dict | None) -> dict | None:
"""去掉任务结果里的调试大字段,避免每次轮询重复写大 JSON。
运行器在内存中仍使用完整 result;这里只影响落库和 WS 展示。二维码 URL、
角色状态、积分、订单号等业务字段全部保留。
"""
if result is None:
return None
def clean(value: Any) -> Any:
if isinstance(value, dict):
return {
key: clean(item)
for key, item in value.items()
if key not in _DOUYU_RESULT_DEBUG_KEYS
}
if isinstance(value, list):
return [clean(item) for item in value]
return value
return clean(result)
class DouyuBatchRunnerCore: class DouyuBatchRunnerCore:
"""斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。""" """斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。"""
@@ -196,9 +228,7 @@ class DouyuBatchRunnerCore:
sync_task_from_binding_runner(db, douyu_task_id=int(task.id)) sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果 except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
logger.exception( logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
"[douyu] 同步直播账号履约状态失败: task_id={}", task.id
)
def _sync_platform_fulfillment_operation( def _sync_platform_fulfillment_operation(
self, self,
@@ -220,9 +250,7 @@ class DouyuBatchRunnerCore:
message=message, message=message,
) )
except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果 except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果
logger.exception( logger.exception("[douyu] 写入履约操作时间线失败: task_id={}", task.id)
"[douyu] 写入履约操作时间线失败: task_id={}", task.id
)
def _mark_task( def _mark_task(
self, self,
@@ -235,30 +263,28 @@ class DouyuBatchRunnerCore:
task.status = status task.status = status
task.message = message[:512] task.message = message[:512]
if result is not None: if result is not None:
task.result = result task.result = slim_douyu_task_result_for_storage(result)
task.finished_at = datetime.now(UTC) task.finished_at = datetime.now(UTC)
if task.account_id and status in {"success", "failed", "error", "stopped"}: account = (
from .business_events import record_douyu_task_result db.query(Account).filter(Account.id == task.account_id).first()
if task.account_id and status in {"success", "failed", "error", "stopped"}
account = ( else None
db.query(Account).filter(Account.id == task.account_id).first() )
) event_context = (
if account is not None: {
record_douyu_task_result( "task_id": int(task.id),
db, "task_type": str(task.task_type or ""),
account, "task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
task_id=int(task.id), str(task.task_type or ""),
task_type=str(task.task_type or ""), str(task.task_type or "斗鱼任务"),
task_label=SUPPORTED_DOUYU_TASK_TYPES.get( ),
str(task.task_type or ""), "scope": str(task.handbook_scope or ""),
str(task.task_type or "斗鱼任务"), "finished_at": task.finished_at,
), "created_by": int(task.created_by),
scope=str(task.handbook_scope or ""), }
status=status, if account is not None
message=message, else None
occurred_at=task.finished_at, )
actor_id=int(task.created_by),
)
self._sync_platform_fulfillment_operation( self._sync_platform_fulfillment_operation(
db, db,
task, task,
@@ -267,6 +293,28 @@ class DouyuBatchRunnerCore:
) )
db.commit() db.commit()
self._push_task_event(task) self._push_task_event(task)
if account is not None and event_context is not None:
try:
from .business_events import record_douyu_task_result
record_douyu_task_result(
db,
account,
task_id=event_context["task_id"],
task_type=event_context["task_type"],
task_label=event_context["task_label"],
scope=event_context["scope"],
status=status,
message=message,
occurred_at=event_context["finished_at"],
actor_id=event_context["created_by"],
)
db.commit()
except Exception: # noqa: BLE001 - 时间线旁路不能覆盖已提交的任务结果
db.rollback()
logger.exception(
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
)
self._sync_platform_fulfillment(db, task) self._sync_platform_fulfillment(db, task)
def _update_task_progress( def _update_task_progress(
@@ -280,7 +328,7 @@ class DouyuBatchRunnerCore:
task.status = status task.status = status
task.message = message[:512] task.message = message[:512]
if result is not None: if result is not None:
task.result = result task.result = slim_douyu_task_result_for_storage(result)
db.commit() db.commit()
self._push_task_event(task) self._push_task_event(task)
self._sync_platform_fulfillment(db, task) self._sync_platform_fulfillment(db, task)
+118 -43
View File
@@ -2,9 +2,11 @@
from __future__ import annotations from __future__ import annotations
import os
import uuid import uuid
from datetime import UTC, datetime from datetime import UTC, datetime
from loguru import logger
from sqlalchemy import and_, func, select from sqlalchemy import and_, func, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -146,10 +148,42 @@ DOUYU_ACTIVE_PAYMENT_TASK_TYPES = (
) )
def _env_positive_int(name: str, default: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return value if value > 0 else default
DOUYU_MAX_ACTIVE_TASKS = _env_positive_int("DOUYU_MAX_ACTIVE_TASKS", 250)
class DouyuActivePaymentError(ValueError): class DouyuActivePaymentError(ValueError):
"""同一账号已有未结束的充值任务。""" """同一账号已有未结束的充值任务。"""
class DouyuTaskBackpressureError(ValueError):
"""全局活跃斗鱼任务超过安全上限。"""
def _ensure_active_task_budget(db: Session, additional: int) -> None:
"""为任务创建提供全局背压,避免无限线程和连接同时挤入数据库。"""
if additional <= 0:
return
active = (
db.query(func.count(DouyuTask.id))
.filter(DouyuTask.status.in_(DOUYU_ACTIVE_TASK_STATUSES))
.scalar()
or 0
)
if active + additional > DOUYU_MAX_ACTIVE_TASKS:
raise DouyuTaskBackpressureError(
f"斗鱼任务繁忙:当前活跃 {active} 个,"
f"上限 {DOUYU_MAX_ACTIVE_TASKS} 个,请稍后重试"
)
def douyu_config_value(field: str, value): def douyu_config_value(field: str, value):
"""读取配置值;空值自动回退到默认值。""" """读取配置值;空值自动回退到默认值。"""
default = DOUYU_CONFIG_DEFAULTS[field] default = DOUYU_CONFIG_DEFAULTS[field]
@@ -348,6 +382,7 @@ def create_douyu_planned_tasks(
raise ValueError("该任务不属于当前工作台") raise ValueError("该任务不属于当前工作台")
accounts = visible_douyu_task_accounts(db, account_ids) accounts = visible_douyu_task_accounts(db, account_ids)
_ensure_active_task_budget(db, len(accounts))
_lock_payment_accounts(db, accounts, task_type) _lock_payment_accounts(db, accounts, task_type)
if ( if (
task_type in {"refresh_goods", "refresh_esports_goods", "refresh_xpd_goods"} task_type in {"refresh_goods", "refresh_esports_goods", "refresh_xpd_goods"}
@@ -375,33 +410,39 @@ def create_douyu_planned_tasks(
db.add(task) db.add(task)
task_rows.append((task, account)) task_rows.append((task, account))
db.flush() db.flush()
events: list[dict] = []
if task_rows: if task_rows:
label = SUPPORTED_DOUYU_TASK_TYPES[task_type] label = SUPPORTED_DOUYU_TASK_TYPES[task_type]
emit_bulk( events = [
db, {
[ "account_id": account.id,
{ "event_type": EVENT_TASK_STARTED,
"account_id": account.id, "title": f"{label}:任务已创建",
"event_type": EVENT_TASK_STARTED, "category": CATEGORY_ACTIVITY,
"title": f"{label}:任务已创建", "detail": {
"category": CATEGORY_ACTIVITY, "task_id": task.id,
"detail": { "task_type": task_type,
"task_id": task.id, "scope": handbook_scope,
"task_type": task_type, "status": "planned",
"scope": handbook_scope, },
"status": "planned", "source": "douyu_task",
}, "subject_type": "task",
"source": "douyu_task", "subject_key": str(task.id),
"subject_type": "task", "actor_type": "user",
"subject_key": str(task.id), "actor_id": created_by,
"actor_type": "user", "source_ref": f"douyu_task_started:{task.id}",
"actor_id": created_by, }
"source_ref": f"douyu_task_started:{task.id}", for task, account in task_rows
} ]
for task, account in task_rows # 任务行必须先独立提交。账号事件是旁路,不能在持有任务行锁时批量插入。
],
)
db.commit() db.commit()
if events:
try:
emit_bulk(db, events)
db.commit()
except Exception: # noqa: BLE001 - 时间线不能回滚已经提交的任务
db.rollback()
logger.exception("批量写入斗鱼任务创建事件失败")
return batch_id, len(accounts) return batch_id, len(accounts)
@@ -424,39 +465,73 @@ def cleanup_orphan_douyu_tasks(
if not tasks: if not tasks:
return 0 return 0
now = datetime.now(UTC) now = datetime.now(UTC)
from .business_events import record_douyu_task_result account_ids = {int(task.account_id) for task in tasks if task.account_id}
accounts = (
{
account.id: account
for account in db.query(Account).filter(Account.id.in_(account_ids)).all()
}
if account_ids
else {}
)
event_contexts: list[tuple[dict, Account]] = []
platform_task_ids: list[int] = []
for task in tasks: for task in tasks:
task.status = "stopped" task.status = "stopped"
task.message = message task.message = message
task.finished_at = now task.finished_at = now
if task.account_id: account = accounts.get(int(task.account_id)) if task.account_id else None
account = db.query(Account).filter(Account.id == task.account_id).first() if account is not None:
if account is not None: event_contexts.append(
record_douyu_task_result( (
db, {
"task_id": int(task.id),
"task_type": str(task.task_type or ""),
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
str(task.task_type or ""),
str(task.task_type or "斗鱼任务"),
),
"scope": str(task.handbook_scope or ""),
},
account, account,
task_id=int(task.id),
task_type=str(task.task_type or ""),
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
str(task.task_type or ""),
str(task.task_type or "斗鱼任务"),
),
scope=str(task.handbook_scope or ""),
status="stopped",
message=message,
occurred_at=now,
) )
)
if task.platform_task_id: if task.platform_task_id:
from .platform_fulfillment.service import record_douyu_operation_result platform_task_ids.append(int(task.id))
# 先提交任务终态,再写账号时间线,避免 account_events 与 douyu_tasks
# 在并发创建和结束任务时形成反向锁依赖。
db.commit()
from .business_events import record_douyu_task_result
for context, account in event_contexts:
record_douyu_task_result(
db,
account,
task_id=context["task_id"],
task_type=context["task_type"],
task_label=context["task_label"],
scope=context["scope"],
status="stopped",
message=message,
occurred_at=now,
)
db.commit()
if platform_task_ids:
from .platform_fulfillment.service import record_douyu_operation_result
for task_id in platform_task_ids:
task = db.get(DouyuTask, task_id)
if task is None:
continue
record_douyu_operation_result( record_douyu_operation_result(
db, db,
task, task,
status="stopped", status="stopped",
message=message, message=message,
) )
db.commit() db.commit()
return len(tasks) return len(tasks)
+4 -1
View File
@@ -138,7 +138,10 @@ def _mark_attempt_failed(
失败重试会新建记录或新 attempt,因此是新的一条 —— 这是对的,那是新的事实。 失败重试会新建记录或新 attempt,因此是新的一条 —— 这是对的,那是新的事实。
""" """
completed_at = datetime.now(UTC) completed_at = datetime.now(UTC)
attempt_at = task.attempt_at or completed_at # MySQL DATETIME(0) 会按毫秒四舍五入到整秒。若 source_ref 使用内存里
# 带微秒的原值,第一次写 13:57:13.999 会落成 13:57:14;第二次重读后
# 幂等键变成下一秒,唯一索引就无法拦住重复事件。先截断到整秒再统一使用。
attempt_at = task.attempt_at or completed_at.replace(microsecond=0)
attempt_utc = ( attempt_utc = (
attempt_at.replace(tzinfo=UTC) attempt_at.replace(tzinfo=UTC)
if attempt_at.tzinfo is None if attempt_at.tzinfo is None