perf(douyu): 消除任务锁竞争并增加创建背压
This commit is contained in:
@@ -68,6 +68,12 @@ DB_POOL_SIZE=20
|
|||||||
DB_MAX_OVERFLOW=20
|
DB_MAX_OVERFLOW=20
|
||||||
DB_POOL_TIMEOUT=30
|
DB_POOL_TIMEOUT=30
|
||||||
DB_POOL_RECYCLE=1800
|
DB_POOL_RECYCLE=1800
|
||||||
|
# 行锁 10 秒仍未拿到就失败,避免把连接池和请求线程长期堵住。
|
||||||
|
DB_LOCK_WAIT_TIMEOUT=10
|
||||||
|
# 斗鱼批次创建并发与全局活跃量背压。
|
||||||
|
DOUYU_BATCH_CREATE_CONCURRENCY=2
|
||||||
|
DOUYU_MAX_ACTIVE_BATCHES=20
|
||||||
|
DOUYU_MAX_ACTIVE_TASKS=250
|
||||||
|
|
||||||
# Cookie 安全标志(生产环境 HTTPS 部署时设为 true)
|
# Cookie 安全标志(生产环境 HTTPS 部署时设为 true)
|
||||||
COOKIE_SECURE=false
|
COOKIE_SECURE=false
|
||||||
|
|||||||
@@ -40,6 +40,10 @@ services:
|
|||||||
- DB_MAX_OVERFLOW=${DB_MAX_OVERFLOW:-20}
|
- DB_MAX_OVERFLOW=${DB_MAX_OVERFLOW:-20}
|
||||||
- DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30}
|
- DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30}
|
||||||
- DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-1800}
|
- DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-1800}
|
||||||
|
- DB_LOCK_WAIT_TIMEOUT=${DB_LOCK_WAIT_TIMEOUT:-10}
|
||||||
|
- DOUYU_BATCH_CREATE_CONCURRENCY=${DOUYU_BATCH_CREATE_CONCURRENCY:-2}
|
||||||
|
- DOUYU_MAX_ACTIVE_BATCHES=${DOUYU_MAX_ACTIVE_BATCHES:-20}
|
||||||
|
- DOUYU_MAX_ACTIVE_TASKS=${DOUYU_MAX_ACTIVE_TASKS:-250}
|
||||||
# JWT 密钥(生产环境务必修改,可用 python -c "import secrets; print(secrets.token_urlsafe(32))" 生成)
|
# JWT 密钥(生产环境务必修改,可用 python -c "import secrets; print(secrets.token_urlsafe(32))" 生成)
|
||||||
- JWT_SECRET_KEY=${JWT_SECRET_KEY:-}
|
- JWT_SECRET_KEY=${JWT_SECRET_KEY:-}
|
||||||
# 后台登录滑动会话:24h 空闲过期,7d 绝对上限。
|
# 后台登录滑动会话:24h 空闲过期,7d 绝对上限。
|
||||||
|
|||||||
@@ -0,0 +1,227 @@
|
|||||||
|
import inspect
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from tests.support.db import make_test_session
|
||||||
|
from web.backend.database import engine as app_engine
|
||||||
|
from web.backend.main import health
|
||||||
|
from web.backend.models import Account, DouyuTask, User
|
||||||
|
from web.backend.routers import douyu as douyu_router
|
||||||
|
from web.backend.services import business_events, douyu_service
|
||||||
|
from web.backend.services.douyu_runner_core import (
|
||||||
|
DouyuBatchRunnerCore,
|
||||||
|
slim_douyu_task_result_for_storage,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _fixtures(db, *, status: str = "planned") -> tuple[User, Account, DouyuTask]:
|
||||||
|
user = User(username="concurrency-user", password_hash="hash", role="operation")
|
||||||
|
db.add(user)
|
||||||
|
db.flush()
|
||||||
|
account = Account(
|
||||||
|
username="concurrency-account",
|
||||||
|
password="password",
|
||||||
|
email="concurrency@example.com",
|
||||||
|
email_password="mail-password",
|
||||||
|
cookie="acf_uid=1",
|
||||||
|
)
|
||||||
|
db.add(account)
|
||||||
|
db.flush()
|
||||||
|
task = DouyuTask(
|
||||||
|
batch_id="concurrency-batch",
|
||||||
|
account_id=account.id,
|
||||||
|
account_assigned_to=user.id,
|
||||||
|
task_type="get_bind_qr",
|
||||||
|
handbook_scope="elite",
|
||||||
|
status=status,
|
||||||
|
created_by=user.id,
|
||||||
|
)
|
||||||
|
db.add(task)
|
||||||
|
db.commit()
|
||||||
|
return user, account, task
|
||||||
|
|
||||||
|
|
||||||
|
def test_task_rows_commit_before_started_events(db, monkeypatch):
|
||||||
|
user = User(username="event-order", password_hash="hash", role="operation")
|
||||||
|
db.add(user)
|
||||||
|
db.flush()
|
||||||
|
account = Account(
|
||||||
|
username="event-order-account",
|
||||||
|
password="password",
|
||||||
|
email="event-order@example.com",
|
||||||
|
email_password="mail-password",
|
||||||
|
cookie="acf_uid=2",
|
||||||
|
)
|
||||||
|
db.add(account)
|
||||||
|
db.commit()
|
||||||
|
observed: list[int] = []
|
||||||
|
|
||||||
|
def fake_emit_bulk_read_committed(_db, events):
|
||||||
|
other = make_test_session()
|
||||||
|
try:
|
||||||
|
observed.append(
|
||||||
|
other.query(DouyuTask).filter(DouyuTask.status == "planned").count()
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
other.close()
|
||||||
|
return len(events)
|
||||||
|
|
||||||
|
monkeypatch.setattr(douyu_service, "emit_bulk", fake_emit_bulk_read_committed)
|
||||||
|
batch_id, count = douyu_service.create_douyu_planned_tasks(
|
||||||
|
db,
|
||||||
|
[account.id],
|
||||||
|
"query_points",
|
||||||
|
"elite",
|
||||||
|
user.id,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert batch_id
|
||||||
|
assert count == 1
|
||||||
|
assert observed == [1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_mark_task_commits_result_before_account_event(db, monkeypatch):
|
||||||
|
_, account, task = _fixtures(db, status="running")
|
||||||
|
task.account = account
|
||||||
|
runner = DouyuBatchRunnerCore(
|
||||||
|
db,
|
||||||
|
task.batch_id,
|
||||||
|
task.task_type,
|
||||||
|
close_db=False,
|
||||||
|
)
|
||||||
|
observed: list[tuple[str, object]] = []
|
||||||
|
|
||||||
|
def fake_record(_db, _account, **kwargs):
|
||||||
|
other = make_test_session()
|
||||||
|
try:
|
||||||
|
row = other.get(DouyuTask, task.id)
|
||||||
|
observed.append((row.status, row.result))
|
||||||
|
finally:
|
||||||
|
other.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
|
||||||
|
runner._mark_task(
|
||||||
|
db,
|
||||||
|
task,
|
||||||
|
"success",
|
||||||
|
"done",
|
||||||
|
{
|
||||||
|
"url": "https://example.test/qr",
|
||||||
|
"raw": {"large": "payload"},
|
||||||
|
"before_bind_info": {
|
||||||
|
"role_name": "role-a",
|
||||||
|
"raw": {"large": "nested"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert observed == [
|
||||||
|
(
|
||||||
|
"success",
|
||||||
|
{
|
||||||
|
"url": "https://example.test/qr",
|
||||||
|
"before_bind_info": {"role_name": "role-a"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_commits_stopped_status_before_account_event(db, monkeypatch):
|
||||||
|
_, _, task = _fixtures(db, status="running")
|
||||||
|
observed: list[str] = []
|
||||||
|
|
||||||
|
def fake_record(_db, _account, **kwargs):
|
||||||
|
other = make_test_session()
|
||||||
|
try:
|
||||||
|
observed.append(other.get(DouyuTask, task.id).status)
|
||||||
|
finally:
|
||||||
|
other.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
|
||||||
|
cleaned = douyu_service.cleanup_orphan_douyu_tasks(
|
||||||
|
db,
|
||||||
|
active_batch_ids=set(),
|
||||||
|
statuses=("running",),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cleaned == 1
|
||||||
|
assert observed == ["stopped"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_active_task_budget_rejects_new_batch(db, monkeypatch):
|
||||||
|
user = User(username="budget-user", password_hash="hash", role="operation")
|
||||||
|
db.add(user)
|
||||||
|
db.flush()
|
||||||
|
account = Account(
|
||||||
|
username="budget-account",
|
||||||
|
password="password",
|
||||||
|
email="budget@example.com",
|
||||||
|
email_password="mail-password",
|
||||||
|
cookie="acf_uid=3",
|
||||||
|
)
|
||||||
|
db.add(account)
|
||||||
|
db.commit()
|
||||||
|
monkeypatch.setattr(douyu_service, "DOUYU_MAX_ACTIVE_TASKS", 1)
|
||||||
|
|
||||||
|
douyu_service.create_douyu_planned_tasks(
|
||||||
|
db,
|
||||||
|
[account.id],
|
||||||
|
"query_points",
|
||||||
|
"elite",
|
||||||
|
user.id,
|
||||||
|
)
|
||||||
|
with pytest.raises(douyu_service.DouyuTaskBackpressureError, match="上限 1"):
|
||||||
|
douyu_service.create_douyu_planned_tasks(
|
||||||
|
db,
|
||||||
|
[account.id],
|
||||||
|
"query_game_name",
|
||||||
|
"elite",
|
||||||
|
user.id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_active_batch_budget_returns_429(monkeypatch):
|
||||||
|
monkeypatch.setattr(douyu_router, "DOUYU_MAX_ACTIVE_BATCHES", 1)
|
||||||
|
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
douyu_router._ensure_active_batch_budget(1)
|
||||||
|
|
||||||
|
assert exc_info.value.status_code == 429
|
||||||
|
|
||||||
|
|
||||||
|
def test_slim_task_result_removes_debug_payloads():
|
||||||
|
result = slim_douyu_task_result_for_storage(
|
||||||
|
{
|
||||||
|
"url": "https://example.test/qr",
|
||||||
|
"raw": {"debug": True},
|
||||||
|
"query_params": {"token": "secret"},
|
||||||
|
"before_bind_info": {
|
||||||
|
"role_name": "role-a",
|
||||||
|
"raw": {"debug": True},
|
||||||
|
},
|
||||||
|
"items": [{"name": "kept", "raw": {"debug": True}}],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
"url": "https://example.test/qr",
|
||||||
|
"before_bind_info": {"role_name": "role-a"},
|
||||||
|
"items": [{"name": "kept"}],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_health_endpoint_does_not_use_db_threadpool():
|
||||||
|
assert inspect.iscoroutinefunction(health)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mysql_lock_wait_timeout_is_bounded():
|
||||||
|
with app_engine.connect() as connection:
|
||||||
|
value = connection.execute(
|
||||||
|
text("SELECT @@SESSION.innodb_lock_wait_timeout")
|
||||||
|
).scalar()
|
||||||
|
assert value is not None
|
||||||
|
assert int(value) <= 10
|
||||||
@@ -142,7 +142,11 @@ class TestLoginFailureEvents(_Base):
|
|||||||
self.session.commit()
|
self.session.commit()
|
||||||
_mark_attempt_failed(task, "极验验证失败: slide")
|
_mark_attempt_failed(task, "极验验证失败: slide")
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
assert len(self.events(account.id)) == 1
|
events = self.events(account.id)
|
||||||
|
assert len(events) == 1, [
|
||||||
|
(event.source_ref, event.occurred_at) for event in events
|
||||||
|
]
|
||||||
|
assert events[0].occurred_at.microsecond == 0
|
||||||
|
|
||||||
def test_event_does_not_break_when_task_has_no_session(self):
|
def test_event_does_not_break_when_task_has_no_session(self):
|
||||||
"""纯内存构造的任务(没有 session)不能因为写事件而炸掉主流程。"""
|
"""纯内存构造的任务(没有 session)不能因为写事件而炸掉主流程。"""
|
||||||
|
|||||||
@@ -82,6 +82,19 @@ if DATABASE_URL.startswith("sqlite"):
|
|||||||
cursor.execute("PRAGMA busy_timeout=30000")
|
cursor.execute("PRAGMA busy_timeout=30000")
|
||||||
cursor.execute("PRAGMA foreign_keys=ON")
|
cursor.execute("PRAGMA foreign_keys=ON")
|
||||||
cursor.close()
|
cursor.close()
|
||||||
|
else:
|
||||||
|
|
||||||
|
@event.listens_for(engine, "connect")
|
||||||
|
def _set_mysql_session_guards(dbapi_connection, connection_record):
|
||||||
|
"""让行锁等待快速失败,避免单个事务把连接池堵住几十秒。"""
|
||||||
|
cursor = dbapi_connection.cursor()
|
||||||
|
try:
|
||||||
|
cursor.execute(
|
||||||
|
"SET SESSION innodb_lock_wait_timeout = "
|
||||||
|
f"{_env_int('DB_LOCK_WAIT_TIMEOUT', 10)}"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
cursor.close()
|
||||||
|
|
||||||
|
|
||||||
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)
|
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)
|
||||||
|
|||||||
+1
-1
@@ -201,7 +201,7 @@ app.include_router(files.router)
|
|||||||
|
|
||||||
|
|
||||||
@app.get("/api/health")
|
@app.get("/api/health")
|
||||||
def health():
|
async def health():
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
|
from weakref import WeakKeyDictionary
|
||||||
|
|
||||||
from fastapi import (
|
from fastapi import (
|
||||||
APIRouter,
|
APIRouter,
|
||||||
@@ -18,6 +19,7 @@ from fastapi import (
|
|||||||
from loguru import logger
|
from loguru import logger
|
||||||
from sqlalchemy import case, func, or_, select
|
from sqlalchemy import case, func, or_, select
|
||||||
from sqlalchemy.orm import Session, joinedload
|
from sqlalchemy.orm import Session, joinedload
|
||||||
|
from starlette.concurrency import run_in_threadpool
|
||||||
|
|
||||||
from core.douyu import (
|
from core.douyu import (
|
||||||
DouyuActivityError,
|
DouyuActivityError,
|
||||||
@@ -56,15 +58,20 @@ from ..services.account_binding import bind_payload
|
|||||||
from ..services.audit_service import record_audit
|
from ..services.audit_service import record_audit
|
||||||
from ..services.business_events import record_recharge
|
from ..services.business_events import record_recharge
|
||||||
from ..services.douyu_activity_query import activity_client, query_activity_goods
|
from ..services.douyu_activity_query import activity_client, query_activity_goods
|
||||||
from ..services.douyu_batch_launcher import start_douyu_batch
|
from ..services.douyu_batch_launcher import (
|
||||||
|
DOUYU_BATCH_CREATE_CONCURRENCY,
|
||||||
|
DOUYU_MAX_ACTIVE_BATCHES,
|
||||||
|
start_douyu_batch,
|
||||||
|
)
|
||||||
from ..services.douyu_goods_service import serialize_goods
|
from ..services.douyu_goods_service import serialize_goods
|
||||||
from ..services.douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
from ..services.douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||||
from ..services.douyu_service import (
|
from ..services.douyu_service import (
|
||||||
|
DOUYU_ACTIVE_TASK_STATUSES,
|
||||||
DOUYU_CONFIG_FIELDS,
|
DOUYU_CONFIG_FIELDS,
|
||||||
DOUYU_HANDBOOK_TASK_TYPES,
|
DOUYU_HANDBOOK_TASK_TYPES,
|
||||||
DOUYU_ACTIVE_TASK_STATUSES,
|
|
||||||
SUPPORTED_DOUYU_TASK_TYPES,
|
SUPPORTED_DOUYU_TASK_TYPES,
|
||||||
DouyuActivePaymentError,
|
DouyuActivePaymentError,
|
||||||
|
DouyuTaskBackpressureError,
|
||||||
apply_douyu_config_defaults,
|
apply_douyu_config_defaults,
|
||||||
cleanup_orphan_douyu_tasks,
|
cleanup_orphan_douyu_tasks,
|
||||||
cookie_holding_filter,
|
cookie_holding_filter,
|
||||||
@@ -77,6 +84,33 @@ from ..services.platform_fulfillment import service as platform_fulfillment_serv
|
|||||||
|
|
||||||
router = APIRouter(prefix="/api/douyu", tags=["斗鱼活动"])
|
router = APIRouter(prefix="/api/douyu", tags=["斗鱼活动"])
|
||||||
|
|
||||||
|
_BATCH_CREATE_SEMAPHORES: WeakKeyDictionary[
|
||||||
|
asyncio.AbstractEventLoop,
|
||||||
|
asyncio.Semaphore,
|
||||||
|
] = WeakKeyDictionary()
|
||||||
|
|
||||||
|
|
||||||
|
def _batch_create_semaphore() -> asyncio.Semaphore:
|
||||||
|
"""按事件循环复用创建锁,避免测试和开发热重载跨 loop 复用 Semaphore。"""
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
semaphore = _BATCH_CREATE_SEMAPHORES.get(loop)
|
||||||
|
if semaphore is None:
|
||||||
|
semaphore = asyncio.Semaphore(DOUYU_BATCH_CREATE_CONCURRENCY)
|
||||||
|
_BATCH_CREATE_SEMAPHORES[loop] = semaphore
|
||||||
|
return semaphore
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_active_batch_budget(active_count: int) -> None:
|
||||||
|
if active_count >= DOUYU_MAX_ACTIVE_BATCHES:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=429,
|
||||||
|
detail=(
|
||||||
|
f"斗鱼任务繁忙:当前有 {active_count} 个批次运行中,"
|
||||||
|
f"上限 {DOUYU_MAX_ACTIVE_BATCHES} 个,请稍后重试"
|
||||||
|
),
|
||||||
|
headers={"Retry-After": "5"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _supplier_response_value(payload: dict, *keys: str):
|
def _supplier_response_value(payload: dict, *keys: str):
|
||||||
"""兼容供应商把订单字段放在根节点、data 或 result 中。"""
|
"""兼容供应商把订单字段放在根节点、data 或 result 中。"""
|
||||||
@@ -782,13 +816,12 @@ def list_xpd_goods(
|
|||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@router.post("/tasks/batch")
|
def _prepare_task_batch(
|
||||||
async def create_task_batch(
|
|
||||||
req: DouyuTaskBatchRequest,
|
req: DouyuTaskBatchRequest,
|
||||||
db: Session = Depends(get_db),
|
db: Session,
|
||||||
current: User = Depends(require_permission("douyu:task")),
|
current: User,
|
||||||
):
|
) -> tuple[str, int]:
|
||||||
"""创建斗鱼任务记录并启动后台执行器。"""
|
"""在线程池中完成全部同步 DB 工作,避免阻塞 Uvicorn 事件循环。"""
|
||||||
if not req.account_ids:
|
if not req.account_ids:
|
||||||
raise HTTPException(status_code=400, detail="请选择斗鱼账号")
|
raise HTTPException(status_code=400, detail="请选择斗鱼账号")
|
||||||
_require_task_account_access(
|
_require_task_account_access(
|
||||||
@@ -818,12 +851,14 @@ async def create_task_batch(
|
|||||||
)
|
)
|
||||||
platform_task = db.get(PlatformFulfillmentTask, int(platform_task.id))
|
platform_task = db.get(PlatformFulfillmentTask, int(platform_task.id))
|
||||||
|
|
||||||
|
active_batch_ids = douyu_batch_registry.active_ids()
|
||||||
cleanup_orphan_douyu_tasks(
|
cleanup_orphan_douyu_tasks(
|
||||||
db,
|
db,
|
||||||
active_batch_ids=douyu_batch_registry.active_ids(),
|
active_batch_ids=active_batch_ids,
|
||||||
statuses=DOUYU_ACTIVE_TASK_STATUSES,
|
statuses=DOUYU_ACTIVE_TASK_STATUSES,
|
||||||
message="任务已中断(无执行器接管)",
|
message="任务已中断(无执行器接管)",
|
||||||
)
|
)
|
||||||
|
_ensure_active_batch_budget(len(active_batch_ids))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
batch_id, count = create_douyu_planned_tasks(
|
batch_id, count = create_douyu_planned_tasks(
|
||||||
@@ -837,6 +872,12 @@ async def create_task_batch(
|
|||||||
)
|
)
|
||||||
except DouyuActivePaymentError as exc:
|
except DouyuActivePaymentError as exc:
|
||||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||||
|
except DouyuTaskBackpressureError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=429,
|
||||||
|
detail=str(exc),
|
||||||
|
headers={"Retry-After": "5"},
|
||||||
|
) from exc
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||||
if count == 0:
|
if count == 0:
|
||||||
@@ -911,12 +952,33 @@ async def create_task_batch(
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
start_douyu_batch(
|
return batch_id, count
|
||||||
batch_id=batch_id,
|
|
||||||
task_type=req.task_type,
|
|
||||||
payload=req.payload,
|
@router.post("/tasks/batch")
|
||||||
concurrency=req.concurrency,
|
async def create_task_batch(
|
||||||
)
|
req: DouyuTaskBatchRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current: User = Depends(require_permission("douyu:task")),
|
||||||
|
):
|
||||||
|
"""创建斗鱼任务记录并启动后台执行器。"""
|
||||||
|
# 同步 SQLAlchemy 段必须在线程池执行;事件循环只负责最后启动后台线程。
|
||||||
|
async with _batch_create_semaphore():
|
||||||
|
batch_id, count = await run_in_threadpool(
|
||||||
|
_prepare_task_batch,
|
||||||
|
req,
|
||||||
|
db,
|
||||||
|
current,
|
||||||
|
)
|
||||||
|
await run_in_threadpool(
|
||||||
|
start_douyu_batch,
|
||||||
|
batch_id=batch_id,
|
||||||
|
task_type=req.task_type,
|
||||||
|
payload=req.payload,
|
||||||
|
concurrency=req.concurrency,
|
||||||
|
loop=asyncio.get_running_loop(),
|
||||||
|
log_queue=asyncio.Queue(),
|
||||||
|
)
|
||||||
|
|
||||||
return {"batch_id": batch_id, "count": count, "success": True}
|
return {"batch_id": batch_id, "count": count, "success": True}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import os
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
from web.backend.database import SessionLocal
|
from web.backend.database import SessionLocal
|
||||||
@@ -13,16 +14,33 @@ from web.backend.database import SessionLocal
|
|||||||
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||||
|
|
||||||
|
|
||||||
|
def _env_positive_int(name: str, default: int) -> int:
|
||||||
|
try:
|
||||||
|
value = int(os.getenv(name, str(default)))
|
||||||
|
except ValueError:
|
||||||
|
return default
|
||||||
|
return value if value > 0 else default
|
||||||
|
|
||||||
|
|
||||||
|
DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
|
||||||
|
"DOUYU_BATCH_CREATE_CONCURRENCY",
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
|
||||||
|
|
||||||
|
|
||||||
def start_douyu_batch(
|
def start_douyu_batch(
|
||||||
*,
|
*,
|
||||||
batch_id: str,
|
batch_id: str,
|
||||||
task_type: str,
|
task_type: str,
|
||||||
payload: dict,
|
payload: dict,
|
||||||
concurrency: int,
|
concurrency: int,
|
||||||
|
loop: asyncio.AbstractEventLoop | None = None,
|
||||||
|
log_queue: asyncio.Queue | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""必须在运行中的 asyncio 事件循环里调用。"""
|
"""创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。"""
|
||||||
log_queue = asyncio.Queue()
|
loop = loop or asyncio.get_running_loop()
|
||||||
loop = asyncio.get_running_loop()
|
log_queue = log_queue or asyncio.Queue()
|
||||||
thread_db = SessionLocal()
|
thread_db = SessionLocal()
|
||||||
runner = DouyuBatchRunner(
|
runner = DouyuBatchRunner(
|
||||||
db=thread_db,
|
db=thread_db,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import asyncio
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -59,6 +59,38 @@ DOUYU_PROXY_TASK_TYPES = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_DOUYU_RESULT_DEBUG_KEYS = {
|
||||||
|
"raw",
|
||||||
|
"query_params",
|
||||||
|
"qq_app_scheme",
|
||||||
|
"role_param_url",
|
||||||
|
"game_role_bind_url",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def slim_douyu_task_result_for_storage(result: dict | None) -> dict | None:
|
||||||
|
"""去掉任务结果里的调试大字段,避免每次轮询重复写大 JSON。
|
||||||
|
|
||||||
|
运行器在内存中仍使用完整 result;这里只影响落库和 WS 展示。二维码 URL、
|
||||||
|
角色状态、积分、订单号等业务字段全部保留。
|
||||||
|
"""
|
||||||
|
if result is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def clean(value: Any) -> Any:
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {
|
||||||
|
key: clean(item)
|
||||||
|
for key, item in value.items()
|
||||||
|
if key not in _DOUYU_RESULT_DEBUG_KEYS
|
||||||
|
}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [clean(item) for item in value]
|
||||||
|
return value
|
||||||
|
|
||||||
|
return clean(result)
|
||||||
|
|
||||||
|
|
||||||
class DouyuBatchRunnerCore:
|
class DouyuBatchRunnerCore:
|
||||||
"""斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。"""
|
"""斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。"""
|
||||||
|
|
||||||
@@ -196,9 +228,7 @@ class DouyuBatchRunnerCore:
|
|||||||
|
|
||||||
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
|
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
|
||||||
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
|
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
|
||||||
logger.exception(
|
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
|
||||||
"[douyu] 同步直播账号履约状态失败: task_id={}", task.id
|
|
||||||
)
|
|
||||||
|
|
||||||
def _sync_platform_fulfillment_operation(
|
def _sync_platform_fulfillment_operation(
|
||||||
self,
|
self,
|
||||||
@@ -220,9 +250,7 @@ class DouyuBatchRunnerCore:
|
|||||||
message=message,
|
message=message,
|
||||||
)
|
)
|
||||||
except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果
|
except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果
|
||||||
logger.exception(
|
logger.exception("[douyu] 写入履约操作时间线失败: task_id={}", task.id)
|
||||||
"[douyu] 写入履约操作时间线失败: task_id={}", task.id
|
|
||||||
)
|
|
||||||
|
|
||||||
def _mark_task(
|
def _mark_task(
|
||||||
self,
|
self,
|
||||||
@@ -235,30 +263,28 @@ class DouyuBatchRunnerCore:
|
|||||||
task.status = status
|
task.status = status
|
||||||
task.message = message[:512]
|
task.message = message[:512]
|
||||||
if result is not None:
|
if result is not None:
|
||||||
task.result = result
|
task.result = slim_douyu_task_result_for_storage(result)
|
||||||
task.finished_at = datetime.now(UTC)
|
task.finished_at = datetime.now(UTC)
|
||||||
if task.account_id and status in {"success", "failed", "error", "stopped"}:
|
account = (
|
||||||
from .business_events import record_douyu_task_result
|
db.query(Account).filter(Account.id == task.account_id).first()
|
||||||
|
if task.account_id and status in {"success", "failed", "error", "stopped"}
|
||||||
account = (
|
else None
|
||||||
db.query(Account).filter(Account.id == task.account_id).first()
|
)
|
||||||
)
|
event_context = (
|
||||||
if account is not None:
|
{
|
||||||
record_douyu_task_result(
|
"task_id": int(task.id),
|
||||||
db,
|
"task_type": str(task.task_type or ""),
|
||||||
account,
|
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
|
||||||
task_id=int(task.id),
|
str(task.task_type or ""),
|
||||||
task_type=str(task.task_type or ""),
|
str(task.task_type or "斗鱼任务"),
|
||||||
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
|
),
|
||||||
str(task.task_type or ""),
|
"scope": str(task.handbook_scope or ""),
|
||||||
str(task.task_type or "斗鱼任务"),
|
"finished_at": task.finished_at,
|
||||||
),
|
"created_by": int(task.created_by),
|
||||||
scope=str(task.handbook_scope or ""),
|
}
|
||||||
status=status,
|
if account is not None
|
||||||
message=message,
|
else None
|
||||||
occurred_at=task.finished_at,
|
)
|
||||||
actor_id=int(task.created_by),
|
|
||||||
)
|
|
||||||
self._sync_platform_fulfillment_operation(
|
self._sync_platform_fulfillment_operation(
|
||||||
db,
|
db,
|
||||||
task,
|
task,
|
||||||
@@ -267,6 +293,28 @@ class DouyuBatchRunnerCore:
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
self._push_task_event(task)
|
self._push_task_event(task)
|
||||||
|
if account is not None and event_context is not None:
|
||||||
|
try:
|
||||||
|
from .business_events import record_douyu_task_result
|
||||||
|
|
||||||
|
record_douyu_task_result(
|
||||||
|
db,
|
||||||
|
account,
|
||||||
|
task_id=event_context["task_id"],
|
||||||
|
task_type=event_context["task_type"],
|
||||||
|
task_label=event_context["task_label"],
|
||||||
|
scope=event_context["scope"],
|
||||||
|
status=status,
|
||||||
|
message=message,
|
||||||
|
occurred_at=event_context["finished_at"],
|
||||||
|
actor_id=event_context["created_by"],
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
except Exception: # noqa: BLE001 - 时间线旁路不能覆盖已提交的任务结果
|
||||||
|
db.rollback()
|
||||||
|
logger.exception(
|
||||||
|
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
|
||||||
|
)
|
||||||
self._sync_platform_fulfillment(db, task)
|
self._sync_platform_fulfillment(db, task)
|
||||||
|
|
||||||
def _update_task_progress(
|
def _update_task_progress(
|
||||||
@@ -280,7 +328,7 @@ class DouyuBatchRunnerCore:
|
|||||||
task.status = status
|
task.status = status
|
||||||
task.message = message[:512]
|
task.message = message[:512]
|
||||||
if result is not None:
|
if result is not None:
|
||||||
task.result = result
|
task.result = slim_douyu_task_result_for_storage(result)
|
||||||
db.commit()
|
db.commit()
|
||||||
self._push_task_event(task)
|
self._push_task_event(task)
|
||||||
self._sync_platform_fulfillment(db, task)
|
self._sync_platform_fulfillment(db, task)
|
||||||
|
|||||||
@@ -2,9 +2,11 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
from sqlalchemy import and_, func, select
|
from sqlalchemy import and_, func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -146,10 +148,42 @@ DOUYU_ACTIVE_PAYMENT_TASK_TYPES = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _env_positive_int(name: str, default: int) -> int:
|
||||||
|
try:
|
||||||
|
value = int(os.getenv(name, str(default)))
|
||||||
|
except ValueError:
|
||||||
|
return default
|
||||||
|
return value if value > 0 else default
|
||||||
|
|
||||||
|
|
||||||
|
DOUYU_MAX_ACTIVE_TASKS = _env_positive_int("DOUYU_MAX_ACTIVE_TASKS", 250)
|
||||||
|
|
||||||
|
|
||||||
class DouyuActivePaymentError(ValueError):
|
class DouyuActivePaymentError(ValueError):
|
||||||
"""同一账号已有未结束的充值任务。"""
|
"""同一账号已有未结束的充值任务。"""
|
||||||
|
|
||||||
|
|
||||||
|
class DouyuTaskBackpressureError(ValueError):
|
||||||
|
"""全局活跃斗鱼任务超过安全上限。"""
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_active_task_budget(db: Session, additional: int) -> None:
|
||||||
|
"""为任务创建提供全局背压,避免无限线程和连接同时挤入数据库。"""
|
||||||
|
if additional <= 0:
|
||||||
|
return
|
||||||
|
active = (
|
||||||
|
db.query(func.count(DouyuTask.id))
|
||||||
|
.filter(DouyuTask.status.in_(DOUYU_ACTIVE_TASK_STATUSES))
|
||||||
|
.scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
if active + additional > DOUYU_MAX_ACTIVE_TASKS:
|
||||||
|
raise DouyuTaskBackpressureError(
|
||||||
|
f"斗鱼任务繁忙:当前活跃 {active} 个,"
|
||||||
|
f"上限 {DOUYU_MAX_ACTIVE_TASKS} 个,请稍后重试"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def douyu_config_value(field: str, value):
|
def douyu_config_value(field: str, value):
|
||||||
"""读取配置值;空值自动回退到默认值。"""
|
"""读取配置值;空值自动回退到默认值。"""
|
||||||
default = DOUYU_CONFIG_DEFAULTS[field]
|
default = DOUYU_CONFIG_DEFAULTS[field]
|
||||||
@@ -348,6 +382,7 @@ def create_douyu_planned_tasks(
|
|||||||
raise ValueError("该任务不属于当前工作台")
|
raise ValueError("该任务不属于当前工作台")
|
||||||
|
|
||||||
accounts = visible_douyu_task_accounts(db, account_ids)
|
accounts = visible_douyu_task_accounts(db, account_ids)
|
||||||
|
_ensure_active_task_budget(db, len(accounts))
|
||||||
_lock_payment_accounts(db, accounts, task_type)
|
_lock_payment_accounts(db, accounts, task_type)
|
||||||
if (
|
if (
|
||||||
task_type in {"refresh_goods", "refresh_esports_goods", "refresh_xpd_goods"}
|
task_type in {"refresh_goods", "refresh_esports_goods", "refresh_xpd_goods"}
|
||||||
@@ -375,33 +410,39 @@ def create_douyu_planned_tasks(
|
|||||||
db.add(task)
|
db.add(task)
|
||||||
task_rows.append((task, account))
|
task_rows.append((task, account))
|
||||||
db.flush()
|
db.flush()
|
||||||
|
events: list[dict] = []
|
||||||
if task_rows:
|
if task_rows:
|
||||||
label = SUPPORTED_DOUYU_TASK_TYPES[task_type]
|
label = SUPPORTED_DOUYU_TASK_TYPES[task_type]
|
||||||
emit_bulk(
|
events = [
|
||||||
db,
|
{
|
||||||
[
|
"account_id": account.id,
|
||||||
{
|
"event_type": EVENT_TASK_STARTED,
|
||||||
"account_id": account.id,
|
"title": f"{label}:任务已创建",
|
||||||
"event_type": EVENT_TASK_STARTED,
|
"category": CATEGORY_ACTIVITY,
|
||||||
"title": f"{label}:任务已创建",
|
"detail": {
|
||||||
"category": CATEGORY_ACTIVITY,
|
"task_id": task.id,
|
||||||
"detail": {
|
"task_type": task_type,
|
||||||
"task_id": task.id,
|
"scope": handbook_scope,
|
||||||
"task_type": task_type,
|
"status": "planned",
|
||||||
"scope": handbook_scope,
|
},
|
||||||
"status": "planned",
|
"source": "douyu_task",
|
||||||
},
|
"subject_type": "task",
|
||||||
"source": "douyu_task",
|
"subject_key": str(task.id),
|
||||||
"subject_type": "task",
|
"actor_type": "user",
|
||||||
"subject_key": str(task.id),
|
"actor_id": created_by,
|
||||||
"actor_type": "user",
|
"source_ref": f"douyu_task_started:{task.id}",
|
||||||
"actor_id": created_by,
|
}
|
||||||
"source_ref": f"douyu_task_started:{task.id}",
|
for task, account in task_rows
|
||||||
}
|
]
|
||||||
for task, account in task_rows
|
# 任务行必须先独立提交。账号事件是旁路,不能在持有任务行锁时批量插入。
|
||||||
],
|
|
||||||
)
|
|
||||||
db.commit()
|
db.commit()
|
||||||
|
if events:
|
||||||
|
try:
|
||||||
|
emit_bulk(db, events)
|
||||||
|
db.commit()
|
||||||
|
except Exception: # noqa: BLE001 - 时间线不能回滚已经提交的任务
|
||||||
|
db.rollback()
|
||||||
|
logger.exception("批量写入斗鱼任务创建事件失败")
|
||||||
return batch_id, len(accounts)
|
return batch_id, len(accounts)
|
||||||
|
|
||||||
|
|
||||||
@@ -424,39 +465,73 @@ def cleanup_orphan_douyu_tasks(
|
|||||||
if not tasks:
|
if not tasks:
|
||||||
return 0
|
return 0
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
from .business_events import record_douyu_task_result
|
account_ids = {int(task.account_id) for task in tasks if task.account_id}
|
||||||
|
accounts = (
|
||||||
|
{
|
||||||
|
account.id: account
|
||||||
|
for account in db.query(Account).filter(Account.id.in_(account_ids)).all()
|
||||||
|
}
|
||||||
|
if account_ids
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
event_contexts: list[tuple[dict, Account]] = []
|
||||||
|
platform_task_ids: list[int] = []
|
||||||
for task in tasks:
|
for task in tasks:
|
||||||
task.status = "stopped"
|
task.status = "stopped"
|
||||||
task.message = message
|
task.message = message
|
||||||
task.finished_at = now
|
task.finished_at = now
|
||||||
if task.account_id:
|
account = accounts.get(int(task.account_id)) if task.account_id else None
|
||||||
account = db.query(Account).filter(Account.id == task.account_id).first()
|
if account is not None:
|
||||||
if account is not None:
|
event_contexts.append(
|
||||||
record_douyu_task_result(
|
(
|
||||||
db,
|
{
|
||||||
|
"task_id": int(task.id),
|
||||||
|
"task_type": str(task.task_type or ""),
|
||||||
|
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
|
||||||
|
str(task.task_type or ""),
|
||||||
|
str(task.task_type or "斗鱼任务"),
|
||||||
|
),
|
||||||
|
"scope": str(task.handbook_scope or ""),
|
||||||
|
},
|
||||||
account,
|
account,
|
||||||
task_id=int(task.id),
|
|
||||||
task_type=str(task.task_type or ""),
|
|
||||||
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
|
|
||||||
str(task.task_type or ""),
|
|
||||||
str(task.task_type or "斗鱼任务"),
|
|
||||||
),
|
|
||||||
scope=str(task.handbook_scope or ""),
|
|
||||||
status="stopped",
|
|
||||||
message=message,
|
|
||||||
occurred_at=now,
|
|
||||||
)
|
)
|
||||||
|
)
|
||||||
if task.platform_task_id:
|
if task.platform_task_id:
|
||||||
from .platform_fulfillment.service import record_douyu_operation_result
|
platform_task_ids.append(int(task.id))
|
||||||
|
# 先提交任务终态,再写账号时间线,避免 account_events 与 douyu_tasks
|
||||||
|
# 在并发创建和结束任务时形成反向锁依赖。
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
from .business_events import record_douyu_task_result
|
||||||
|
|
||||||
|
for context, account in event_contexts:
|
||||||
|
record_douyu_task_result(
|
||||||
|
db,
|
||||||
|
account,
|
||||||
|
task_id=context["task_id"],
|
||||||
|
task_type=context["task_type"],
|
||||||
|
task_label=context["task_label"],
|
||||||
|
scope=context["scope"],
|
||||||
|
status="stopped",
|
||||||
|
message=message,
|
||||||
|
occurred_at=now,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
if platform_task_ids:
|
||||||
|
from .platform_fulfillment.service import record_douyu_operation_result
|
||||||
|
|
||||||
|
for task_id in platform_task_ids:
|
||||||
|
task = db.get(DouyuTask, task_id)
|
||||||
|
if task is None:
|
||||||
|
continue
|
||||||
record_douyu_operation_result(
|
record_douyu_operation_result(
|
||||||
db,
|
db,
|
||||||
task,
|
task,
|
||||||
status="stopped",
|
status="stopped",
|
||||||
message=message,
|
message=message,
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
return len(tasks)
|
return len(tasks)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -138,7 +138,10 @@ def _mark_attempt_failed(
|
|||||||
失败重试会新建记录或新 attempt,因此是新的一条 —— 这是对的,那是新的事实。
|
失败重试会新建记录或新 attempt,因此是新的一条 —— 这是对的,那是新的事实。
|
||||||
"""
|
"""
|
||||||
completed_at = datetime.now(UTC)
|
completed_at = datetime.now(UTC)
|
||||||
attempt_at = task.attempt_at or completed_at
|
# MySQL DATETIME(0) 会按毫秒四舍五入到整秒。若 source_ref 使用内存里
|
||||||
|
# 带微秒的原值,第一次写 13:57:13.999 会落成 13:57:14;第二次重读后
|
||||||
|
# 幂等键变成下一秒,唯一索引就无法拦住重复事件。先截断到整秒再统一使用。
|
||||||
|
attempt_at = task.attempt_at or completed_at.replace(microsecond=0)
|
||||||
attempt_utc = (
|
attempt_utc = (
|
||||||
attempt_at.replace(tzinfo=UTC)
|
attempt_at.replace(tzinfo=UTC)
|
||||||
if attempt_at.tzinfo is None
|
if attempt_at.tzinfo is None
|
||||||
|
|||||||
Reference in New Issue
Block a user