diff --git a/.env.example b/.env.example index 920982e..79fd62d 100644 --- a/.env.example +++ b/.env.example @@ -68,6 +68,12 @@ DB_POOL_SIZE=20 DB_MAX_OVERFLOW=20 DB_POOL_TIMEOUT=30 DB_POOL_RECYCLE=1800 +# 行锁 10 秒仍未拿到就失败,避免把连接池和请求线程长期堵住。 +DB_LOCK_WAIT_TIMEOUT=10 +# 斗鱼批次创建并发与全局活跃量背压。 +DOUYU_BATCH_CREATE_CONCURRENCY=2 +DOUYU_MAX_ACTIVE_BATCHES=20 +DOUYU_MAX_ACTIVE_TASKS=250 # Cookie 安全标志(生产环境 HTTPS 部署时设为 true) COOKIE_SECURE=false diff --git a/docker-compose.yml b/docker-compose.yml index afbb02c..59a1cb7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,10 @@ services: - DB_MAX_OVERFLOW=${DB_MAX_OVERFLOW:-20} - DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30} - DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-1800} + - DB_LOCK_WAIT_TIMEOUT=${DB_LOCK_WAIT_TIMEOUT:-10} + - DOUYU_BATCH_CREATE_CONCURRENCY=${DOUYU_BATCH_CREATE_CONCURRENCY:-2} + - DOUYU_MAX_ACTIVE_BATCHES=${DOUYU_MAX_ACTIVE_BATCHES:-20} + - DOUYU_MAX_ACTIVE_TASKS=${DOUYU_MAX_ACTIVE_TASKS:-250} # JWT 密钥(生产环境务必修改,可用 python -c "import secrets; print(secrets.token_urlsafe(32))" 生成) - JWT_SECRET_KEY=${JWT_SECRET_KEY:-} # 后台登录滑动会话:24h 空闲过期,7d 绝对上限。 diff --git a/tests/test_douyu_task_concurrency.py b/tests/test_douyu_task_concurrency.py new file mode 100644 index 0000000..7fdbec6 --- /dev/null +++ b/tests/test_douyu_task_concurrency.py @@ -0,0 +1,227 @@ +import inspect + +import pytest +from fastapi import HTTPException +from sqlalchemy import text + +from tests.support.db import make_test_session +from web.backend.database import engine as app_engine +from web.backend.main import health +from web.backend.models import Account, DouyuTask, User +from web.backend.routers import douyu as douyu_router +from web.backend.services import business_events, douyu_service +from web.backend.services.douyu_runner_core import ( + DouyuBatchRunnerCore, + slim_douyu_task_result_for_storage, +) + + +def _fixtures(db, *, status: str = "planned") -> tuple[User, Account, DouyuTask]: + user = User(username="concurrency-user", password_hash="hash", role="operation") + db.add(user) + db.flush() + account = Account( + username="concurrency-account", + password="password", + email="concurrency@example.com", + email_password="mail-password", + cookie="acf_uid=1", + ) + db.add(account) + db.flush() + task = DouyuTask( + batch_id="concurrency-batch", + account_id=account.id, + account_assigned_to=user.id, + task_type="get_bind_qr", + handbook_scope="elite", + status=status, + created_by=user.id, + ) + db.add(task) + db.commit() + return user, account, task + + +def test_task_rows_commit_before_started_events(db, monkeypatch): + user = User(username="event-order", password_hash="hash", role="operation") + db.add(user) + db.flush() + account = Account( + username="event-order-account", + password="password", + email="event-order@example.com", + email_password="mail-password", + cookie="acf_uid=2", + ) + db.add(account) + db.commit() + observed: list[int] = [] + + def fake_emit_bulk_read_committed(_db, events): + other = make_test_session() + try: + observed.append( + other.query(DouyuTask).filter(DouyuTask.status == "planned").count() + ) + finally: + other.close() + return len(events) + + monkeypatch.setattr(douyu_service, "emit_bulk", fake_emit_bulk_read_committed) + batch_id, count = douyu_service.create_douyu_planned_tasks( + db, + [account.id], + "query_points", + "elite", + user.id, + ) + + assert batch_id + assert count == 1 + assert observed == [1] + + +def test_mark_task_commits_result_before_account_event(db, monkeypatch): + _, account, task = _fixtures(db, status="running") + task.account = account + runner = DouyuBatchRunnerCore( + db, + task.batch_id, + task.task_type, + close_db=False, + ) + observed: list[tuple[str, object]] = [] + + def fake_record(_db, _account, **kwargs): + other = make_test_session() + try: + row = other.get(DouyuTask, task.id) + observed.append((row.status, row.result)) + finally: + other.close() + return True + + monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record) + runner._mark_task( + db, + task, + "success", + "done", + { + "url": "https://example.test/qr", + "raw": {"large": "payload"}, + "before_bind_info": { + "role_name": "role-a", + "raw": {"large": "nested"}, + }, + }, + ) + + assert observed == [ + ( + "success", + { + "url": "https://example.test/qr", + "before_bind_info": {"role_name": "role-a"}, + }, + ) + ] + + +def test_cleanup_commits_stopped_status_before_account_event(db, monkeypatch): + _, _, task = _fixtures(db, status="running") + observed: list[str] = [] + + def fake_record(_db, _account, **kwargs): + other = make_test_session() + try: + observed.append(other.get(DouyuTask, task.id).status) + finally: + other.close() + return True + + monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record) + cleaned = douyu_service.cleanup_orphan_douyu_tasks( + db, + active_batch_ids=set(), + statuses=("running",), + ) + + assert cleaned == 1 + assert observed == ["stopped"] + + +def test_active_task_budget_rejects_new_batch(db, monkeypatch): + user = User(username="budget-user", password_hash="hash", role="operation") + db.add(user) + db.flush() + account = Account( + username="budget-account", + password="password", + email="budget@example.com", + email_password="mail-password", + cookie="acf_uid=3", + ) + db.add(account) + db.commit() + monkeypatch.setattr(douyu_service, "DOUYU_MAX_ACTIVE_TASKS", 1) + + douyu_service.create_douyu_planned_tasks( + db, + [account.id], + "query_points", + "elite", + user.id, + ) + with pytest.raises(douyu_service.DouyuTaskBackpressureError, match="上限 1"): + douyu_service.create_douyu_planned_tasks( + db, + [account.id], + "query_game_name", + "elite", + user.id, + ) + + +def test_active_batch_budget_returns_429(monkeypatch): + monkeypatch.setattr(douyu_router, "DOUYU_MAX_ACTIVE_BATCHES", 1) + + with pytest.raises(HTTPException) as exc_info: + douyu_router._ensure_active_batch_budget(1) + + assert exc_info.value.status_code == 429 + + +def test_slim_task_result_removes_debug_payloads(): + result = slim_douyu_task_result_for_storage( + { + "url": "https://example.test/qr", + "raw": {"debug": True}, + "query_params": {"token": "secret"}, + "before_bind_info": { + "role_name": "role-a", + "raw": {"debug": True}, + }, + "items": [{"name": "kept", "raw": {"debug": True}}], + } + ) + + assert result == { + "url": "https://example.test/qr", + "before_bind_info": {"role_name": "role-a"}, + "items": [{"name": "kept"}], + } + + +def test_health_endpoint_does_not_use_db_threadpool(): + assert inspect.iscoroutinefunction(health) + + +def test_mysql_lock_wait_timeout_is_bounded(): + with app_engine.connect() as connection: + value = connection.execute( + text("SELECT @@SESSION.innodb_lock_wait_timeout") + ).scalar() + assert value is not None + assert int(value) <= 10 diff --git a/tests/test_login_failure_events.py b/tests/test_login_failure_events.py index ed5b548..ab981d8 100644 --- a/tests/test_login_failure_events.py +++ b/tests/test_login_failure_events.py @@ -142,7 +142,11 @@ class TestLoginFailureEvents(_Base): self.session.commit() _mark_attempt_failed(task, "极验验证失败: slide") self.session.commit() - assert len(self.events(account.id)) == 1 + events = self.events(account.id) + assert len(events) == 1, [ + (event.source_ref, event.occurred_at) for event in events + ] + assert events[0].occurred_at.microsecond == 0 def test_event_does_not_break_when_task_has_no_session(self): """纯内存构造的任务(没有 session)不能因为写事件而炸掉主流程。""" diff --git a/web/backend/database.py b/web/backend/database.py index 13cba26..868d56f 100644 --- a/web/backend/database.py +++ b/web/backend/database.py @@ -82,6 +82,19 @@ if DATABASE_URL.startswith("sqlite"): cursor.execute("PRAGMA busy_timeout=30000") cursor.execute("PRAGMA foreign_keys=ON") cursor.close() +else: + + @event.listens_for(engine, "connect") + def _set_mysql_session_guards(dbapi_connection, connection_record): + """让行锁等待快速失败,避免单个事务把连接池堵住几十秒。""" + cursor = dbapi_connection.cursor() + try: + cursor.execute( + "SET SESSION innodb_lock_wait_timeout = " + f"{_env_int('DB_LOCK_WAIT_TIMEOUT', 10)}" + ) + finally: + cursor.close() SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False) diff --git a/web/backend/main.py b/web/backend/main.py index 85885a8..e495696 100644 --- a/web/backend/main.py +++ b/web/backend/main.py @@ -201,7 +201,7 @@ app.include_router(files.router) @app.get("/api/health") -def health(): +async def health(): return {"status": "ok"} diff --git a/web/backend/routers/douyu.py b/web/backend/routers/douyu.py index d6eab4a..dbbf893 100644 --- a/web/backend/routers/douyu.py +++ b/web/backend/routers/douyu.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio from datetime import UTC, datetime from typing import Any, cast +from weakref import WeakKeyDictionary from fastapi import ( APIRouter, @@ -18,6 +19,7 @@ from fastapi import ( from loguru import logger from sqlalchemy import case, func, or_, select from sqlalchemy.orm import Session, joinedload +from starlette.concurrency import run_in_threadpool from core.douyu import ( DouyuActivityError, @@ -56,15 +58,20 @@ from ..services.account_binding import bind_payload from ..services.audit_service import record_audit from ..services.business_events import record_recharge from ..services.douyu_activity_query import activity_client, query_activity_goods -from ..services.douyu_batch_launcher import start_douyu_batch +from ..services.douyu_batch_launcher import ( + DOUYU_BATCH_CREATE_CONCURRENCY, + DOUYU_MAX_ACTIVE_BATCHES, + start_douyu_batch, +) from ..services.douyu_goods_service import serialize_goods from ..services.douyu_runner import DouyuBatchRunner, douyu_batch_registry from ..services.douyu_service import ( + DOUYU_ACTIVE_TASK_STATUSES, DOUYU_CONFIG_FIELDS, DOUYU_HANDBOOK_TASK_TYPES, - DOUYU_ACTIVE_TASK_STATUSES, SUPPORTED_DOUYU_TASK_TYPES, DouyuActivePaymentError, + DouyuTaskBackpressureError, apply_douyu_config_defaults, cleanup_orphan_douyu_tasks, cookie_holding_filter, @@ -77,6 +84,33 @@ from ..services.platform_fulfillment import service as platform_fulfillment_serv router = APIRouter(prefix="/api/douyu", tags=["斗鱼活动"]) +_BATCH_CREATE_SEMAPHORES: WeakKeyDictionary[ + asyncio.AbstractEventLoop, + asyncio.Semaphore, +] = WeakKeyDictionary() + + +def _batch_create_semaphore() -> asyncio.Semaphore: + """按事件循环复用创建锁,避免测试和开发热重载跨 loop 复用 Semaphore。""" + loop = asyncio.get_running_loop() + semaphore = _BATCH_CREATE_SEMAPHORES.get(loop) + if semaphore is None: + semaphore = asyncio.Semaphore(DOUYU_BATCH_CREATE_CONCURRENCY) + _BATCH_CREATE_SEMAPHORES[loop] = semaphore + return semaphore + + +def _ensure_active_batch_budget(active_count: int) -> None: + if active_count >= DOUYU_MAX_ACTIVE_BATCHES: + raise HTTPException( + status_code=429, + detail=( + f"斗鱼任务繁忙:当前有 {active_count} 个批次运行中," + f"上限 {DOUYU_MAX_ACTIVE_BATCHES} 个,请稍后重试" + ), + headers={"Retry-After": "5"}, + ) + def _supplier_response_value(payload: dict, *keys: str): """兼容供应商把订单字段放在根节点、data 或 result 中。""" @@ -782,13 +816,12 @@ def list_xpd_goods( return rows -@router.post("/tasks/batch") -async def create_task_batch( +def _prepare_task_batch( req: DouyuTaskBatchRequest, - db: Session = Depends(get_db), - current: User = Depends(require_permission("douyu:task")), -): - """创建斗鱼任务记录并启动后台执行器。""" + db: Session, + current: User, +) -> tuple[str, int]: + """在线程池中完成全部同步 DB 工作,避免阻塞 Uvicorn 事件循环。""" if not req.account_ids: raise HTTPException(status_code=400, detail="请选择斗鱼账号") _require_task_account_access( @@ -818,12 +851,14 @@ async def create_task_batch( ) platform_task = db.get(PlatformFulfillmentTask, int(platform_task.id)) + active_batch_ids = douyu_batch_registry.active_ids() cleanup_orphan_douyu_tasks( db, - active_batch_ids=douyu_batch_registry.active_ids(), + active_batch_ids=active_batch_ids, statuses=DOUYU_ACTIVE_TASK_STATUSES, message="任务已中断(无执行器接管)", ) + _ensure_active_batch_budget(len(active_batch_ids)) try: batch_id, count = create_douyu_planned_tasks( @@ -837,6 +872,12 @@ async def create_task_batch( ) except DouyuActivePaymentError as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc + except DouyuTaskBackpressureError as exc: + raise HTTPException( + status_code=429, + detail=str(exc), + headers={"Retry-After": "5"}, + ) from exc except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc if count == 0: @@ -911,12 +952,33 @@ async def create_task_batch( ) db.commit() - start_douyu_batch( - batch_id=batch_id, - task_type=req.task_type, - payload=req.payload, - concurrency=req.concurrency, - ) + return batch_id, count + + +@router.post("/tasks/batch") +async def create_task_batch( + req: DouyuTaskBatchRequest, + db: Session = Depends(get_db), + current: User = Depends(require_permission("douyu:task")), +): + """创建斗鱼任务记录并启动后台执行器。""" + # 同步 SQLAlchemy 段必须在线程池执行;事件循环只负责最后启动后台线程。 + async with _batch_create_semaphore(): + batch_id, count = await run_in_threadpool( + _prepare_task_batch, + req, + db, + current, + ) + await run_in_threadpool( + start_douyu_batch, + batch_id=batch_id, + task_type=req.task_type, + payload=req.payload, + concurrency=req.concurrency, + loop=asyncio.get_running_loop(), + log_queue=asyncio.Queue(), + ) return {"batch_id": batch_id, "count": count, "success": True} diff --git a/web/backend/services/douyu_batch_launcher.py b/web/backend/services/douyu_batch_launcher.py index da6a52e..69bc9d3 100644 --- a/web/backend/services/douyu_batch_launcher.py +++ b/web/backend/services/douyu_batch_launcher.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import os import threading from web.backend.database import SessionLocal @@ -13,16 +14,33 @@ from web.backend.database import SessionLocal from .douyu_runner import DouyuBatchRunner, douyu_batch_registry +def _env_positive_int(name: str, default: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except ValueError: + return default + return value if value > 0 else default + + +DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int( + "DOUYU_BATCH_CREATE_CONCURRENCY", + 2, +) +DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20) + + def start_douyu_batch( *, batch_id: str, task_type: str, payload: dict, concurrency: int, + loop: asyncio.AbstractEventLoop | None = None, + log_queue: asyncio.Queue | None = None, ) -> None: - """必须在运行中的 asyncio 事件循环里调用。""" - log_queue = asyncio.Queue() - loop = asyncio.get_running_loop() + """创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。""" + loop = loop or asyncio.get_running_loop() + log_queue = log_queue or asyncio.Queue() thread_db = SessionLocal() runner = DouyuBatchRunner( db=thread_db, diff --git a/web/backend/services/douyu_runner_core.py b/web/backend/services/douyu_runner_core.py index 3935696..9a6bb67 100644 --- a/web/backend/services/douyu_runner_core.py +++ b/web/backend/services/douyu_runner_core.py @@ -6,7 +6,7 @@ import asyncio import threading import time from datetime import UTC, datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from loguru import logger from sqlalchemy.orm import Session @@ -59,6 +59,38 @@ DOUYU_PROXY_TASK_TYPES = { } +_DOUYU_RESULT_DEBUG_KEYS = { + "raw", + "query_params", + "qq_app_scheme", + "role_param_url", + "game_role_bind_url", +} + + +def slim_douyu_task_result_for_storage(result: dict | None) -> dict | None: + """去掉任务结果里的调试大字段,避免每次轮询重复写大 JSON。 + + 运行器在内存中仍使用完整 result;这里只影响落库和 WS 展示。二维码 URL、 + 角色状态、积分、订单号等业务字段全部保留。 + """ + if result is None: + return None + + def clean(value: Any) -> Any: + if isinstance(value, dict): + return { + key: clean(item) + for key, item in value.items() + if key not in _DOUYU_RESULT_DEBUG_KEYS + } + if isinstance(value, list): + return [clean(item) for item in value] + return value + + return clean(result) + + class DouyuBatchRunnerCore: """斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。""" @@ -196,9 +228,7 @@ class DouyuBatchRunnerCore: sync_task_from_binding_runner(db, douyu_task_id=int(task.id)) except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果 - logger.exception( - "[douyu] 同步直播账号履约状态失败: task_id={}", task.id - ) + logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id) def _sync_platform_fulfillment_operation( self, @@ -220,9 +250,7 @@ class DouyuBatchRunnerCore: message=message, ) except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果 - logger.exception( - "[douyu] 写入履约操作时间线失败: task_id={}", task.id - ) + logger.exception("[douyu] 写入履约操作时间线失败: task_id={}", task.id) def _mark_task( self, @@ -235,30 +263,28 @@ class DouyuBatchRunnerCore: task.status = status task.message = message[:512] if result is not None: - task.result = result + task.result = slim_douyu_task_result_for_storage(result) task.finished_at = datetime.now(UTC) - if task.account_id and status in {"success", "failed", "error", "stopped"}: - from .business_events import record_douyu_task_result - - account = ( - db.query(Account).filter(Account.id == task.account_id).first() - ) - if account is not None: - record_douyu_task_result( - db, - account, - task_id=int(task.id), - task_type=str(task.task_type or ""), - task_label=SUPPORTED_DOUYU_TASK_TYPES.get( - str(task.task_type or ""), - str(task.task_type or "斗鱼任务"), - ), - scope=str(task.handbook_scope or ""), - status=status, - message=message, - occurred_at=task.finished_at, - actor_id=int(task.created_by), - ) + account = ( + db.query(Account).filter(Account.id == task.account_id).first() + if task.account_id and status in {"success", "failed", "error", "stopped"} + else None + ) + event_context = ( + { + "task_id": int(task.id), + "task_type": str(task.task_type or ""), + "task_label": SUPPORTED_DOUYU_TASK_TYPES.get( + str(task.task_type or ""), + str(task.task_type or "斗鱼任务"), + ), + "scope": str(task.handbook_scope or ""), + "finished_at": task.finished_at, + "created_by": int(task.created_by), + } + if account is not None + else None + ) self._sync_platform_fulfillment_operation( db, task, @@ -267,6 +293,28 @@ class DouyuBatchRunnerCore: ) db.commit() self._push_task_event(task) + if account is not None and event_context is not None: + try: + from .business_events import record_douyu_task_result + + record_douyu_task_result( + db, + account, + task_id=event_context["task_id"], + task_type=event_context["task_type"], + task_label=event_context["task_label"], + scope=event_context["scope"], + status=status, + message=message, + occurred_at=event_context["finished_at"], + actor_id=event_context["created_by"], + ) + db.commit() + except Exception: # noqa: BLE001 - 时间线旁路不能覆盖已提交的任务结果 + db.rollback() + logger.exception( + "[douyu] 写入账号任务结果事件失败: task_id={}", task.id + ) self._sync_platform_fulfillment(db, task) def _update_task_progress( @@ -280,7 +328,7 @@ class DouyuBatchRunnerCore: task.status = status task.message = message[:512] if result is not None: - task.result = result + task.result = slim_douyu_task_result_for_storage(result) db.commit() self._push_task_event(task) self._sync_platform_fulfillment(db, task) diff --git a/web/backend/services/douyu_service.py b/web/backend/services/douyu_service.py index 0c74543..955dc7f 100644 --- a/web/backend/services/douyu_service.py +++ b/web/backend/services/douyu_service.py @@ -2,9 +2,11 @@ from __future__ import annotations +import os import uuid from datetime import UTC, datetime +from loguru import logger from sqlalchemy import and_, func, select from sqlalchemy.orm import Session @@ -146,10 +148,42 @@ DOUYU_ACTIVE_PAYMENT_TASK_TYPES = ( ) +def _env_positive_int(name: str, default: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except ValueError: + return default + return value if value > 0 else default + + +DOUYU_MAX_ACTIVE_TASKS = _env_positive_int("DOUYU_MAX_ACTIVE_TASKS", 250) + + class DouyuActivePaymentError(ValueError): """同一账号已有未结束的充值任务。""" +class DouyuTaskBackpressureError(ValueError): + """全局活跃斗鱼任务超过安全上限。""" + + +def _ensure_active_task_budget(db: Session, additional: int) -> None: + """为任务创建提供全局背压,避免无限线程和连接同时挤入数据库。""" + if additional <= 0: + return + active = ( + db.query(func.count(DouyuTask.id)) + .filter(DouyuTask.status.in_(DOUYU_ACTIVE_TASK_STATUSES)) + .scalar() + or 0 + ) + if active + additional > DOUYU_MAX_ACTIVE_TASKS: + raise DouyuTaskBackpressureError( + f"斗鱼任务繁忙:当前活跃 {active} 个," + f"上限 {DOUYU_MAX_ACTIVE_TASKS} 个,请稍后重试" + ) + + def douyu_config_value(field: str, value): """读取配置值;空值自动回退到默认值。""" default = DOUYU_CONFIG_DEFAULTS[field] @@ -348,6 +382,7 @@ def create_douyu_planned_tasks( raise ValueError("该任务不属于当前工作台") accounts = visible_douyu_task_accounts(db, account_ids) + _ensure_active_task_budget(db, len(accounts)) _lock_payment_accounts(db, accounts, task_type) if ( task_type in {"refresh_goods", "refresh_esports_goods", "refresh_xpd_goods"} @@ -375,33 +410,39 @@ def create_douyu_planned_tasks( db.add(task) task_rows.append((task, account)) db.flush() + events: list[dict] = [] if task_rows: label = SUPPORTED_DOUYU_TASK_TYPES[task_type] - emit_bulk( - db, - [ - { - "account_id": account.id, - "event_type": EVENT_TASK_STARTED, - "title": f"{label}:任务已创建", - "category": CATEGORY_ACTIVITY, - "detail": { - "task_id": task.id, - "task_type": task_type, - "scope": handbook_scope, - "status": "planned", - }, - "source": "douyu_task", - "subject_type": "task", - "subject_key": str(task.id), - "actor_type": "user", - "actor_id": created_by, - "source_ref": f"douyu_task_started:{task.id}", - } - for task, account in task_rows - ], - ) + events = [ + { + "account_id": account.id, + "event_type": EVENT_TASK_STARTED, + "title": f"{label}:任务已创建", + "category": CATEGORY_ACTIVITY, + "detail": { + "task_id": task.id, + "task_type": task_type, + "scope": handbook_scope, + "status": "planned", + }, + "source": "douyu_task", + "subject_type": "task", + "subject_key": str(task.id), + "actor_type": "user", + "actor_id": created_by, + "source_ref": f"douyu_task_started:{task.id}", + } + for task, account in task_rows + ] + # 任务行必须先独立提交。账号事件是旁路,不能在持有任务行锁时批量插入。 db.commit() + if events: + try: + emit_bulk(db, events) + db.commit() + except Exception: # noqa: BLE001 - 时间线不能回滚已经提交的任务 + db.rollback() + logger.exception("批量写入斗鱼任务创建事件失败") return batch_id, len(accounts) @@ -424,39 +465,73 @@ def cleanup_orphan_douyu_tasks( if not tasks: return 0 now = datetime.now(UTC) - from .business_events import record_douyu_task_result - + account_ids = {int(task.account_id) for task in tasks if task.account_id} + accounts = ( + { + account.id: account + for account in db.query(Account).filter(Account.id.in_(account_ids)).all() + } + if account_ids + else {} + ) + event_contexts: list[tuple[dict, Account]] = [] + platform_task_ids: list[int] = [] for task in tasks: task.status = "stopped" task.message = message task.finished_at = now - if task.account_id: - account = db.query(Account).filter(Account.id == task.account_id).first() - if account is not None: - record_douyu_task_result( - db, + account = accounts.get(int(task.account_id)) if task.account_id else None + if account is not None: + event_contexts.append( + ( + { + "task_id": int(task.id), + "task_type": str(task.task_type or ""), + "task_label": SUPPORTED_DOUYU_TASK_TYPES.get( + str(task.task_type or ""), + str(task.task_type or "斗鱼任务"), + ), + "scope": str(task.handbook_scope or ""), + }, account, - task_id=int(task.id), - task_type=str(task.task_type or ""), - task_label=SUPPORTED_DOUYU_TASK_TYPES.get( - str(task.task_type or ""), - str(task.task_type or "斗鱼任务"), - ), - scope=str(task.handbook_scope or ""), - status="stopped", - message=message, - occurred_at=now, ) + ) if task.platform_task_id: - from .platform_fulfillment.service import record_douyu_operation_result + platform_task_ids.append(int(task.id)) + # 先提交任务终态,再写账号时间线,避免 account_events 与 douyu_tasks + # 在并发创建和结束任务时形成反向锁依赖。 + db.commit() + from .business_events import record_douyu_task_result + + for context, account in event_contexts: + record_douyu_task_result( + db, + account, + task_id=context["task_id"], + task_type=context["task_type"], + task_label=context["task_label"], + scope=context["scope"], + status="stopped", + message=message, + occurred_at=now, + ) + db.commit() + + if platform_task_ids: + from .platform_fulfillment.service import record_douyu_operation_result + + for task_id in platform_task_ids: + task = db.get(DouyuTask, task_id) + if task is None: + continue record_douyu_operation_result( db, task, status="stopped", message=message, ) - db.commit() + db.commit() return len(tasks) diff --git a/web/backend/services/login_service.py b/web/backend/services/login_service.py index e299857..aa667f5 100644 --- a/web/backend/services/login_service.py +++ b/web/backend/services/login_service.py @@ -138,7 +138,10 @@ def _mark_attempt_failed( 失败重试会新建记录或新 attempt,因此是新的一条 —— 这是对的,那是新的事实。 """ completed_at = datetime.now(UTC) - attempt_at = task.attempt_at or completed_at + # MySQL DATETIME(0) 会按毫秒四舍五入到整秒。若 source_ref 使用内存里 + # 带微秒的原值,第一次写 13:57:13.999 会落成 13:57:14;第二次重读后 + # 幂等键变成下一秒,唯一索引就无法拦住重复事件。先截断到整秒再统一使用。 + attempt_at = task.attempt_at or completed_at.replace(microsecond=0) attempt_utc = ( attempt_at.replace(tzinfo=UTC) if attempt_at.tzinfo is None