diff --git a/.env.example b/.env.example index 34d381a..6b8b312 100644 --- a/.env.example +++ b/.env.example @@ -66,6 +66,8 @@ DEV_DB_PORT=3307 # 数据库连接池配置(仅 MySQL 生效) DB_POOL_SIZE=20 DB_MAX_OVERFLOW=20 +WORKER_DB_POOL_SIZE=12 +WORKER_DB_MAX_OVERFLOW=4 DB_POOL_TIMEOUT=30 DB_POOL_RECYCLE=1800 # 行锁 10 秒仍未拿到就失败,避免把连接池和请求线程长期堵住。 @@ -75,7 +77,6 @@ DOUYU_BATCH_CREATE_CONCURRENCY=2 DOUYU_BATCH_TASK_CONCURRENCY=5 DOUYU_MAX_ACTIVE_BATCHES=10 # 所有批次共享的执行槽与一键查询槽,避免单个批次占满数据库连接。 -DOUYU_MAX_RUNNING_TASKS=12 DOUYU_MAX_ACTIVE_OVERVIEW_TASKS=2 DOUYU_MAX_ACTIVE_TASKS=250 DOUYU_PROBE_CACHE_SECONDS=30 diff --git a/dev.sh b/dev.sh index 5cd8e03..fc93399 100755 --- a/dev.sh +++ b/dev.sh @@ -60,6 +60,9 @@ DB_POOL_SIZE="${DB_POOL_SIZE:-20}" DB_MAX_OVERFLOW="${DB_MAX_OVERFLOW:-20}" DB_POOL_TIMEOUT="${DB_POOL_TIMEOUT:-30}" DB_POOL_RECYCLE="${DB_POOL_RECYCLE:-1800}" +WORKER_DB_POOL_SIZE="${WORKER_DB_POOL_SIZE:-12}" +WORKER_DB_MAX_OVERFLOW="${WORKER_DB_MAX_OVERFLOW:-4}" +WORKER_DB_POOL_TIMEOUT="${WORKER_DB_POOL_TIMEOUT:-30}" YYB_WORKER_PORT="${YYB_WORKER_PORT:-8810}" DEV_YYB_WORKER_URL="${DEV_YYB_WORKER_URL:-http://127.0.0.1:${YYB_WORKER_PORT}}" YYB_WORKER_DATA_DIR="${YYB_WORKER_DATA_DIR:-$ROOT_DIR/data/yyb-worker-dev}" @@ -356,6 +359,7 @@ echo "" ( export DB_HOST DB_PORT DB_NAME DB_USER DB_PASSWORD export DB_POOL_SIZE DB_MAX_OVERFLOW DB_POOL_TIMEOUT DB_POOL_RECYCLE + export WORKER_DB_POOL_SIZE WORKER_DB_MAX_OVERFLOW WORKER_DB_POOL_TIMEOUT export YYB_WORKER_KEY LOG_LEVEL LOG_DIR export YYB_WORKER_URL="$DEV_YYB_WORKER_URL" uv run python -m uvicorn web.backend.main:app \ diff --git a/docker-compose.yml b/docker-compose.yml index fdc7e36..31efdea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,10 +41,12 @@ services: - DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30} - DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-1800} - DB_LOCK_WAIT_TIMEOUT=${DB_LOCK_WAIT_TIMEOUT:-10} + - WORKER_DB_POOL_SIZE=${WORKER_DB_POOL_SIZE:-12} + - WORKER_DB_MAX_OVERFLOW=${WORKER_DB_MAX_OVERFLOW:-4} + - WORKER_DB_POOL_TIMEOUT=${WORKER_DB_POOL_TIMEOUT:-30} - DOUYU_BATCH_CREATE_CONCURRENCY=${DOUYU_BATCH_CREATE_CONCURRENCY:-2} - DOUYU_BATCH_TASK_CONCURRENCY=${DOUYU_BATCH_TASK_CONCURRENCY:-5} - DOUYU_MAX_ACTIVE_BATCHES=${DOUYU_MAX_ACTIVE_BATCHES:-10} - - DOUYU_MAX_RUNNING_TASKS=${DOUYU_MAX_RUNNING_TASKS:-12} - DOUYU_MAX_ACTIVE_OVERVIEW_TASKS=${DOUYU_MAX_ACTIVE_OVERVIEW_TASKS:-2} - DOUYU_MAX_ACTIVE_TASKS=${DOUYU_MAX_ACTIVE_TASKS:-250} - DOUYU_PROBE_CACHE_SECONDS=${DOUYU_PROBE_CACHE_SECONDS:-30} diff --git a/docs/直播账号履约工作台.md b/docs/直播账号履约工作台.md index 219ced1..5bf8dea 100644 --- a/docs/直播账号履约工作台.md +++ b/docs/直播账号履约工作台.md @@ -1,6 +1,6 @@ # 直播账号履约工作台 -更新时间:2026-11-18 +更新时间:2026-11-20 ## 目标 @@ -14,7 +14,7 @@ - 客户扫码绑定游戏角色 - 不做自动充值 - 不做自动兑换 -- 不向 `order_site` 回写状态 +- 状态变化通过事务 Outbox 回写 `order_site` - 退款、取消由运营人工处理 ## 数据边界 @@ -167,6 +167,43 @@ live-hub 管理 API。同步客户会话接口位于: `PLATFORM_BINDING_QR_TTL_SECONDS` 调整。过期、失败或客户主动刷新时会在 live-hub 创建新的绑定任务。 +## 后台任务槽与扫码轮询 + +后台任务按业务域使用独立有界任务槽,不再共用一个全局执行器: + +- `interactive_binding`:绑定预检、二维码生成和确认提交; +- `binding_poll`:每次只执行一次扫码角色查询; +- `login`、`account_check`:登录、检测与重登; +- `automation`:定时自动化作业; +- `query_batch`:账号概览、商品和角色查询; +- `payment_wait`:二维码生成后的支付/到账等待; +- `ops_write`:兑换、支付、锁单和送礼; +- `huya_batch`、`huya_register`、`integration`、`maintenance`:各自独立。 + +批次只负责向对应任务槽投递和收集结果,不能在批次内部再创建一个 Worker 池。 +新增任务类型必须显式映射到任务域;任务槽和后台数据库池状态可在自动化页面的 +运行时信息中查看。 + +扫码绑定拆成两个阶段: + +1. 二维码任务只做预检和出码,出码后写入 `queue_lane=binding_poll`、 + `next_run_at` 和 `run_deadline_at`,立即释放 Worker; +2. `binding-poll-scheduler` 从数据库领取到期任务,每次只查询一次角色, + 查到待确认角色或到期后收敛状态。 + +`douyu_tasks.active_binding_key` 在数据库层保证同一履约任务最多只有一个 +活动二维码任务。进程重启后,未到期的扫码任务继续由数据库调度,不会因内存 +批次丢失而终止。 + +客户绑定接口是幂等异步契约: + +- 新建排队返回 `202`,已存在有效二维码或已完成返回 `200`; +- `mapping_missing`、尚未分配账号、租约失效都返回当前业务快照,不使用 409 + 驱动客户端重复提交; +- 绑定任务创建时写入 `platform_task_id`,二维码就绪、角色识别和确认完成都会 + 在状态事务内写 Outbox,由 order_site Webhook 和 SSE 推动页面重读; +- “刷新状态”不启动绑定,“刷新二维码”先读取权威状态,有效期内直接复用。 + ## 权限 - `platform_fulfillment:view` diff --git a/tests/support/db.py b/tests/support/db.py index b7cc4f0..fdb7e7b 100644 --- a/tests/support/db.py +++ b/tests/support/db.py @@ -114,7 +114,9 @@ def get_test_engine() -> Engine: pool_recycle=1800, ) database_module.engine = _test_engine + database_module.worker_engine = _test_engine database_module.SessionLocal.configure(bind=_test_engine) + database_module.WorkerSessionLocal.configure(bind=_test_engine) return _test_engine @@ -163,6 +165,23 @@ def _add_missing_columns(engine: Engine) -> None: ) +def _add_missing_indexes(engine: Engine) -> None: + """Create model indexes added after the reusable test schema.""" + inspector = inspect(engine) + with engine.begin() as conn: + for table in Base.metadata.sorted_tables: + if not inspector.has_table(table.name): + continue + existing = { + str(index["name"]) + for index in inspector.get_indexes(table.name) + if index.get("name") + } + for index in table.indexes: + if index.name and index.name not in existing: + index.create(bind=conn) + + def init_test_schema(engine: Engine | None = None) -> None: """确保测试库中所有表结构存在并清空所有数据(幂等高效)。""" eng = engine or get_test_engine() @@ -170,6 +189,7 @@ def init_test_schema(engine: Engine | None = None) -> None: _add_missing_columns(eng) Base.metadata.create_all(eng) clear_tables(eng) + _add_missing_indexes(eng) except OperationalError as exc: # 测试不再使用 SQLite,连不上库时必须给出可操作的提示, # 而不是抛一段 SQLAlchemy 原始堆栈。 diff --git a/tests/test_douyu_task_concurrency.py b/tests/test_douyu_task_concurrency.py index a9f16b2..4aa7f96 100644 --- a/tests/test_douyu_task_concurrency.py +++ b/tests/test_douyu_task_concurrency.py @@ -1,6 +1,7 @@ import inspect import threading import time +from datetime import UTC, datetime, timedelta import pytest from fastapi import HTTPException @@ -13,11 +14,18 @@ from web.backend.main import health from web.backend.models import Account, DouyuTask, User from web.backend.routers import douyu as douyu_router from web.backend.schemas import DouyuTaskBatchRequest -from web.backend.services import business_events, douyu_concurrency, douyu_service +from web.backend.services import ( + binding_poll_scheduler, + business_events, + douyu_concurrency, + douyu_service, +) +from web.backend.services.douyu_runner import DouyuBatchRunner from web.backend.services.douyu_runner_core import ( DouyuBatchRunnerCore, slim_douyu_task_result_for_storage, ) +from web.backend.services.task_runtime import TaskLane, douyu_lane_for_task def _fixtures(db, *, status: str = "planned") -> tuple[User, Account, DouyuTask]: @@ -156,6 +164,176 @@ def test_cleanup_commits_stopped_status_before_account_event(db, monkeypatch): assert observed == ["stopped"] +def test_cleanup_preserves_recoverable_binding_poll(db): + _, _, task = _fixtures(db, status="running") + task.queue_lane = "binding_poll" + task.next_run_at = datetime.now(UTC) + timedelta(seconds=5) + task.run_deadline_at = datetime.now(UTC) + timedelta(minutes=10) + db.commit() + + cleaned = douyu_service.cleanup_orphan_douyu_tasks( + db, + active_batch_ids=set(), + statuses=("running",), + ) + + db.refresh(task) + assert cleaned == 0 + assert task.status == "running" + assert task.queue_lane == "binding_poll" + + +def test_binding_poll_scheduler_claims_due_task_without_worker_wait(db): + _, _, task = _fixtures(db, status="running") + due_at = datetime.now(UTC) - timedelta(seconds=1) + task.queue_lane = "binding_poll" + task.next_run_at = due_at + task.run_deadline_at = datetime.now(UTC) + timedelta(minutes=10) + db.commit() + + try: + claimed = binding_poll_scheduler._claim_due_binding_tasks(10) + db.refresh(task) + assert claimed == [task.id] + assert task.next_run_at is not None + assert task.next_run_at > due_at + finally: + with binding_poll_scheduler._in_flight_lock: + binding_poll_scheduler._in_flight.discard(task.id) + + +def test_bind_qr_schedules_poll_and_releases_worker(db, monkeypatch): + _, account, task = _fixtures(db, status="running") + runner = DouyuBatchRunner( + db, + task.batch_id, + "get_bind_qr", + close_db=False, + ) + info = { + "act_alias": "bind-act", + "role_name": "old-role", + "is_bound_act": True, + } + snapshot = { + "role_name": "old-role", + "area_name": "", + "plat_name": "", + "game_id": "", + "game_name": "", + "game_channel": "", + "change_role_wait_time": None, + "can_change_time": None, + "can_change_role": True, + "is_bound_act": True, + "is_bound_role": True, + "is_bound_account": False, + "need_bind_act": False, + "need_bind_role": False, + } + client = type( + "Client", + (), + {"get_bind_qr": lambda self, _alias: {"url": "https://example.test/qr"}}, + )() + monkeypatch.setattr(runner, "_client", lambda _cookie: client) + monkeypatch.setattr(runner, "_fetch_bind_info_candidates", lambda *_args: [info]) + monkeypatch.setattr(runner, "_pick_baseline_bind_info", lambda *_args: info) + monkeypatch.setattr(runner, "_pick_change_wait_bind_info", lambda *_args: None) + monkeypatch.setattr(runner, "_pick_bind_info", lambda *_args, **_kwargs: info) + monkeypatch.setattr(runner, "_bind_snapshot", lambda _info: snapshot) + monkeypatch.setattr( + runner, + "_wait_bind_role_result", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("scan wait must not block the worker") + ), + ) + + runner._execute_get_bind_qr( + db, + task, + account, + "PHPSESSID=x", + { + "bind_act_alias": "bind-act", + "legacy_act_alias": "legacy-act", + }, + ) + + db.refresh(task) + assert task.status == "running" + assert task.queue_lane == "binding_poll" + assert task.next_run_at is not None + assert task.run_deadline_at is not None + assert task.result["url"] == "https://example.test/qr" + + +def test_binding_poll_once_finishes_when_role_is_found(db, monkeypatch): + _, account, task = _fixtures(db, status="running") + task.queue_lane = "binding_poll" + task.next_run_at = datetime.now(UTC) + task.run_deadline_at = datetime.now(UTC) + timedelta(minutes=10) + task.result = { + "url": "https://example.test/qr", + "query_act_aliases": ["bind-act"], + "baseline_role_name": "old-role", + "bind_polling": True, + } + db.commit() + runner = DouyuBatchRunner( + db, + task.batch_id, + "get_bind_qr", + close_db=False, + ) + info = {"act_alias": "bind-act", "role_name": "new-role"} + snapshot = { + "role_name": "new-role", + "area_name": "微信", + "plat_name": "安卓", + "game_id": "", + "game_name": "", + "game_channel": "", + "change_role_wait_time": None, + "can_change_time": None, + "can_change_role": True, + "is_bound_act": False, + "is_bound_role": True, + "is_bound_account": False, + "need_bind_act": True, + "need_bind_role": True, + } + client = type("Client", (), {})() + monkeypatch.setattr(runner, "_client", lambda _cookie: client) + monkeypatch.setattr(runner, "_fetch_bind_info_candidates", lambda *_args: [info]) + monkeypatch.setattr(runner, "_pick_bind_info", lambda *_args, **_kwargs: info) + monkeypatch.setattr(runner, "_bind_snapshot", lambda _info: snapshot) + monkeypatch.setattr(runner, "_is_pending_role", lambda *_args, **_kwargs: True) + monkeypatch.setattr( + runner, + "_format_bind_summary", + lambda *_args, **_kwargs: "role=new-role", + ) + + runner.poll_bind_role_once( + db, + task, + account, + { + "bind_act_alias": "bind-act", + "legacy_act_alias": "legacy-act", + }, + ) + + db.refresh(task) + assert task.status == "success" + assert task.queue_lane == "" + assert task.next_run_at is None + assert task.result["role_name"] == "new-role" + assert task.result["bind_ready_for_confirm"] is True + + def test_active_task_budget_rejects_new_batch(db, monkeypatch): user = User(username="budget-user", password_hash="hash", role="operation") db.add(user) @@ -211,6 +389,13 @@ def test_batch_task_concurrency_is_capped_at_five(): assert douyu_concurrency.clamp_batch_task_concurrency(20) == 5 +def test_douyu_lanes_separate_interactive_query_payment_and_writes(): + assert douyu_lane_for_task("get_bind_qr") == TaskLane.INTERACTIVE_BINDING + assert douyu_lane_for_task("query_account_overview") == TaskLane.QUERY_BATCH + assert douyu_lane_for_task("create_gold_qr") == TaskLane.PAYMENT_WAIT + assert douyu_lane_for_task("exchange_goods") == TaskLane.OPS_WRITE + + def test_overview_limiter_blocks_extra_workers(monkeypatch): slots = threading.BoundedSemaphore(1) monkeypatch.setattr(douyu_concurrency, "_overview_slots", slots) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 1468fb2..5a196a9 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -40,7 +40,7 @@ TESTS_DIR = Path(__file__).resolve().parent PROJECT_ROOT = TESTS_DIR.parent VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" -HEAD_REVISION = "20261120_0100" +HEAD_REVISION = "20261120_0101" # 迁移新增、但不声明在模型里的复合查询索引。 EXTRA_INDEXES = ( diff --git a/tests/test_platform_fulfillment.py b/tests/test_platform_fulfillment.py index f50aa0b..3e7bc96 100644 --- a/tests/test_platform_fulfillment.py +++ b/tests/test_platform_fulfillment.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import threading from datetime import UTC, datetime, timedelta from unittest.mock import patch @@ -1398,6 +1399,118 @@ class TestBindingAndCompletion(_Base): assert task.issue_code == "" assert task.last_error == "" + def test_customer_start_mapping_missing_is_idempotent(self, monkeypatch): + task = self.task(mapped=False) + task.issue_code = "mapping_missing" + self.session.commit() + monkeypatch.setattr( + service, + "allocate_task", + lambda **_kwargs: (_ for _ in ()).throw( + AssertionError("mapping_missing must not allocate synchronously") + ), + ) + + for _ in range(5): + payload = asyncio.run( + service.customer_start_binding(self.session, task_id=task.id) + ) + assert payload["accepted"] is False + assert payload["queueState"] == "mapping_missing" + + self.session.refresh(task) + assert task.account_id is None + assert task.bind_task_id is None + + def test_customer_start_queues_allocation_without_sync_network(self, monkeypatch): + task = self.task() + calls: list[list[int]] = [] + monkeypatch.setattr( + "web.backend.services.platform_fulfillment.auto_allocator.wake_auto_allocator", + lambda task_ids=None: calls.append(list(task_ids or [])), + ) + monkeypatch.setattr( + service, + "allocate_task", + lambda **_kwargs: (_ for _ in ()).throw( + AssertionError("binding start must not allocate in the request") + ), + ) + + payload = asyncio.run(service.customer_start_binding(self.session, task_id=task.id)) + + assert payload["accepted"] is True + assert payload["queueState"] == "allocation_queued" + assert calls == [[task.id]] + + def test_customer_start_creates_one_binding_task_and_links_platform(self, monkeypatch): + task = self._allocated() + monkeypatch.setattr(service, "start_douyu_batch", lambda **_kwargs: None) + + first = asyncio.run(service.customer_start_binding(self.session, task_id=task.id)) + second = asyncio.run(service.customer_start_binding(self.session, task_id=task.id)) + + assert first["accepted"] is True + assert second["accepted"] is False + bind_tasks = ( + self.session.query(DouyuTask) + .filter( + DouyuTask.platform_task_id == task.id, + DouyuTask.task_type == "get_bind_qr", + ) + .all() + ) + assert len(bind_tasks) == 1 + self.session.refresh(task) + assert task.bind_task_id == bind_tasks[0].id + + def test_concurrent_start_binding_creates_one_task(self, monkeypatch): + task = self._allocated() + monkeypatch.setattr(service, "start_douyu_batch", lambda **_kwargs: None) + barrier = threading.Barrier(2) + errors: list[BaseException] = [] + other_session = make_test_session(self.engine) + other_admin = other_session.get(User, self.admin_id) + assert other_admin is not None + + def start(db, current): + try: + barrier.wait(timeout=3) + asyncio.run( + service.start_binding( + db, + task_id=task.id, + current=current, + ) + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + first = threading.Thread(target=start, args=(self.session, self.admin)) + second = threading.Thread(target=start, args=(other_session, other_admin)) + first.start() + second.start() + first.join(timeout=10) + second.join(timeout=10) + other_session.close() + + assert not first.is_alive() + assert not second.is_alive() + assert errors == [] + verify = make_test_session(self.engine) + try: + bind_tasks = ( + verify.query(DouyuTask) + .filter( + DouyuTask.platform_task_id == task.id, + DouyuTask.task_type == "get_bind_qr", + ) + .all() + ) + assert len(bind_tasks) == 1 + finally: + verify.close() + def test_confirm_binding_pins_expected_role_for_two_role_snapshot( self, monkeypatch ): diff --git a/web/backend/database.py b/web/backend/database.py index 868d56f..bba89e2 100644 --- a/web/backend/database.py +++ b/web/backend/database.py @@ -71,6 +71,21 @@ if not DATABASE_URL.startswith("sqlite"): engine = create_engine(DATABASE_URL, **engine_options) +worker_engine_options = { + "connect_args": connect_args, + "echo": False, +} +if not DATABASE_URL.startswith("sqlite"): + worker_engine_options.update( + pool_pre_ping=True, + pool_size=_env_int("WORKER_DB_POOL_SIZE", 12), + max_overflow=_env_int("WORKER_DB_MAX_OVERFLOW", 4), + pool_timeout=_env_int("WORKER_DB_POOL_TIMEOUT", 30), + pool_recycle=_env_int("DB_POOL_RECYCLE", 1800), + ) + +worker_engine = create_engine(DATABASE_URL, **worker_engine_options) + if DATABASE_URL.startswith("sqlite"): @@ -82,6 +97,14 @@ if DATABASE_URL.startswith("sqlite"): cursor.execute("PRAGMA busy_timeout=30000") cursor.execute("PRAGMA foreign_keys=ON") cursor.close() + + @event.listens_for(worker_engine, "connect") + def _set_worker_sqlite_pragmas(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA busy_timeout=30000") + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() else: @event.listens_for(engine, "connect") @@ -96,8 +119,24 @@ else: finally: cursor.close() + @event.listens_for(worker_engine, "connect") + def _set_worker_mysql_session_guards(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + try: + cursor.execute( + "SET SESSION innodb_lock_wait_timeout = " + f"{_env_int('DB_LOCK_WAIT_TIMEOUT', 10)}" + ) + finally: + cursor.close() + SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False) +WorkerSessionLocal = sessionmaker( + bind=worker_engine, + autocommit=False, + autoflush=False, +) Base = declarative_base() diff --git a/web/backend/main.py b/web/backend/main.py index e495696..1c2277c 100644 --- a/web/backend/main.py +++ b/web/backend/main.py @@ -101,6 +101,10 @@ async def lifespan(app: FastAPI): register_default_jobs() start_scheduler() + from .services.binding_poll_scheduler import ( + start_binding_poll_scheduler, + stop_binding_poll_scheduler, + ) from .services.integration import ( start_integration_outbox_worker, stop_integration_outbox_worker, @@ -113,17 +117,21 @@ async def lifespan(app: FastAPI): start_platform_fulfillment_reconciler, stop_platform_fulfillment_reconciler, ) + from .services.task_runtime import shutdown as shutdown_task_runtime start_integration_outbox_worker() start_platform_fulfillment_reconciler() start_platform_fulfillment_auto_allocator() + start_binding_poll_scheduler() try: yield finally: + stop_binding_poll_scheduler() stop_platform_fulfillment_auto_allocator() stop_platform_fulfillment_reconciler() stop_integration_outbox_worker() stop_scheduler() + shutdown_task_runtime(wait=False) app = FastAPI( diff --git a/web/backend/migrations/versions/20261120_0101_douyu_task_durable_schedule.py b/web/backend/migrations/versions/20261120_0101_douyu_task_durable_schedule.py new file mode 100644 index 0000000..b586d0b --- /dev/null +++ b/web/backend/migrations/versions/20261120_0101_douyu_task_durable_schedule.py @@ -0,0 +1,145 @@ +"""add durable scheduling fields to douyu_tasks + +Long waits such as QR scan polling must survive a process restart without +holding a worker thread. The scheduler claims due rows by lane and next_run_at. +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261120_0101" +down_revision: str | None = "20261120_0100" +branch_labels: str | None = None +depends_on: str | None = None + +INDEX_NAME = "ix_douyu_tasks_lane_due" + + +def _columns() -> set[str]: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table("douyu_tasks"): + return set() + return {column["name"] for column in inspector.get_columns("douyu_tasks")} + + +def _indexes() -> set[str]: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table("douyu_tasks"): + return set() + return {index["name"] for index in inspector.get_indexes("douyu_tasks")} + + +def upgrade() -> None: + columns = _columns() + if "queue_lane" not in columns: + op.add_column( + "douyu_tasks", + sa.Column( + "queue_lane", + sa.String(length=32), + nullable=False, + server_default="", + ), + ) + if "next_run_at" not in columns: + op.add_column( + "douyu_tasks", + sa.Column("next_run_at", sa.DateTime(), nullable=True), + ) + if "run_deadline_at" not in columns: + op.add_column( + "douyu_tasks", + sa.Column("run_deadline_at", sa.DateTime(), nullable=True), + ) + if "last_run_at" not in columns: + op.add_column( + "douyu_tasks", + sa.Column("last_run_at", sa.DateTime(), nullable=True), + ) + if "attempt_count" not in columns: + op.add_column( + "douyu_tasks", + sa.Column( + "attempt_count", + sa.Integer(), + nullable=False, + server_default="0", + ), + ) + if "active_binding_key" not in columns: + op.add_column( + "douyu_tasks", + sa.Column( + "active_binding_key", + sa.String(length=64), + sa.Computed( + "CASE WHEN task_type = 'get_bind_qr' " + "AND platform_task_id IS NOT NULL " + "AND status IN ('planned', 'pending', 'running') " + "THEN CONCAT('bind:', platform_task_id) ELSE NULL END", + persisted=True, + ), + nullable=True, + ), + ) + # Keep the newest active QR task if legacy concurrency produced duplicates. + # Older runtime rows are superseded; historical executed tasks are untouched. + op.execute( + sa.text( + """ + UPDATE douyu_tasks AS task + JOIN ( + SELECT platform_task_id, MAX(id) AS keep_id + FROM douyu_tasks + WHERE task_type = 'get_bind_qr' + AND platform_task_id IS NOT NULL + AND status IN ('planned', 'pending', 'running') + GROUP BY platform_task_id + HAVING COUNT(*) > 1 + ) AS duplicate + ON duplicate.platform_task_id = task.platform_task_id + SET task.status = 'stopped', + task.message = '已被新的绑定二维码任务取代(迁移收口)', + task.finished_at = UTC_TIMESTAMP(), + task.next_run_at = NULL + WHERE task.task_type = 'get_bind_qr' + AND task.status IN ('planned', 'pending', 'running') + AND task.id <> duplicate.keep_id + """ + ) + ) + if INDEX_NAME not in _indexes(): + op.create_index( + INDEX_NAME, + "douyu_tasks", + ["queue_lane", "status", "next_run_at", "id"], + unique=False, + ) + indexes = _indexes() + if "uq_douyu_tasks_active_binding" not in indexes: + op.create_index( + "uq_douyu_tasks_active_binding", + "douyu_tasks", + ["active_binding_key"], + unique=True, + ) + + +def downgrade() -> None: + if "uq_douyu_tasks_active_binding" in _indexes(): + op.drop_index("uq_douyu_tasks_active_binding", table_name="douyu_tasks") + if INDEX_NAME in _indexes(): + op.drop_index(INDEX_NAME, table_name="douyu_tasks") + columns = _columns() + for name in ( + "active_binding_key", + "attempt_count", + "last_run_at", + "run_deadline_at", + "next_run_at", + "queue_lane", + ): + if name in columns: + op.drop_column("douyu_tasks", name) diff --git a/web/backend/models.py b/web/backend/models.py index 56f742d..f086ae5 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -360,6 +360,18 @@ class DouyuTask(Base): UTCDateTime, default=_utcnow, index=True ) finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + # Durable background scheduling. Waiting work is not an active worker. + queue_lane: Mapped[str] = mapped_column( + String(32), nullable=False, default="", server_default="" + ) + next_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + run_deadline_at: Mapped[datetime | None] = mapped_column( + UTCDateTime, nullable=True + ) + last_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) + attempt_count: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) platform_task_id: Mapped[int | None] = mapped_column( BigInteger, nullable=True, @@ -369,6 +381,19 @@ class DouyuTask(Base): account_assigned_to: Mapped[int | None] = mapped_column( ForeignKey("users.id"), nullable=True ) + # One active QR task per fulfillment session, enforced by MySQL even when + # two application workers race through the row lock boundary. + active_binding_key: Mapped[str | None] = mapped_column( + String(64), + Computed( + "CASE WHEN task_type = 'get_bind_qr' " + "AND platform_task_id IS NOT NULL " + "AND status IN ('planned', 'pending', 'running') " + "THEN CONCAT('bind:', platform_task_id) ELSE NULL END", + persisted=True, + ), + nullable=True, + ) __table_args__ = ( Index( @@ -388,6 +413,18 @@ class DouyuTask(Base): "handbook_scope", "id", ), + Index( + "ix_douyu_tasks_lane_due", + "queue_lane", + "status", + "next_run_at", + "id", + ), + Index( + "uq_douyu_tasks_active_binding", + "active_binding_key", + unique=True, + ), ) account: Mapped[Account] = relationship("Account", back_populates="douyu_tasks") diff --git a/web/backend/routers/automation.py b/web/backend/routers/automation.py index 0a5c8bf..3b16999 100644 --- a/web/backend/routers/automation.py +++ b/web/backend/routers/automation.py @@ -92,12 +92,24 @@ def list_jobs( # 巡检与重登是两个作业,但通过数据库联动:巡检把行标成 invalid,重登作业的 # 候选条件正是这个字段。这里把「当前有多少条等着重登」一并返回,页面才能把 # 这层上下游关系显示出来,而不是让人以为两个开关互不相干。 + from ..database import worker_engine from ..services.automation.jobs.ck_relogin import count_pending_relogin + from ..services.binding_poll_scheduler import ( + scheduler_status as binding_poll_status, + ) + from ..services.task_runtime import lane_stats pending_relogin = count_pending_relogin(db) return { "scheduler": scheduler_status(), + "runtime": { + "task_lanes": lane_stats(), + "worker_db_pool": getattr(worker_engine.pool, "status", lambda: "")() + if callable(getattr(worker_engine.pool, "status", None)) + else "", + "binding_poll": binding_poll_status(), + }, "pending_relogin": pending_relogin, "jobs": [ { diff --git a/web/backend/routers/cookies.py b/web/backend/routers/cookies.py index c30cc65..875b58d 100644 --- a/web/backend/routers/cookies.py +++ b/web/backend/routers/cookies.py @@ -27,7 +27,7 @@ from fastapi.responses import StreamingResponse from sqlalchemy import case, func, or_ from sqlalchemy.orm import Session, defer, joinedload -from ..database import SessionLocal, get_db +from ..database import WorkerSessionLocal, get_db from ..deps import get_current_user, require_permission from ..models import Account, AuditLog, LoginTask, User from ..models import ProxyConfig as ProxyConfigModel @@ -649,7 +649,7 @@ def _start_relogin_accounts( ) proxy = db.query(ProxyConfigModel).first() - thread_db = SessionLocal() + thread_db = WorkerSessionLocal() max_login_retries, max_total_time = get_relogin_limits() runner = LoginBatchRunner( db=thread_db, diff --git a/web/backend/routers/douyu.py b/web/backend/routers/douyu.py index d128baa..fd49336 100644 --- a/web/backend/routers/douyu.py +++ b/web/backend/routers/douyu.py @@ -28,7 +28,7 @@ from core.douyu import ( FishFinRechargeConfigError, ) -from ..database import SessionLocal, get_db +from ..database import WorkerSessionLocal, get_db from ..deps import authenticate_websocket, get_current_user, require_permission from ..models import ( Account, @@ -1321,7 +1321,7 @@ async def ws_douyu_logs(websocket: WebSocket, batch_id: str): if not user_has_permission(user, "douyu:task"): await websocket.close(code=1008, reason="无权限") return - db = SessionLocal() + db = WorkerSessionLocal() try: _require_batch_owner(db, user, batch_id) except HTTPException: diff --git a/web/backend/routers/huya/_common.py b/web/backend/routers/huya/_common.py index babbd8b..5136b6e 100644 --- a/web/backend/routers/huya/_common.py +++ b/web/backend/routers/huya/_common.py @@ -45,7 +45,7 @@ from core.huya.device_profile import ( ) from core.sms_provider import parse_sms_lines -from ...database import SessionLocal, get_db +from ...database import get_db from ...deps import authenticate_websocket, get_current_user, require_permission from ...models import ( HuyaAccount, diff --git a/web/backend/routers/huya/workbench.py b/web/backend/routers/huya/workbench.py index 8e22ab7..1bb137b 100644 --- a/web/backend/routers/huya/workbench.py +++ b/web/backend/routers/huya/workbench.py @@ -5,6 +5,7 @@ from fastapi import APIRouter, Response from core.huya.cookie_utils import cookie_value from core.huya.http_client import HuyaHttpClient +from ...database import WorkerSessionLocal from ...schemas import HuyaWorkbenchTransferToCollab from ...services.huya_points_center import ( fetch_points_center_catalog, @@ -339,7 +340,7 @@ async def create_task_batch( log_queue = asyncio.Queue() loop = asyncio.get_running_loop() - thread_db = SessionLocal() + thread_db = WorkerSessionLocal() runner = HuyaBatchRunner( db=thread_db, batch_id=batch_id, @@ -494,7 +495,7 @@ async def ws_huya_logs(websocket: WebSocket, batch_id: str): if not _has_huya_perm(user, "huya:task"): await websocket.close(code=1008, reason="无权限") return - db = SessionLocal() + db = WorkerSessionLocal() try: _require_huya_batch_owner(db, user, batch_id) except HTTPException: diff --git a/web/backend/routers/login.py b/web/backend/routers/login.py index b7229d1..005b1fd 100644 --- a/web/backend/routers/login.py +++ b/web/backend/routers/login.py @@ -17,7 +17,7 @@ from fastapi.responses import StreamingResponse from sqlalchemy import and_, func, or_ from sqlalchemy.orm import Session -from ..database import SessionLocal, get_db +from ..database import WorkerSessionLocal, get_db from ..deps import authenticate_websocket, get_current_user, require_permission from ..models import Account, LoginTask, User from ..models import ProxyConfig as ProxyConfigModel @@ -182,7 +182,7 @@ async def create_batch( loop = asyncio.get_running_loop() # 创建执行器(用独立的 DB 会话,因为在线程中运行) - thread_db = SessionLocal() + thread_db = WorkerSessionLocal() runner = LoginBatchRunner( db=thread_db, account_ids=valid_ids, @@ -687,7 +687,7 @@ async def ws_login_logs(websocket: WebSocket, batch_id: str): # 归属校验:非 view_all 只能订阅自己创建的批次(与虎牙批次一致)。 if not user_has_permission(user, "login:view_all"): - check_db = SessionLocal() + check_db = WorkerSessionLocal() try: owned = ( check_db.query(LoginTask.id) diff --git a/web/backend/routers/platform_fulfillment_integration.py b/web/backend/routers/platform_fulfillment_integration.py index 51d9593..01df94d 100644 --- a/web/backend/routers/platform_fulfillment_integration.py +++ b/web/backend/routers/platform_fulfillment_integration.py @@ -9,6 +9,7 @@ from hmac import compare_digest from typing import Any from fastapi import APIRouter, Depends, Header, HTTPException, Request +from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from starlette.requests import ClientDisconnect @@ -132,12 +133,16 @@ async def start_customer_binding( task_id: int, req: PlatformFulfillmentCustomerBindingAction, db: Session = Depends(get_db), -) -> dict[str, Any]: - return await service.customer_start_binding( +) -> JSONResponse: + payload = await service.customer_start_binding( db, task_id=task_id, force_refresh=req.force_refresh, ) + return JSONResponse( + status_code=202 if payload.get("accepted") else 200, + content=payload, + ) @router.get( diff --git a/web/backend/services/account_check_service.py b/web/backend/services/account_check_service.py index 0997024..1d1d2a2 100644 --- a/web/backend/services/account_check_service.py +++ b/web/backend/services/account_check_service.py @@ -6,7 +6,7 @@ import io import threading import uuid import zipfile -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import as_completed from dataclasses import dataclass, field from datetime import UTC, datetime from types import SimpleNamespace @@ -18,10 +18,11 @@ from core.douyu import DouyuLogin, create_login_api_strategy from core.douyu.login import AccountLike from core.douyu.proxy_fetcher import ProxyFetcher -from ..database import SessionLocal +from ..database import WorkerSessionLocal from ..models import ProxyConfig as ProxyConfigModel from .account_events_service import CATEGORY_AUTH, emit from .proxy_service import build_proxy_runtime +from .task_runtime import TaskLane, submit def _now() -> datetime: @@ -251,7 +252,7 @@ class AccountCheckRunner: status = str(item.get("status") or "") message = str(item.get("message") or "") try: - db = SessionLocal() + db = WorkerSessionLocal() try: emit( db, @@ -422,21 +423,29 @@ class AccountCheckRunner: if self._shared_proxy_fetcher: self._shared_proxy_fetcher.warmup_whitelist() - with ThreadPoolExecutor(max_workers=self.batch.concurrency) as executor: - futures = [] - for index, account in enumerate(self.accounts): - if self._stop.is_set(): - self._set_item( - index, - status="stopped", - message="已停止", - finished_at=_now(), - ) - continue - futures.append(executor.submit(self._run_one, index, account)) + futures = [] + for index, account in enumerate(self.accounts): + if self._stop.is_set(): + self._set_item( + index, + status="stopped", + message="已停止", + finished_at=_now(), + ) + continue + future = submit( + TaskLane.ACCOUNT_CHECK, + self._run_one, + index, + account, + stopped=self._stop.is_set, + ) + if future is None: + break + futures.append(future) - for future in as_completed(futures): - future.result() + for future in as_completed(futures): + future.result() except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 self._set_batch( status="error", message=f"批次执行异常: {exc}", finished_at=_now() diff --git a/web/backend/services/automation/jobs/ck_health.py b/web/backend/services/automation/jobs/ck_health.py index a707f32..6d5bdd2 100644 --- a/web/backend/services/automation/jobs/ck_health.py +++ b/web/backend/services/automation/jobs/ck_health.py @@ -9,7 +9,7 @@ from __future__ import annotations -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import as_completed from datetime import UTC, datetime, timedelta from typing import Any @@ -34,6 +34,7 @@ from ...douyu_account_probe import ( probe_session, ) from ...douyu_service import cookie_holding_filter, cookie_operable_filter +from ...task_runtime import TaskLane, submit JOB_KEY = "ck_health" FLUSH_EVERY = 25 @@ -146,11 +147,14 @@ def run_ck_health(db: Session, job: AutomationJob) -> dict[str, Any]: results: dict[int, dict[str, Any]] = {} concurrency = max(min(int(job.concurrency or 1), 16), 1) - with ThreadPoolExecutor(max_workers=concurrency) as pool: - futures = {pool.submit(_detect, account): account for account in accounts} - for future in as_completed(futures): - account = futures[future] - results[account.id] = future.result() + futures = {} + for account in accounts: + future = submit(TaskLane.ACCOUNT_CHECK, _detect, account) + if future is not None: + futures[future] = account + for future in as_completed(futures): + account = futures[future] + results[account.id] = future.result() # 只写检测结果三件套;cookie / status / attempt_* 一律不动。 valid = invalid = errors = 0 diff --git a/web/backend/services/automation/jobs/ck_relogin.py b/web/backend/services/automation/jobs/ck_relogin.py index 330bc9a..60e2773 100644 --- a/web/backend/services/automation/jobs/ck_relogin.py +++ b/web/backend/services/automation/jobs/ck_relogin.py @@ -20,7 +20,7 @@ from typing import Any from sqlalchemy import func, or_ from sqlalchemy.orm import Session -from ....database import SessionLocal +from ....database import WorkerSessionLocal from ....models import Account, AutomationJob, LoginTask, User from ....models import ProxyConfig as ProxyConfigModel from ...account_events_service import EVENT_RELOGIN_FAILED, emit @@ -148,7 +148,7 @@ def settle_fresh_results( 退避时间,其中 2 条还被标成「极验」进了待人工队列。SQLite 的读一致性与 MySQL 不同,这类问题在 SQLite 上复现不出来。 """ - session = SessionLocal() + session = WorkerSessionLocal() try: fresh, order = _load_batch_state(session, batch_id) counters = settle_results( @@ -241,7 +241,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: _, max_total_time = get_relogin_limits() # 批次用独立会话(runner 会在自己的线程/会话里写库),跑完再回来读结果。 - runner_db = SessionLocal() + runner_db = WorkerSessionLocal() runner = LoginBatchRunner( db=runner_db, account_ids=account_ids, diff --git a/web/backend/services/automation/scheduler.py b/web/backend/services/automation/scheduler.py index 18ea3bc..b34c6e3 100644 --- a/web/backend/services/automation/scheduler.py +++ b/web/backend/services/automation/scheduler.py @@ -24,7 +24,7 @@ from typing import Any from loguru import logger from sqlalchemy.orm import Session -from ...database import SessionLocal +from ...database import WorkerSessionLocal from ...models import AutomationJob, AutomationRun # 调度轮询间隔,也是作业的运行节奏:每个 tick 跑一轮。 @@ -71,7 +71,7 @@ _REGISTRY: dict[str, JobSpec] = {} #: 同时最多几个作业在跑。作业之间互不依赖,并行是为了**慢作业不挡住快作业** #: (以前串行:一个作业跑 4 分钟,CK 巡检就得等 4 分钟)。 -_JOB_MAX_WORKERS = 4 +_JOB_MAX_WORKERS = 2 #: 运行间隔的下限:太短会变成忙轮询,没有意义 MIN_RUN_INTERVAL_SECONDS = 30 @@ -190,7 +190,7 @@ def _run_job_isolated(job_key: str) -> dict[str, Any] | None: 多个作业共用一个 session 会互相踩事务(而他们现在是真的并行了)。 重新按 key 取行,是因为调度线程里的那行可能已经过期。 """ - db = SessionLocal() + db = WorkerSessionLocal() try: job = db.query(AutomationJob).filter(AutomationJob.key == job_key).first() spec = _REGISTRY.get(job_key) @@ -337,7 +337,7 @@ def _run_one(db: Session, job: AutomationJob, spec: JobSpec) -> dict[str, Any]: db.rollback() logger.exception("自动化作业 {} 提交状态失败,尝试独立连接回写", job.key) try: - rescue_db = SessionLocal() + rescue_db = WorkerSessionLocal() try: r = ( rescue_db.query(AutomationRun) @@ -412,7 +412,7 @@ def run_job_now(db: Session, key: str) -> dict[str, Any]: def _loop() -> None: - db = SessionLocal() + db = WorkerSessionLocal() try: ensure_jobs(db) except Exception: # noqa: BLE001 初始化失败不应让线程静默退出 @@ -424,7 +424,7 @@ def _loop() -> None: "自动化调度器已启动:心跳 {}s,作业周期 {}s", TICK_SECONDS, JOB_INTERVAL_SECONDS ) while not _stop_event.wait(TICK_SECONDS): - db = SessionLocal() + db = WorkerSessionLocal() try: # wait=False:派发完就返回 —— tick 不再被慢作业拖住 run_due_jobs(db, wait=False) diff --git a/web/backend/services/binding_poll_scheduler.py b/web/backend/services/binding_poll_scheduler.py new file mode 100644 index 0000000..9de2245 --- /dev/null +++ b/web/backend/services/binding_poll_scheduler.py @@ -0,0 +1,212 @@ +"""Database-backed scheduler for scan-wait binding tasks.""" + +from __future__ import annotations + +import os +import threading +from concurrent.futures import Future +from datetime import UTC, datetime, timedelta + +from loguru import logger +from sqlalchemy.orm import joinedload + +from ..database import WorkerSessionLocal +from ..models import DouyuTask +from .douyu_runner import DouyuBatchRunner +from .task_runtime import TaskLane, submit + +DEFAULT_INTERVAL_SECONDS = 1 +DEFAULT_BATCH_SIZE = 20 +CLAIM_SECONDS = 60 + +_stop_event = threading.Event() +_thread: threading.Thread | None = None +_lock = threading.Lock() +_in_flight: set[int] = set() +_in_flight_lock = threading.Lock() + + +def _env_int(name: str, default: int, minimum: int, maximum: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except ValueError: + return default + return max(minimum, min(maximum, value)) + + +def _claim_due_binding_tasks(limit: int) -> list[int]: + db = WorkerSessionLocal() + now = datetime.now(UTC) + claimed: list[int] = [] + try: + rows = ( + db.query(DouyuTask) + .filter( + DouyuTask.task_type == "get_bind_qr", + DouyuTask.status == "running", + DouyuTask.queue_lane == "binding_poll", + DouyuTask.next_run_at.isnot(None), + DouyuTask.next_run_at <= now, + ) + .order_by(DouyuTask.next_run_at.asc(), DouyuTask.id.asc()) + .with_for_update(skip_locked=True) + .limit(limit) + .all() + ) + for row in rows: + task_id = int(row.id) + with _in_flight_lock: + if task_id in _in_flight: + continue + _in_flight.add(task_id) + row.next_run_at = now + timedelta(seconds=CLAIM_SECONDS) + claimed.append(task_id) + db.commit() + return claimed + except Exception: + db.rollback() + with _in_flight_lock: + for task_id in claimed: + _in_flight.discard(task_id) + raise + finally: + db.close() + + +def poll_binding_task_once(task_id: int) -> None: + db = WorkerSessionLocal() + try: + task = ( + db.query(DouyuTask) + .options(joinedload(DouyuTask.account)) + .filter(DouyuTask.id == int(task_id)) + .first() + ) + if ( + task is None + or task.status != "running" + or task.task_type != "get_bind_qr" + or task.queue_lane != "binding_poll" + ): + return + account = task.account + if account is None or not (account.cookie or ""): + task.queue_lane = "" + task.next_run_at = None + runner = DouyuBatchRunner( + db, + task.batch_id, + "get_bind_qr", + close_db=False, + ) + runner._mark_task(db, task, "failed", "账号 Cookie 已失效,无法继续检测绑定") + return + runner = DouyuBatchRunner( + db, + task.batch_id, + "get_bind_qr", + close_db=False, + ) + config = runner._config_info(db) + runner.poll_bind_role_once(db, task, account, config) + except Exception as exc: # noqa: BLE001 - the claimed row is retried later + db.rollback() + logger.warning("绑定扫码定时检测失败 task_id={}: {}", task_id, exc) + finally: + db.close() + with _in_flight_lock: + _in_flight.discard(int(task_id)) + + +def _release_callback(_future: Future) -> None: + try: + _future.result() + except Exception as exc: # noqa: BLE001 - worker already logs; keep scheduler alive + logger.warning("绑定扫码任务异常: {}", exc) + + +def run_binding_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int: + claimed = _claim_due_binding_tasks(max(1, limit)) + submitted = 0 + for task_id in claimed: + future = submit( + TaskLane.BINDING_POLL, + poll_binding_task_once, + task_id, + stopped=_stop_event.is_set, + ) + if future is None: + with _in_flight_lock: + _in_flight.discard(task_id) + break + future.add_done_callback(_release_callback) + submitted += 1 + return submitted + + +def _run() -> None: + interval = _env_int( + "BINDING_POLL_DISPATCH_SECONDS", + DEFAULT_INTERVAL_SECONDS, + 1, + 30, + ) + while not _stop_event.is_set(): + try: + batch_size = _env_int( + "BINDING_POLL_DISPATCH_BATCH", + DEFAULT_BATCH_SIZE, + 1, + 200, + ) + run_binding_poll_dispatch_once(batch_size) + except Exception as exc: # noqa: BLE001 - scheduler must survive one bad tick + logger.warning("绑定扫码调度轮次失败: {}", exc) + _stop_event.wait(interval) + + +def start_binding_poll_scheduler() -> bool: + global _thread + with _lock: + if _thread is not None and _thread.is_alive(): + return False + _stop_event.clear() + _thread = threading.Thread( + target=_run, + name="binding-poll-scheduler", + daemon=True, + ) + _thread.start() + logger.info("绑定扫码定时调度器已启动") + return True + + +def stop_binding_poll_scheduler() -> None: + global _thread + _stop_event.set() + with _lock: + thread = _thread + _thread = None + if thread is not None and thread.is_alive(): + thread.join(timeout=3) + + +def scheduler_status() -> dict[str, object]: + with _lock: + running = _thread is not None and _thread.is_alive() + with _in_flight_lock: + in_flight = len(_in_flight) + return { + "running": running, + "in_flight": in_flight, + "claim_seconds": CLAIM_SECONDS, + } + + +__all__ = [ + "poll_binding_task_once", + "run_binding_poll_dispatch_once", + "scheduler_status", + "start_binding_poll_scheduler", + "stop_binding_poll_scheduler", +] diff --git a/web/backend/services/douyu_batch_launcher.py b/web/backend/services/douyu_batch_launcher.py index 165c58b..a8b53bc 100644 --- a/web/backend/services/douyu_batch_launcher.py +++ b/web/backend/services/douyu_batch_launcher.py @@ -9,7 +9,7 @@ import asyncio import os import threading -from web.backend.database import SessionLocal +from web.backend.database import WorkerSessionLocal from .douyu_concurrency import clamp_batch_task_concurrency from .douyu_runner import DouyuBatchRunner, douyu_batch_registry @@ -39,10 +39,15 @@ def start_douyu_batch( loop: asyncio.AbstractEventLoop | None = None, log_queue: asyncio.Queue | None = None, ) -> None: - """创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。""" - loop = loop or asyncio.get_running_loop() - log_queue = log_queue or asyncio.Queue() - thread_db = SessionLocal() + """Create the batch coordinator; websocket logging is optional.""" + if loop is None: + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + if log_queue is None and loop is not None: + log_queue = asyncio.Queue() + thread_db = WorkerSessionLocal() runner = DouyuBatchRunner( db=thread_db, batch_id=batch_id, diff --git a/web/backend/services/douyu_concurrency.py b/web/backend/services/douyu_concurrency.py index 10edc5f..895a765 100644 --- a/web/backend/services/douyu_concurrency.py +++ b/web/backend/services/douyu_concurrency.py @@ -1,7 +1,7 @@ -"""斗鱼后台任务的进程级并发限制。 +"""斗鱼批次级并发限制。 -任务线程和数据库连接池共享同一个 Uvicorn 进程。这里把「每批并发」和 -「全进程同时执行数」分开限制,避免多个批次叠加后把请求线程和连接池挤空。 +Process-wide execution is owned by ``task_runtime`` lanes; this module only +keeps the per-batch cap and the overview sub-limit. """ from __future__ import annotations @@ -23,16 +23,11 @@ DOUYU_BATCH_TASK_CONCURRENCY = _env_positive_int( "DOUYU_BATCH_TASK_CONCURRENCY", 5, ) -DOUYU_MAX_RUNNING_TASKS = _env_positive_int( - "DOUYU_MAX_RUNNING_TASKS", - 12, -) DOUYU_MAX_ACTIVE_OVERVIEW_TASKS = _env_positive_int( "DOUYU_MAX_ACTIVE_OVERVIEW_TASKS", 2, ) -_task_slots = threading.BoundedSemaphore(DOUYU_MAX_RUNNING_TASKS) _overview_slots = threading.BoundedSemaphore(DOUYU_MAX_ACTIVE_OVERVIEW_TASKS) @@ -47,15 +42,6 @@ def _acquire(slots: threading.BoundedSemaphore, stopped: Callable[[], bool]) -> return False -def acquire_running_task_slot(stopped: Callable[[], bool]) -> bool: - """等待一个全进程任务槽;停止时返回 False。""" - return _acquire(_task_slots, stopped) - - -def release_running_task_slot() -> None: - _task_slots.release() - - def acquire_overview_slot(stopped: Callable[[], bool]) -> bool: """等待一个一键查询槽;该限制跨所有批次共享。""" return _acquire(_overview_slots, stopped) @@ -68,10 +54,7 @@ def release_overview_slot() -> None: __all__ = [ "DOUYU_BATCH_TASK_CONCURRENCY", "DOUYU_MAX_ACTIVE_OVERVIEW_TASKS", - "DOUYU_MAX_RUNNING_TASKS", "acquire_overview_slot", - "acquire_running_task_slot", "clamp_batch_task_concurrency", "release_overview_slot", - "release_running_task_slot", ] diff --git a/web/backend/services/douyu_runner.py b/web/backend/services/douyu_runner.py index f5c0159..decda00 100644 --- a/web/backend/services/douyu_runner.py +++ b/web/backend/services/douyu_runner.py @@ -2,21 +2,20 @@ from __future__ import annotations -from concurrent.futures import ThreadPoolExecutor, as_completed +import time +from concurrent.futures import as_completed from sqlalchemy.orm import joinedload from core.douyu import DouyuActivityError -from ..database import SessionLocal +from ..database import WorkerSessionLocal from ..models import DouyuTask from .cookie_check_service import check_douyu_cookie from .douyu_account_probe import persist_probe_result, probe_session from .douyu_concurrency import ( acquire_overview_slot, - acquire_running_task_slot, release_overview_slot, - release_running_task_slot, ) from .douyu_runner_bind import BindMixin from .douyu_runner_core import ( # noqa: F401 (douyu_batch_registry 供 routers 重导出) @@ -30,6 +29,11 @@ from .douyu_runner_manual import ManualMixin from .douyu_runner_overview import OverviewMixin from .douyu_runner_xpd import XpdMixin from .douyu_service import update_account_profile_from_cookie +from .task_runtime import douyu_lane_for_task, submit + +# Compatibility alias for tests and integrations that replace the worker +# session factory at the runner boundary. +SessionLocal = WorkerSessionLocal class DouyuBatchRunner( @@ -47,19 +51,13 @@ class DouyuBatchRunner( def _execute_one(self, task_id: int, config: dict, total: int): overview_task = self.task_type == "query_account_overview" overview_slot = False - running_slot = False if overview_task: overview_slot = acquire_overview_slot(self._stop.is_set) if not overview_slot: return try: - running_slot = acquire_running_task_slot(self._stop.is_set) - if not running_slot: - return self._execute_one_with_slot(task_id, config, total) finally: - if running_slot: - release_running_task_slot() if overview_slot: release_overview_slot() @@ -221,19 +219,52 @@ class DouyuBatchRunner( self._push_task_event(task) total = len(tasks) - with ThreadPoolExecutor(max_workers=self.concurrency) as executor: - futures = [] - for task in tasks: - if self._stop.is_set(): - break - futures.append( - executor.submit(self._execute_one, task.id, config, total) + lane = douyu_lane_for_task(self.task_type) + pending: dict = {} + next_index = 0 + while next_index < total or pending: + while ( + not self._stop.is_set() + and next_index < total + and len(pending) < self.concurrency + ): + task = tasks[next_index] + future = submit( + lane, + self._execute_one, + task.id, + config, + total, + stopped=self._stop.is_set, ) - for future in as_completed(futures): + if future is None: + break + pending[future] = (int(task.id), time.monotonic()) + next_index += 1 + + if not pending: + break + + try: + completed = list(as_completed(pending, timeout=0.5)) + except TimeoutError: + continue + for future in completed: + task_id, submitted_at = pending.pop(future) try: future.result() except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 - self._push_log("error", f"Worker 异常: {exc}") + self._push_log( + "error", + f"Worker 异常 task_id={task_id}: {exc}", + ) + else: + waited = time.monotonic() - submitted_at + if waited >= 5: + self._push_log( + "info", + f"任务 task_id={task_id} 在 {lane.value} 槽排队 {waited:.1f}s", + ) if self._stop.is_set(): self._push_log("warning", f"斗鱼任务批次 {self.batch_id} 已停止") diff --git a/web/backend/services/douyu_runner_bind.py b/web/backend/services/douyu_runner_bind.py index 50df095..788fdea 100644 --- a/web/backend/services/douyu_runner_bind.py +++ b/web/backend/services/douyu_runner_bind.py @@ -632,6 +632,172 @@ class BindMixin: ) return "", result + @staticmethod + def _schedule_next_bind_poll( + task: DouyuTask, + *, + now: datetime, + interval_seconds: int, + ) -> None: + task.queue_lane = "binding_poll" + task.next_run_at = now + timedelta(seconds=max(1, interval_seconds)) + task.last_run_at = now + task.attempt_count = int(task.attempt_count or 0) + 1 + + def poll_bind_role_once( + self, + db: Session, + task: DouyuTask, + account: Account, + config: dict, + ) -> None: + """Run one scheduled binding poll and immediately release the worker.""" + now = datetime.now(UTC) + task.last_run_at = now + task.attempt_count = int(task.attempt_count or 0) + 1 + result = dict(task.result or {}) + query_aliases = [ + str(value) + for value in (result.get("query_act_aliases") or []) + if str(value or "").strip() + ] + if not query_aliases: + query_aliases = self._query_bind_act_aliases(config) + deadline = task.run_deadline_at + if deadline is not None and now >= deadline: + result.update( + { + "bind_polling": False, + "bind_ready_for_confirm": False, + "bind_phase": "role_timeout", + } + ) + task.queue_lane = "" + task.next_run_at = None + self._mark_task( + db, + task, + "failed", + "绑定二维码已过期,未识别到新扫码角色", + result, + ) + return + + client = self._client(str(account.cookie or "")) + candidates = self._fetch_bind_info_candidates(client, query_aliases) + if not candidates: + attempts = max(1, int(task.attempt_count or 1)) + backoff = min(30, max(DOUYU_BIND_ROLE_POLL_INTERVAL, attempts * 2)) + result["bind_poll_error"] = "所有 actAlias 查询绑定信息失败" + result["bind_polling"] = True + result["bind_phase"] = "waiting_scan" + self._schedule_next_bind_poll(task, now=now, interval_seconds=backoff) + self._update_task_progress( + db, + task, + "running", + "等待扫码绑定:角色查询暂时失败", + result, + sync_platform=False, + ) + return + + baseline_role_name = str(result.get("baseline_role_name") or "") + baseline_is_bound_act = bool(result.get("baseline_is_bound_act")) + baseline_info = result.get("before_bind_info") + if not isinstance(baseline_info, dict): + baseline_info = None + bind_info = ( + self._pick_bind_info( + candidates, + baseline_role_name=baseline_role_name, + baseline_is_bound_act=baseline_is_bound_act, + baseline_info=baseline_info, + prefer_pending=True, + prefer_aliases=query_aliases, + ) + or candidates[0] + ) + snapshot = self._bind_snapshot(bind_info) + is_pending_role = self._is_pending_role( + bind_info, + baseline_role_name=baseline_role_name, + baseline_is_bound_act=baseline_is_bound_act, + baseline_info=baseline_info, + ) + query_alias = str(bind_info.get("act_alias") or "") + summary = ( + f"act={query_alias or '-'} " + f"{self._format_bind_summary(bind_info, pending=is_pending_role)}" + ) + + if is_pending_role: + task.queue_lane = "" + task.next_run_at = None + result.update( + { + **snapshot, + "act_alias": query_alias, + "query_act_alias": query_alias, + "bind_ready_for_confirm": True, + "bind_confirmed": False, + "bind_phase": "role_ready", + "bind_polling": False, + "poll_count": int(result.get("poll_count") or 0) + 1, + "bind_summary": summary, + } + ) + account.updated_at = now + self._mark_task( + db, + task, + "success", + f"已识别角色: {snapshot['role_name']},待确认绑定", + result, + ) + return + + if snapshot["role_name"] and snapshot["is_bound_act"]: + result["current_role_name"] = snapshot["role_name"] + result["current_area_name"] = snapshot["area_name"] + result["current_plat_name"] = snapshot["plat_name"] + result.update( + { + "bind_info": bind_info, + "act_alias": query_alias, + "query_act_alias": query_alias, + "is_bound_act": snapshot["is_bound_act"], + "is_bound_role": snapshot["is_bound_role"], + "is_bound_account": snapshot["is_bound_account"], + "need_bind_act": snapshot["need_bind_act"], + "need_bind_role": snapshot["need_bind_role"], + "change_role_wait_time": snapshot["change_role_wait_time"], + "can_change_role": snapshot["can_change_role"], + "role_name": "", + "area_name": "", + "plat_name": "", + "bind_ready_for_confirm": False, + "bind_confirmed": False, + "bind_phase": "waiting_scan", + "bind_polling": True, + "poll_count": int(result.get("poll_count") or 0) + 1, + "bind_summary": summary, + } + ) + self._schedule_next_bind_poll( + task, + now=now, + interval_seconds=DOUYU_BIND_ROLE_POLL_INTERVAL, + ) + self._update_task_progress( + db, + task, + "running", + f"等待扫码绑定 ({summary})", + result, + sync_platform=False, + ) + def _execute_get_bind_qr( self, db: Session, task: DouyuTask, account: Account, cookie: str, config: dict ): @@ -763,40 +929,26 @@ class BindMixin: "qr_expires_at": ( qr_generated_at + timedelta(seconds=DOUYU_BIND_ROLE_POLL_SECONDS) ).isoformat(), + "baseline_role_name": current_role_name, + "baseline_is_bound_act": before_snapshot["is_bound_act"], + "poll_count": 0, } account.updated_at = datetime.now(UTC) - # 关键:先把二维码 progress 出去,前端 running 期间即可弹窗扫码。 + task.queue_lane = "binding_poll" + task.next_run_at = qr_generated_at + timedelta( + seconds=DOUYU_BIND_ROLE_POLL_INTERVAL + ) + task.run_deadline_at = qr_generated_at + timedelta( + seconds=DOUYU_BIND_ROLE_POLL_SECONDS + ) + task.last_run_at = qr_generated_at + task.attempt_count = 0 + # Publish the QR immediately, then return without holding a worker while + # the customer scans. The durable scheduler resumes role polling. self._update_task_progress( db, task, "running", "已生成绑定二维码,等待扫码绑定", result ) - role_name, result = self._wait_bind_role_result( - client, - db, - task, - account, - query_aliases, - result, - baseline_role_name=current_role_name, - baseline_is_bound_act=before_snapshot["is_bound_act"], - baseline_info=before, - ) - if role_name: - self._mark_task( - db, task, "success", f"已识别角色: {role_name},待确认绑定", result - ) - return - if result.get("bind_phase") == "stopped": - self._mark_task(db, task, "stopped", "任务已停止", result) - return - last_summary = str(result.get("bind_summary") or "") - timeout_msg = "已生成绑定二维码,未检测到新扫码角色" - if current_role_name: - timeout_msg = f"{timeout_msg}(当前仍是 {current_role_name})" - if last_summary: - timeout_msg = f"{timeout_msg} | {last_summary}" - self._mark_task(db, task, "success", timeout_msg, result) - def _execute_confirm_bind( self, db: Session, task: DouyuTask, account: Account, cookie: str, config: dict ): diff --git a/web/backend/services/douyu_runner_core.py b/web/backend/services/douyu_runner_core.py index 41a29e3..75b6507 100644 --- a/web/backend/services/douyu_runner_core.py +++ b/web/backend/services/douyu_runner_core.py @@ -228,12 +228,12 @@ class DouyuBatchRunnerCore: if not enabled: return try: - from ..database import SessionLocal + from ..database import WorkerSessionLocal from .platform_fulfillment.service import sync_task_from_binding_runner # 当前批次会话刚刚提交过任务结果。履约投影使用独立短会话,避免 # get_task_detail 留下的只读事务在下一轮外部 HTTP 轮询期间占连接。 - sync_db = SessionLocal() + sync_db = WorkerSessionLocal() try: sync_task_from_binding_runner(sync_db, douyu_task_id=int(douyu_task_id)) finally: @@ -339,6 +339,8 @@ class DouyuBatchRunnerCore: status: str, message: str, result: dict | None = None, + *, + sync_platform: bool = True, ) -> None: task_id = int(task.id) has_platform_task = task.platform_task_id is not None @@ -348,7 +350,8 @@ class DouyuBatchRunnerCore: task.result = slim_douyu_task_result_for_storage(result) db.commit() self._push_task_event(task) - self._sync_platform_fulfillment(task_id, enabled=has_platform_task) + if sync_platform: + self._sync_platform_fulfillment(task_id, enabled=has_platform_task) def _upsert_goods(self, db: Session, goods: list[dict]) -> None: upsert_goods_snapshot(db, goods) @@ -455,8 +458,8 @@ class DouyuBatchRegistry: def register( self, batch_id: str, - log_queue: asyncio.Queue, - loop: asyncio.AbstractEventLoop, + log_queue: asyncio.Queue | None, + loop: asyncio.AbstractEventLoop | None, runner: DouyuBatchRunner, ): self._batches[batch_id] = { diff --git a/web/backend/services/douyu_service.py b/web/backend/services/douyu_service.py index 3e3fd92..9d28151 100644 --- a/web/backend/services/douyu_service.py +++ b/web/backend/services/douyu_service.py @@ -7,7 +7,7 @@ import uuid from datetime import UTC, datetime from loguru import logger -from sqlalchemy import and_, func, select +from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session from core.douyu.cookie_utils import cookie_value @@ -468,10 +468,22 @@ def cleanup_orphan_douyu_tasks( elif active_batch_ids is not None and active_batch_ids: query = query.filter(~DouyuTask.batch_id.in_(list(active_batch_ids))) + now = datetime.now(UTC) + # Scan waits are durable. A restarted scheduler can resume them from the + # database, so startup cleanup must not turn them into stopped tasks. + query = query.filter( + or_( + DouyuTask.task_type != "get_bind_qr", + DouyuTask.queue_lane != "binding_poll", + DouyuTask.next_run_at.is_(None), + DouyuTask.run_deadline_at.is_(None), + DouyuTask.run_deadline_at <= now, + ) + ) + tasks = query.all() if not tasks: return 0 - now = datetime.now(UTC) account_ids = {int(task.account_id) for task in tasks if task.account_id} accounts = ( { diff --git a/web/backend/services/huya_register_runner.py b/web/backend/services/huya_register_runner.py index cca830f..5639aa2 100644 --- a/web/backend/services/huya_register_runner.py +++ b/web/backend/services/huya_register_runner.py @@ -4,7 +4,7 @@ from __future__ import annotations import threading import uuid -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import as_completed from dataclasses import dataclass, field from datetime import UTC, datetime @@ -13,7 +13,7 @@ from core.huya.auto_register import HuyaAutoRegisterResult, register_huya_with_s from core.huya.cookie_utils import normalize_huya_cookie from core.sms_provider import SmsLine -from ..database import SessionLocal +from ..database import WorkerSessionLocal from ..models import ( HuyaRegisterBatch as HuyaRegisterBatchModel, ) @@ -28,6 +28,7 @@ from ..models import ( ) from .huya_service import upsert_huya_cookie from .proxy_service import build_proxy_runtime +from .task_runtime import TaskLane, submit # 运行中状态:服务中断后视为未完成,可继续 RUNNING_ITEM_STATUSES = frozenset({"sending", "waiting", "changing", "logging"}) @@ -209,7 +210,7 @@ def load_batch_snapshot( batch_id: str, *, recover_interrupted: bool = True ) -> dict | None: """从数据库加载批次详情;若服务中断则标记为 interrupted。""" - db = SessionLocal() + db = WorkerSessionLocal() try: batch_row = ( db.query(HuyaRegisterBatchModel) @@ -258,7 +259,7 @@ def list_batch_summaries( ) -> list[dict]: """列出最近的注册批次摘要。live_batch_ids 中的 running 保持运行中。""" live = live_batch_ids or set() - db = SessionLocal() + db = WorkerSessionLocal() try: rows = ( db.query(HuyaRegisterBatchModel) @@ -335,7 +336,7 @@ def export_success_logs_text( limit: int = 5000, ) -> str: """从成功流水表导出 txt。""" - db = SessionLocal() + db = WorkerSessionLocal() try: query = db.query(HuyaRegisterSuccessLog).order_by( HuyaRegisterSuccessLog.id.asc() @@ -368,7 +369,7 @@ def list_success_logs( limit: int = 200, ) -> list[dict]: """列出成功流水(含密码,供管理端展示/导出)。""" - db = SessionLocal() + db = WorkerSessionLocal() try: query = db.query(HuyaRegisterSuccessLog).order_by( HuyaRegisterSuccessLog.id.desc() @@ -450,7 +451,7 @@ class HuyaRegisterRunner: """把批次汇总写回数据库。""" if not self.batch.db_id: return - db = SessionLocal() + db = WorkerSessionLocal() try: row = ( db.query(HuyaRegisterBatchModel) @@ -493,7 +494,7 @@ class HuyaRegisterRunner: item = self.batch.items[index] if not item.db_id: return - db = SessionLocal() + db = WorkerSessionLocal() try: row = ( db.query(HuyaRegisterItemModel) @@ -536,7 +537,7 @@ class HuyaRegisterRunner: self, index: int, result: HuyaAutoRegisterResult ) -> tuple[int | None, str, str]: """成功时:写账号 + 成功流水(成功一个写一条,立即可导出)。""" - db = SessionLocal() + db = WorkerSessionLocal() try: # upsert_huya_cookie 内部会 commit 一次 account = upsert_huya_cookie( @@ -737,22 +738,30 @@ class HuyaRegisterRunner: self.batch.message = f"代理白名单预热失败: {msg}" self._persist_batch_meta() - with ThreadPoolExecutor(max_workers=self.batch.concurrency) as executor: - futures = [] - for index in indices: - if self._stop.is_set(): - self._set_item( - index, - status="stopped", - message="已停止", - finished_at=_now(), - ) - continue - item = self.sms_lines[index] - futures.append(executor.submit(self._run_one, index, item)) + futures = [] + for index in indices: + if self._stop.is_set(): + self._set_item( + index, + status="stopped", + message="已停止", + finished_at=_now(), + ) + continue + item = self.sms_lines[index] + future = submit( + TaskLane.HUYA_REGISTER, + self._run_one, + index, + item, + stopped=self._stop.is_set, + ) + if future is None: + break + futures.append(future) - for future in as_completed(futures): - future.result() + for future in as_completed(futures): + future.result() except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 with self._lock: self.batch.status = "error" @@ -811,7 +820,7 @@ class HuyaRegisterRegistry: for index, item in enumerate(sms_lines) ] - db = SessionLocal() + db = WorkerSessionLocal() try: batch_row = HuyaRegisterBatchModel( batch_id=batch_id, @@ -871,7 +880,7 @@ class HuyaRegisterRegistry: started_at=_now(), ) # DB 同步为 running,避免返回 pending 导致前端误判 - db = SessionLocal() + db = WorkerSessionLocal() try: row = ( db.query(HuyaRegisterBatchModel) @@ -936,7 +945,7 @@ class HuyaRegisterRegistry: empty_msg = "没有可继续的未完成/失败条目" run_label = "续跑未完成" - db = SessionLocal() + db = WorkerSessionLocal() try: batch_row = ( db.query(HuyaRegisterBatchModel) diff --git a/web/backend/services/huya_runner.py b/web/backend/services/huya_runner.py index 6a4321f..29e7244 100644 --- a/web/backend/services/huya_runner.py +++ b/web/backend/services/huya_runner.py @@ -2,14 +2,14 @@ from __future__ import annotations -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import as_completed from datetime import UTC, datetime from sqlalchemy.orm import joinedload from core.huya.cookie_utils import normalize_huya_cookie -from ..database import SessionLocal +from ..database import WorkerSessionLocal from ..models import HuyaAccount, HuyaTask from .huya_runner_bind import BindMixin from .huya_runner_core import ( # noqa: F401 (huya_batch_registry 供 routers 重导出) @@ -20,6 +20,7 @@ from .huya_runner_elite import EliteMixin from .huya_runner_goods import GoodsMixin from .huya_runner_recharge import RechargeMixin from .huya_service import HUYA_CONFIG_FIELDS, ensure_huya_config, huya_config_value +from .task_runtime import TaskLane, submit class HuyaBatchRunner( @@ -34,7 +35,7 @@ class HuyaBatchRunner( def _execute_one( self, task_id: int, account_info: dict, config_info: dict, total: int ): - worker_db = SessionLocal() + worker_db = WorkerSessionLocal() try: task = worker_db.query(HuyaTask).filter(HuyaTask.id == task_id).first() account = ( @@ -193,27 +194,29 @@ class HuyaBatchRunner( self._push_log("result", "") return - with ThreadPoolExecutor(max_workers=self.concurrency) as executor: - futures = [] - for item in task_infos: - if self._stop.is_set(): - self._push_log("warning", "任务已停止,跳过剩余账号") - break - futures.append( - executor.submit( - self._execute_one, - item["task_id"], - item["account_info"], - config_info, - total, - ) - ) + futures = [] + for item in task_infos: + if self._stop.is_set(): + self._push_log("warning", "任务已停止,跳过剩余账号") + break + future = submit( + TaskLane.HUYA_BATCH, + self._execute_one, + item["task_id"], + item["account_info"], + config_info, + total, + stopped=self._stop.is_set, + ) + if future is None: + break + futures.append(future) - for future in as_completed(futures): - try: - future.result() - except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 - self._push_log("error", f"虎牙 Worker 异常: {exc}") + for future in as_completed(futures): + try: + future.result() + except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 + self._push_log("error", f"虎牙 Worker 异常: {exc}") self._push_log("info", f"虎牙批次 {self.batch_id} 完成") self._push_log("result", "") diff --git a/web/backend/services/huya_runner_core.py b/web/backend/services/huya_runner_core.py index 3982e92..206cce3 100644 --- a/web/backend/services/huya_runner_core.py +++ b/web/backend/services/huya_runner_core.py @@ -90,10 +90,10 @@ class HuyaBatchRunnerCore: account_id = getattr(self._wss_sessions, "account_id", None) if not account_id: return - from ..database import SessionLocal + from ..database import WorkerSessionLocal from .huya_service import persist_huya_session_cookie - db = SessionLocal() + db = WorkerSessionLocal() try: if persist_huya_session_cookie( db, int(account_id), original_cookie, filled_cookie diff --git a/web/backend/services/integration/outbox.py b/web/backend/services/integration/outbox.py index b473037..29c7ef5 100644 --- a/web/backend/services/integration/outbox.py +++ b/web/backend/services/integration/outbox.py @@ -14,9 +14,10 @@ from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen from loguru import logger +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session -from ...database import SessionLocal +from ...database import WorkerSessionLocal from ...models import IntegrationOutbox from .security import build_webhook_signature @@ -74,19 +75,29 @@ def enqueue_integration_event( if existing is not None: return False - db.add( - IntegrationOutbox( - event_id=event_id, - destination=destination, - event_type=event_type, - dedupe_key=normalized_dedupe, - payload_json=body, - status="pending", - attempts=0, - next_attempt_at=datetime.now(UTC), - created_at=datetime.now(UTC), - ) - ) + try: + with db.begin_nested(): + db.add( + IntegrationOutbox( + event_id=event_id, + destination=destination, + event_type=event_type, + dedupe_key=normalized_dedupe, + payload_json=body, + status="pending", + attempts=0, + next_attempt_at=datetime.now(UTC), + created_at=datetime.now(UTC), + ) + ) + db.flush() + except IntegrityError: + # A concurrent transaction may have inserted the same stable dedupe key + # after our pre-check. The nested transaction keeps the outer business + # transaction usable and treats the event as already enqueued. + if normalized_dedupe: + return False + raise return True @@ -118,7 +129,7 @@ def _env_int(name: str, default: int, minimum: int, maximum: int) -> int: def _claim_due_events(limit: int) -> list[_ClaimedEvent]: - db = SessionLocal() + db = WorkerSessionLocal() now = datetime.now(UTC) try: ( @@ -202,7 +213,7 @@ def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None: def _mark_sent(event_id: str) -> None: - db = SessionLocal() + db = WorkerSessionLocal() try: ( db.query(IntegrationOutbox) @@ -229,7 +240,7 @@ def _mark_failed(event: _ClaimedEvent, error: Exception) -> None: ) dead = attempts >= max_attempts message = str(error)[:2000] - db = SessionLocal() + db = WorkerSessionLocal() try: ( db.query(IntegrationOutbox) diff --git a/web/backend/services/login_service.py b/web/backend/services/login_service.py index 930401e..7ae5618 100644 --- a/web/backend/services/login_service.py +++ b/web/backend/services/login_service.py @@ -5,7 +5,7 @@ import os import threading import time import uuid -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import as_completed from datetime import UTC, datetime from types import SimpleNamespace from typing import cast @@ -32,6 +32,7 @@ from .douyu_account_probe import persist_probe_result, probe_session from .douyu_service import LOGIN_ATTEMPT_IN_FLIGHT_STATUSES from .login_failure import classify_failure, failure_kind_label, is_account_fault from .proxy_service import build_proxy_runtime +from .task_runtime import TaskLane, submit CHECK_STATUS_MESSAGES = { "account_cancelled": "账号已注销", @@ -296,7 +297,7 @@ def _persist_login_cookie_verification( class LoginBatchRunner: - """批量登录执行器,在线程中运行,通过 ThreadPoolExecutor 并发登录多个账号。""" + """批量登录执行器,通过进程级登录槽并发执行账号。""" def __init__( self, @@ -357,7 +358,7 @@ class LoginBatchRunner: message: str = "重新登录已停止,旧 Cookie 已保留", ) -> None: """把未开始或已中断的重登尝试收敛为终态,旧 CK 原封不动。""" - worker_db = SessionLocal() + worker_db = WorkerSessionLocal() try: task = worker_db.query(LoginTask).filter(LoginTask.id == task_id).first() if task and _attempt_in_flight(task): @@ -376,7 +377,7 @@ class LoginBatchRunner: 与 `_mark_relogin_stopped` 的区别只是入口:批次挂了但记录还没建出来时, 这里是唯一能把「该账号的在途尝试」收干净的路径。 """ - worker_db = SessionLocal() + worker_db = WorkerSessionLocal() try: tasks = ( worker_db.query(LoginTask) @@ -432,7 +433,7 @@ class LoginBatchRunner: self._mark_relogin_stopped(task_id, "任务已停止,旧 Cookie 已保留") return - worker_db = SessionLocal() + worker_db = WorkerSessionLocal() try: task = worker_db.query(LoginTask).filter(LoginTask.id == task_id).first() if not task: @@ -774,33 +775,33 @@ class LoginBatchRunner: self._push_log("result", "") return - # 并发执行登录,每个账号独立获取代理 - with ThreadPoolExecutor(max_workers=concurrency) as executor: - futures = [] - for item_index, item in enumerate(task_infos): - if self._stop.is_set(): - self._push_log("warning", "任务已停止,跳过剩余账号") - # 所有模式都要把没轮到的尝试收敛掉,否则会一直停在 - # 「等待中」;已有 CK 的记录本身不受影响。 - for pending_item in task_infos[item_index:]: - self._mark_relogin_stopped( - pending_item["task_id"], "任务已停止,旧 Cookie 已保留" - ) - break - future = executor.submit( - self._execute_one, - item["task_id"], - item["acc_info"], - total, - ) - futures.append(future) + # 全局登录槽负责并发;批次只提交任务并收集结果。 + futures = [] + for item_index, item in enumerate(task_infos): + if self._stop.is_set(): + self._push_log("warning", "任务已停止,跳过剩余账号") + for pending_item in task_infos[item_index:]: + self._mark_relogin_stopped( + pending_item["task_id"], "任务已停止,旧 Cookie 已保留" + ) + break + future = submit( + TaskLane.LOGIN, + self._execute_one, + item["task_id"], + item["acc_info"], + total, + stopped=self._stop.is_set, + ) + if future is None: + break + futures.append(future) - # 等待所有任务完成 - for future in as_completed(futures): - try: - future.result() - except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 - self._push_log("error", f"Worker 异常: {e}") + for future in as_completed(futures): + try: + future.result() + except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底 + self._push_log("error", f"Worker 异常: {e}") self._push_log("info", f"批量{action_name}任务 {batch_id} 完成") self._push_log("result", "") @@ -902,5 +903,5 @@ def cleanup_orphan_login_tasks( return _converge_interrupted_attempts(db, message) -# 在模块末尾导入 SessionLocal(避免循环导入) -from ..database import SessionLocal +# 在模块末尾导入 WorkerSessionLocal(避免循环导入) +from ..database import WorkerSessionLocal diff --git a/web/backend/services/platform_fulfillment/auto_allocator.py b/web/backend/services/platform_fulfillment/auto_allocator.py index 9eb6267..89e8fb8 100644 --- a/web/backend/services/platform_fulfillment/auto_allocator.py +++ b/web/backend/services/platform_fulfillment/auto_allocator.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import os import threading from collections.abc import Iterable @@ -12,7 +13,7 @@ from loguru import logger from sqlalchemy import or_ from sqlalchemy.orm import Session -from ...database import SessionLocal +from ...database import WorkerSessionLocal from ...models import PlatformFulfillmentTask from . import service @@ -142,6 +143,21 @@ def run_auto_allocation_once( if allocated_task is not None: allocated_task.next_allocate_at = None db.commit() + try: + asyncio.run( + service.start_binding( + db, + task_id=candidate_task_id, + current=actor, + ) + ) + except Exception as exc: # noqa: BLE001 - allocation remains valid + db.rollback() + logger.warning( + "直播账号履约自动开始绑定失败 task_id={}: {}", + candidate_task_id, + exc, + ) else: failed += 1 _schedule_retry( @@ -171,6 +187,48 @@ def run_auto_allocation_once( } +def start_allocated_bindings_once( + db: Session, + *, + limit: int = DEFAULT_BATCH_SIZE, +) -> int: + """Start durable binding preparation for allocated, unstarted sessions.""" + rows = ( + db.query(PlatformFulfillmentTask) + .filter( + PlatformFulfillmentTask.execution_status == "allocated", + PlatformFulfillmentTask.binding_status == "not_started", + PlatformFulfillmentTask.account_id.isnot(None), + PlatformFulfillmentTask.lease_id.isnot(None), + ) + .order_by(PlatformFulfillmentTask.updated_at.asc(), PlatformFulfillmentTask.id.asc()) + .limit(max(1, limit)) + .all() + ) + if not rows: + return 0 + actor = service._integration_actor(db) + started = 0 + for task in rows: + try: + asyncio.run( + service.start_binding( + db, + task_id=int(task.id), + current=actor, + ) + ) + started += 1 + except Exception as exc: # noqa: BLE001 - one session cannot block the rest + db.rollback() + logger.warning( + "直播账号履约补启动绑定失败 task_id={}: {}", + task.id, + exc, + ) + return started + + def _run() -> None: interval = _env_int( "PLATFORM_FULFILLMENT_AUTO_ALLOCATE_SECONDS", @@ -179,7 +237,7 @@ def _run() -> None: 3600, ) while not _stop_event.is_set(): - db = SessionLocal() + db = WorkerSessionLocal() try: pending_ids = _take_pending_task_ids( _env_int( @@ -190,8 +248,11 @@ def _run() -> None: ) ) result = run_auto_allocation_once(db, task_ids=pending_ids or None) + started = start_allocated_bindings_once(db) if result["checked"]: logger.info("直播账号履约自动选号: {}", result) + if started: + logger.info("直播账号履约补启动绑定: {} 条", started) except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出 db.rollback() logger.warning("直播账号履约自动选号轮次失败: {}", exc) diff --git a/web/backend/services/platform_fulfillment/order_site_notifier.py b/web/backend/services/platform_fulfillment/order_site_notifier.py index 5ae2cef..016b688 100644 --- a/web/backend/services/platform_fulfillment/order_site_notifier.py +++ b/web/backend/services/platform_fulfillment/order_site_notifier.py @@ -29,6 +29,8 @@ def _state_dedupe_key(payload: dict[str, Any]) -> str: "platName": payload.get("platName"), "bindReadyForConfirm": payload.get("bindReadyForConfirm"), "bindConfirmed": payload.get("bindConfirmed"), + "queueState": payload.get("queueState"), + "queueLane": payload.get("queueLane"), "customerProfileReset": payload.get("customerProfileReset"), } digest = hashlib.sha256( diff --git a/web/backend/services/platform_fulfillment/reconciler.py b/web/backend/services/platform_fulfillment/reconciler.py index fba2f44..0bda9e4 100644 --- a/web/backend/services/platform_fulfillment/reconciler.py +++ b/web/backend/services/platform_fulfillment/reconciler.py @@ -7,7 +7,7 @@ import threading from loguru import logger -from ...database import SessionLocal +from ...database import WorkerSessionLocal _stop_event = threading.Event() _thread: threading.Thread | None = None @@ -31,7 +31,7 @@ def _run() -> None: # 不抢占启动时的 Webhook;首轮延后一分钟。 if _stop_event.wait(min(60, _interval_seconds())): break - db = SessionLocal() + db = WorkerSessionLocal() try: queued = 0 for _page in range(20): diff --git a/web/backend/services/platform_fulfillment/service.py b/web/backend/services/platform_fulfillment/service.py index 1a415df..281ccd7 100644 --- a/web/backend/services/platform_fulfillment/service.py +++ b/web/backend/services/platform_fulfillment/service.py @@ -12,6 +12,7 @@ from typing import Any from fastapi import HTTPException from loguru import logger from sqlalchemy import or_ +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from ...models import ( @@ -763,6 +764,19 @@ def _load_task(db: Session, task_id: int) -> PlatformFulfillmentTask: return task +def _load_task_for_update(db: Session, task_id: int) -> PlatformFulfillmentTask: + task = ( + db.query(PlatformFulfillmentTask) + .filter(PlatformFulfillmentTask.id == int(task_id)) + .populate_existing() + .with_for_update() + .first() + ) + if task is None: + raise HTTPException(status_code=404, detail="直播账号履约任务不存在") + return task + + def _retain_lease_for_active_collab( db: Session, *, @@ -1241,6 +1255,40 @@ def _binding_failure_issue(bind_task: DouyuTask, result: dict[str, Any]) -> str: return task_issue if messages.get(task_issue) else "binding_failed" +def _customer_binding_queue_state( + task: PlatformFulfillmentTask, + bind_task: DouyuTask | None, + *, + lease_ready: bool, +) -> tuple[str, int]: + if task.execution_status == "completed": + return "completed", 0 + if task.execution_status == "cancelled": + return "cancelled", 0 + if task.issue_code == "mapping_missing": + return "mapping_missing", 0 + if not task.account_id or not task.lease_id: + return "allocation_queued", 2000 + if not lease_ready: + return "lease_inactive", 0 + if bind_task is None: + return "ready_to_start", 0 + result = bind_task.result or {} + if bool(result.get("bind_confirmed")) or task.binding_status == "bound": + return "confirmed", 0 + if pending_binding_role_from_result(result): + return "role_ready", 0 + if bind_task.status in {"failed", "error", "stopped"}: + return "failed", 0 + if task.binding_status == "confirming": + return "confirming", 1000 + if bind_task.status in {"planned", "pending"}: + return "qr_queued", 1000 + if result.get("url"): + return "waiting_scan", 5000 if result.get("bind_polling") else 0 + return "preparing", 1000 + + def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]: """面向前端客户页的安全绑定快照,不包含账号、CK、积分和租约。""" task = _load_task(db, task_id) @@ -1259,6 +1307,11 @@ def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]: lease_ready = bool(task.account_id and lease and lease.status == "active") binding_failed = task.binding_status == "failed" binding_confirmed = task.binding_status == "bound" + queue_state, retry_after_ms = _customer_binding_queue_state( + task, + bind_task, + lease_ready=lease_ready, + ) return { "sessionId": task.id, "platformOrderId": task.platform_order_id, @@ -1282,6 +1335,10 @@ def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]: "platName": str(bind_result.get("plat_name") or ""), "bindReadyForConfirm": (bool(pending_role) and not binding_failed), "bindConfirmed": bool(bind_result.get("bind_confirmed")) and binding_confirmed, + "queueState": queue_state, + "retryAfterMs": retry_after_ms, + "queueLane": bind_task.queue_lane if bind_task else "", + "accepted": False, "stateVersion": int(task.state_version or 0), "completedAt": _jsonable(task.completed_at), "updatedAt": _jsonable(task.updated_at), @@ -1318,8 +1375,9 @@ def ensure_customer_session( if task.execution_status in {"pending_mapping", "ready", "manual_required"}: if task.issue_code == "mapping_missing": return customer_binding_out(db, task.id) - actor = _integration_actor(db) - allocate_task(db, task_id=task.id, current=actor) + from .auto_allocator import wake_auto_allocator + + wake_auto_allocator([int(task.id)]) return customer_binding_out(db, task.id) @@ -1330,18 +1388,40 @@ async def customer_start_binding( force_refresh: bool = False, ) -> dict[str, Any]: task = _load_task(db, task_id) + if task.execution_status in {"completed", "cancelled"}: + return customer_binding_out(db, task.id) + if task.issue_code == "mapping_missing": + return customer_binding_out(db, task.id) + + if not task.account_id or not task.lease_id: + from .auto_allocator import wake_auto_allocator + + wake_auto_allocator([int(task.id)]) + payload = customer_binding_out(db, task.id) + payload["accepted"] = True + payload["queueState"] = "allocation_queued" + payload["retryAfterMs"] = 2000 + return payload + + lease = db.query(AccountLease).filter(AccountLease.id == task.lease_id).first() + if lease is None or lease.status != "active": + return customer_binding_out(db, task.id) + + previous_bind_task_id = task.bind_task_id actor = _integration_actor(db) - if task.execution_status not in {"completed", "cancelled"} and ( - not task.account_id or not task.lease_id - ): - allocate_task(db, task_id=task.id, current=actor) await start_binding( db, task_id=task.id, current=actor, force_refresh=force_refresh, ) - return customer_binding_out(db, task.id) + payload = customer_binding_out(db, task.id) + accepted = task.bind_task_id != previous_bind_task_id + payload["accepted"] = accepted + if accepted: + payload["queueState"] = "qr_queued" + payload["retryAfterMs"] = 1000 + return payload def customer_refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]: @@ -1881,7 +1961,7 @@ async def start_binding( current: User, force_refresh: bool = False, ) -> dict[str, Any]: - task = _load_task(db, task_id) + task = _load_task_for_update(db, task_id) if not task.account_id or not task.lease_id: raise HTTPException(status_code=409, detail="请先分配账号") lease = db.query(AccountLease).filter(AccountLease.id == task.lease_id).first() @@ -1924,9 +2004,16 @@ async def start_binding( message="等待生成绑定二维码", result={"payload": {}}, created_by=current.id, + platform_task_id=task.id, ) db.add(bind_task) - db.flush() + try: + db.flush() + except IntegrityError: + # The database uniqueness gate wins a race that started before either + # request could acquire the fulfillment row lock. + db.rollback() + return get_task_detail(db, task_id) account = db.query(Account).filter(Account.id == task.account_id).first() if account is not None: record_douyu_task_started( @@ -2113,7 +2200,7 @@ def reconcile_binding_state_outbox( async def confirm_binding( db: Session, *, task_id: int, current: User ) -> dict[str, Any]: - task = _load_task(db, task_id) + task = _load_task_for_update(db, task_id) task = _load_task(db, refresh_binding(db, task_id=task.id)["task"]["id"]) if task.binding_status != "role_ready": raise HTTPException(status_code=409, detail="当前没有待确认的游戏角色") @@ -2146,6 +2233,7 @@ async def confirm_binding( "baseline_role_name": baseline_role_name, }, created_by=current.id, + platform_task_id=task.id, ) db.add(confirm_task) db.flush() diff --git a/web/backend/services/task_runtime.py b/web/backend/services/task_runtime.py new file mode 100644 index 0000000..62f54eb --- /dev/null +++ b/web/backend/services/task_runtime.py @@ -0,0 +1,195 @@ +"""Bounded, process-local execution lanes for background work. + +Each lane owns its workers and admission budget. A slow or saturated domain can +therefore no longer consume every worker slot in the process. This runtime is +intended for the current single-Uvicorn deployment; durable scheduling remains +in the database. +""" + +from __future__ import annotations + +import os +import threading +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from enum import StrEnum +from typing import Any + + +class TaskLane(StrEnum): + INTERACTIVE_BINDING = "interactive_binding" + BINDING_POLL = "binding_poll" + LOGIN = "login" + ACCOUNT_CHECK = "account_check" + AUTOMATION = "automation" + QUERY_BATCH = "query_batch" + COLLAB_RECOMMEND = "collab_recommend" + PAYMENT_WAIT = "payment_wait" + OPS_WRITE = "ops_write" + HUYA_BATCH = "huya_batch" + HUYA_REGISTER = "huya_register" + INTEGRATION = "integration" + MAINTENANCE = "maintenance" + + +def _env_int(name: str, default: int, minimum: int = 1) -> int: + try: + value = int(os.getenv(name, str(default))) + except ValueError: + return default + return max(minimum, value) + + +@dataclass(frozen=True) +class LaneConfig: + workers: int + max_in_flight: int + + +_DEFAULT_WORKERS: dict[TaskLane, int] = { + TaskLane.INTERACTIVE_BINDING: 3, + TaskLane.BINDING_POLL: 2, + TaskLane.LOGIN: 4, + TaskLane.ACCOUNT_CHECK: 4, + TaskLane.AUTOMATION: 2, + TaskLane.QUERY_BATCH: 3, + TaskLane.COLLAB_RECOMMEND: 2, + TaskLane.PAYMENT_WAIT: 2, + TaskLane.OPS_WRITE: 2, + TaskLane.HUYA_BATCH: 2, + TaskLane.HUYA_REGISTER: 2, + TaskLane.INTEGRATION: 2, + TaskLane.MAINTENANCE: 1, +} + + +def _lane_config(lane: TaskLane) -> LaneConfig: + key = lane.value.upper() + workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane]) + max_in_flight = _env_int( + f"TASK_LANE_{key}_MAX_IN_FLIGHT", + max(workers * 32, workers), + ) + return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers)) + + +@dataclass +class _LaneRuntime: + config: LaneConfig + executor: ThreadPoolExecutor + admission: threading.BoundedSemaphore + + +_LOCK = threading.RLock() +_RUNTIMES: dict[TaskLane, _LaneRuntime] = {} + + +def _runtime(lane: TaskLane) -> _LaneRuntime: + with _LOCK: + runtime = _RUNTIMES.get(lane) + if runtime is not None: + return runtime + config = _lane_config(lane) + runtime = _LaneRuntime( + config=config, + executor=ThreadPoolExecutor( + max_workers=config.workers, + thread_name_prefix=f"lane-{lane.value}", + ), + admission=threading.BoundedSemaphore(config.max_in_flight), + ) + _RUNTIMES[lane] = runtime + return runtime + + +def submit[T]( + lane: TaskLane, + fn: Callable[..., T], + /, + *args: Any, + stopped: Callable[[], bool] | None = None, + **kwargs: Any, +) -> Future[T] | None: + """Submit work to a lane, waiting for admission when the lane is full. + + Batch coordinators are cheap threads and are allowed to block here. Worker + threads themselves must not recursively submit to the same lane. + """ + runtime = _runtime(lane) + while True: + if stopped is not None and stopped(): + return None + if runtime.admission.acquire(timeout=0.2): + break + + def run() -> T: + try: + return fn(*args, **kwargs) + finally: + runtime.admission.release() + + return runtime.executor.submit(run) + + +def lane_stats() -> dict[str, dict[str, int]]: + with _LOCK: + runtimes = list(_RUNTIMES.items()) + stats: dict[str, dict[str, int]] = {} + for lane, runtime in runtimes: + stats[lane.value] = { + "workers": runtime.config.workers, + "max_in_flight": runtime.config.max_in_flight, + "admission_value": runtime.admission._value, + } + return stats + + +def shutdown(wait: bool = False) -> None: + with _LOCK: + runtimes = list(_RUNTIMES.values()) + _RUNTIMES.clear() + for runtime in runtimes: + runtime.executor.shutdown(wait=wait, cancel_futures=False) + + +def douyu_lane_for_task(task_type: str) -> TaskLane: + if task_type in { + "get_bind_qr", + "confirm_bind", + "prepare_esports_bind", + "get_esports_bind_qr", + "confirm_esports_bind", + "get_xpd_bind_qr", + "query_xpd_bind_info", + "confirm_xpd_bind", + "query_xpd_role", + }: + return TaskLane.INTERACTIVE_BINDING + if task_type in { + "create_elite_qr", + "create_esports_qr", + "create_gold_qr", + }: + return TaskLane.PAYMENT_WAIT + if task_type in { + "lock_goods", + "pay_locked_order", + "exchange_goods", + "exchange_esports_goods", + "exchange_xpd_goods", + "donate_elite_gift", + "donate_esports_chicken_gift", + "donate_esports_firework_gift", + }: + return TaskLane.OPS_WRITE + return TaskLane.QUERY_BATCH + + +__all__ = [ + "TaskLane", + "douyu_lane_for_task", + "lane_stats", + "shutdown", + "submit", +]