fix(platform): 防止绑定任务压垮连接池和事件循环

This commit is contained in:
yml
2026-11-21 15:22:46 +08:00
parent d199e04d27
commit e1d979a296
13 changed files with 336 additions and 46 deletions
+6
View File
@@ -82,14 +82,18 @@ DOUYU_MAX_ACTIVE_BATCHES=10
# 一键查询使用独立任务槽;普通查询仍为 3 个 worker。
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS=5
# 业务域任务槽。支付出码与到账轮询分离后,各自按实际耗时独立限流。
TASK_LANE_BATCH_COORDINATOR_WORKERS=2
TASK_LANE_BATCH_COORDINATOR_MAX_IN_FLIGHT=2
TASK_LANE_INTERACTIVE_BINDING_WORKERS=3
TASK_LANE_BINDING_POLL_WORKERS=2
TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT=2
TASK_LANE_LOGIN_WORKERS=4
TASK_LANE_ACCOUNT_CHECK_WORKERS=4
TASK_LANE_OVERVIEW_QUERY_WORKERS=5
TASK_LANE_QUERY_BATCH_WORKERS=3
TASK_LANE_PAYMENT_QR_WORKERS=10
TASK_LANE_PAYMENT_POLL_WORKERS=3
TASK_LANE_PAYMENT_POLL_MAX_IN_FLIGHT=3
TASK_LANE_OPS_WRITE_WORKERS=2
DOUYU_MAX_ACTIVE_TASKS=250
DOUYU_PROBE_CACHE_SECONDS=30
@@ -191,6 +195,8 @@ PLATFORM_FULFILLMENT_RECONCILE_SECONDS=900
ACCOUNT_COMMODITY_STATE_TTL_SECONDS=900
# legacy=旧实时扫号 / hybrid=能力索引优先、失败回退旧逻辑 / capability=只读能力索引
PLATFORM_FULFILLMENT_MATCH_MODE=hybrid
# 订单 Webhook 连续唤醒时保持最小选号间隔,避免忙轮询压满数据库。
PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS=5
# 客户绑定二维码有效期
PLATFORM_BINDING_QR_TTL_SECONDS=600
+5
View File
@@ -49,14 +49,18 @@ services:
- DOUYU_PAYMENT_QR_CONCURRENCY=${DOUYU_PAYMENT_QR_CONCURRENCY:-10}
- DOUYU_MAX_ACTIVE_BATCHES=${DOUYU_MAX_ACTIVE_BATCHES:-10}
- DOUYU_MAX_ACTIVE_OVERVIEW_TASKS=${DOUYU_MAX_ACTIVE_OVERVIEW_TASKS:-5}
- TASK_LANE_BATCH_COORDINATOR_WORKERS=${TASK_LANE_BATCH_COORDINATOR_WORKERS:-2}
- TASK_LANE_BATCH_COORDINATOR_MAX_IN_FLIGHT=${TASK_LANE_BATCH_COORDINATOR_MAX_IN_FLIGHT:-2}
- TASK_LANE_INTERACTIVE_BINDING_WORKERS=${TASK_LANE_INTERACTIVE_BINDING_WORKERS:-3}
- TASK_LANE_BINDING_POLL_WORKERS=${TASK_LANE_BINDING_POLL_WORKERS:-2}
- TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT=${TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT:-2}
- TASK_LANE_LOGIN_WORKERS=${TASK_LANE_LOGIN_WORKERS:-4}
- TASK_LANE_ACCOUNT_CHECK_WORKERS=${TASK_LANE_ACCOUNT_CHECK_WORKERS:-4}
- TASK_LANE_OVERVIEW_QUERY_WORKERS=${TASK_LANE_OVERVIEW_QUERY_WORKERS:-5}
- TASK_LANE_QUERY_BATCH_WORKERS=${TASK_LANE_QUERY_BATCH_WORKERS:-3}
- TASK_LANE_PAYMENT_QR_WORKERS=${TASK_LANE_PAYMENT_QR_WORKERS:-10}
- TASK_LANE_PAYMENT_POLL_WORKERS=${TASK_LANE_PAYMENT_POLL_WORKERS:-3}
- TASK_LANE_PAYMENT_POLL_MAX_IN_FLIGHT=${TASK_LANE_PAYMENT_POLL_MAX_IN_FLIGHT:-3}
- TASK_LANE_OPS_WRITE_WORKERS=${TASK_LANE_OPS_WRITE_WORKERS:-2}
- DOUYU_MAX_ACTIVE_TASKS=${DOUYU_MAX_ACTIVE_TASKS:-250}
- DOUYU_PROBE_CACHE_SECONDS=${DOUYU_PROBE_CACHE_SECONDS:-30}
@@ -112,6 +116,7 @@ services:
- PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID=${PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID:-}
- ACCOUNT_COMMODITY_STATE_TTL_SECONDS=${ACCOUNT_COMMODITY_STATE_TTL_SECONDS:-900}
- PLATFORM_FULFILLMENT_MATCH_MODE=${PLATFORM_FULFILLMENT_MATCH_MODE:-hybrid}
- PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS=${PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS:-5}
- PLATFORM_BINDING_QR_TTL_SECONDS=${PLATFORM_BINDING_QR_TTL_SECONDS:-600}
# 用户上传图片的桶(在线协作 / P2 资料收集共用;生产用内网 endpoint)。
# OSS_MODE=oss 才走对象存储,否则落容器内 ./data/file_storage。
+56
View File
@@ -15,6 +15,7 @@ from web.backend.schemas import DouyuTaskBatchRequest
from web.backend.services import (
binding_poll_scheduler,
business_events,
douyu_batch_launcher,
douyu_concurrency,
douyu_service,
payment_poll_scheduler,
@@ -225,6 +226,61 @@ def test_binding_poll_scheduler_claims_due_task_without_worker_wait(db):
binding_poll_scheduler._in_flight.discard(task.id)
def test_binding_poll_dispatch_skips_claim_when_lane_is_saturated(monkeypatch):
claimed_limits: list[int] = []
monkeypatch.setattr(
binding_poll_scheduler,
"available_slots",
lambda _lane: 0,
)
monkeypatch.setattr(
binding_poll_scheduler,
"_claim_due_binding_tasks",
lambda limit: claimed_limits.append(limit) or [],
)
submitted = binding_poll_scheduler.run_binding_poll_dispatch_once(20)
assert submitted == 0
assert claimed_limits == []
def test_batch_launcher_defers_database_session_to_worker(monkeypatch):
submitted: dict[str, object] = {}
def fake_submit(lane, fn, /, *args, **kwargs):
submitted.update(
{
"lane": lane,
"fn": fn,
"args": args,
"kwargs": kwargs,
}
)
return object()
monkeypatch.setattr(
douyu_batch_launcher,
"WorkerSessionLocal",
lambda: (_ for _ in ()).throw(
AssertionError("队列阶段不能创建数据库 Session")
),
)
monkeypatch.setattr(douyu_batch_launcher, "submit", fake_submit)
queued = douyu_batch_launcher.start_douyu_batch(
batch_id="deferred-session",
task_type="get_bind_qr",
payload={},
concurrency=1,
capture_event_loop=False,
)
assert queued is True
assert submitted["lane"] == TaskLane.BATCH_COORDINATOR
assert submitted["fn"] is douyu_batch_launcher._run_batch
def test_payment_poll_scheduler_claims_due_task_without_worker_wait(db):
_, _, task = _fixtures(db, status="running")
due_at = datetime.now(UTC) - timedelta(seconds=1)
+1 -1
View File
@@ -42,7 +42,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261121_0107"
HEAD_REVISION = "20261121_0108"
# 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = (
@@ -429,6 +429,10 @@
"20261121_0107_repair_douyu_active_binding_key.py": {
"revision": "20261121_0107",
"sha256": "29539b5fab9c6d8eaf7219b7d19376dfa6032d8b6cf02b76fc2a843c09891302"
},
"20261121_0108_account_pool_query_indexes.py": {
"revision": "20261121_0108",
"sha256": "b45a7f0c563e43e6516cf00ad12ce7a5fb78bb7bde6b3f8de02cb888ad761666"
}
}
}
@@ -0,0 +1,63 @@
"""为账号池自动选号增加组合索引。
自动选号会反复执行 accounts 候选扫描,并用 NOT EXISTS 排除活动协作单。
没有可用的组合索引时,生产库每次都要扫描约四万行账号再排序,造成 MySQL CPU
和请求延迟一起放大。
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261121_0108"
down_revision: str | None = "20261121_0107"
branch_labels: str | None = None
depends_on: str | None = None
ACCOUNT_INDEX = "ix_accounts_pool_candidate"
COLLAB_INDEX = "ix_douyu_external_orders_active_account"
def _index_exists(table: str, name: str) -> bool:
conn = op.get_bind()
return bool(
conn.execute(
sa.text(
"SELECT COUNT(*) FROM information_schema.statistics "
"WHERE table_schema = DATABASE() "
"AND table_name = :table AND index_name = :name"
),
{"table": table, "name": name},
).scalar()
)
def upgrade() -> None:
if not _index_exists("accounts", ACCOUNT_INDEX):
op.create_index(
ACCOUNT_INDEX,
"accounts",
[
"auto_managed",
"status",
"role_change_available_at",
"ck_check_status",
"points_expire_at",
"points",
"id",
],
)
if not _index_exists("douyu_external_orders", COLLAB_INDEX):
op.create_index(
COLLAB_INDEX,
"douyu_external_orders",
["account_id", "archived_at", "status"],
)
def downgrade() -> None:
if _index_exists("douyu_external_orders", COLLAB_INDEX):
op.drop_index(COLLAB_INDEX, table_name="douyu_external_orders")
if _index_exists("accounts", ACCOUNT_INDEX):
op.drop_index(ACCOUNT_INDEX, table_name="accounts")
+19
View File
@@ -270,6 +270,18 @@ class Account(Base):
Index("ix_accounts_status", "status"),
# 高价值三态的过期判定和待刷新筛选
Index("ix_accounts_high_value_checked_at", "high_value_checked_at"),
# 账号池选号:先按自动管理/生命周期过滤,再按 CK、积分和到期时间排序。
# 这条索引避免每次选号都回表扫描全部账号后再排序。
Index(
"ix_accounts_pool_candidate",
"auto_managed",
"status",
"role_change_available_at",
"ck_check_status",
"points_expire_at",
"points",
"id",
),
)
assigned_user: Mapped[User | None] = relationship(
@@ -800,6 +812,13 @@ class DouyuExternalOrder(Base):
"source_task_id",
unique=True,
),
# 账号池排除活动协作单的 NOT EXISTS 每轮都会执行。
Index(
"ix_douyu_external_orders_active_account",
"account_id",
"archived_at",
"status",
),
)
@@ -13,7 +13,7 @@ from sqlalchemy.orm import joinedload
from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .douyu_runner import DouyuBatchRunner
from .task_runtime import TaskLane, submit
from .task_runtime import TaskLane, available_slots, try_submit
DEFAULT_INTERVAL_SECONDS = 1
DEFAULT_BATCH_SIZE = 20
@@ -126,10 +126,13 @@ def _release_callback(_future: Future) -> None:
def run_binding_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
claimed = _claim_due_binding_tasks(max(1, limit))
capacity = available_slots(TaskLane.BINDING_POLL)
if capacity <= 0:
return 0
claimed = _claim_due_binding_tasks(min(max(1, limit), capacity))
submitted = 0
for task_id in claimed:
future = submit(
future = try_submit(
TaskLane.BINDING_POLL,
poll_binding_task_once,
task_id,
+42 -10
View File
@@ -7,12 +7,12 @@ from __future__ import annotations
import asyncio
import os
import threading
from web.backend.database import WorkerSessionLocal
from .douyu_concurrency import clamp_douyu_task_concurrency
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
from .task_runtime import TaskLane, submit
def _env_positive_int(name: str, default: int) -> int:
@@ -30,6 +30,33 @@ DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 10)
def _run_batch(
*,
batch_id: str,
task_type: str,
payload: dict,
concurrency: int,
loop: asyncio.AbstractEventLoop | None,
log_queue: asyncio.Queue | None,
) -> None:
"""Run one coordinator with a short-lived session owned by the worker."""
thread_db = WorkerSessionLocal()
try:
runner = DouyuBatchRunner(
db=thread_db,
batch_id=batch_id,
task_type=task_type,
payload=payload,
log_queue=log_queue,
loop=loop,
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
)
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
runner.run()
finally:
thread_db.close()
def start_douyu_batch(
*,
batch_id: str,
@@ -39,8 +66,14 @@ def start_douyu_batch(
loop: asyncio.AbstractEventLoop | None = None,
log_queue: asyncio.Queue | None = None,
capture_event_loop: bool = True,
) -> None:
"""Create the batch coordinator; websocket logging is optional."""
) -> bool:
"""Queue a batch coordinator without creating a thread per order.
The coordinator uses the dedicated bounded lane and opens its database
session only after a worker slot is available. Callers may therefore
safely invoke this from a request thread without holding a database
connection while waiting in the queue.
"""
if loop is None and capture_event_loop:
try:
loop = asyncio.get_running_loop()
@@ -48,18 +81,17 @@ def start_douyu_batch(
loop = None
if log_queue is None and loop is not None:
log_queue = asyncio.Queue()
thread_db = WorkerSessionLocal()
runner = DouyuBatchRunner(
db=thread_db,
future = submit(
TaskLane.BATCH_COORDINATOR,
_run_batch,
batch_id=batch_id,
task_type=task_type,
payload=payload,
log_queue=log_queue,
concurrency=concurrency,
loop=loop,
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
log_queue=log_queue,
)
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
threading.Thread(target=runner.run, daemon=True).start()
return future is not None
def stop_douyu_batches(batch_ids: list[str]) -> int:
@@ -14,7 +14,7 @@ from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .douyu_runner import DouyuBatchRunner
from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL
from .task_runtime import TaskLane, submit
from .task_runtime import TaskLane, available_slots, try_submit
DEFAULT_INTERVAL_SECONDS = 1
DEFAULT_BATCH_SIZE = 30
@@ -182,10 +182,13 @@ def _release_callback(future: Future) -> None:
def run_payment_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
claimed = _claim_due_payment_tasks(max(1, limit))
capacity = available_slots(TaskLane.PAYMENT_POLL)
if capacity <= 0:
return 0
claimed = _claim_due_payment_tasks(min(max(1, limit), capacity))
submitted = 0
for task_id in claimed:
future = submit(
future = try_submit(
TaskLane.PAYMENT_POLL,
poll_payment_task_once,
task_id,
@@ -2,9 +2,9 @@
from __future__ import annotations
import asyncio
import os
import threading
import time
from collections.abc import Iterable
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -18,6 +18,7 @@ from ...models import PlatformFulfillmentTask
from . import service
DEFAULT_INTERVAL_SECONDS = 20
DEFAULT_MIN_INTERVAL_SECONDS = 5
DEFAULT_BATCH_SIZE = 2
DEFAULT_RETRY_SECONDS = 45
@@ -144,12 +145,10 @@ def run_auto_allocation_once(
allocated_task.next_allocate_at = None
db.commit()
try:
asyncio.run(
service.start_binding(
db,
task_id=candidate_task_id,
current=actor,
)
service.start_binding_sync(
db,
task_id=candidate_task_id,
current=actor,
)
except Exception as exc: # noqa: BLE001 - allocation remains valid
db.rollback()
@@ -211,12 +210,10 @@ def start_allocated_bindings_once(
started = 0
for task in rows:
try:
asyncio.run(
service.start_binding(
db,
task_id=int(task.id),
current=actor,
)
service.start_binding_sync(
db,
task_id=int(task.id),
current=actor,
)
started += 1
except Exception as exc: # noqa: BLE001 - one session cannot block the rest
@@ -237,6 +234,7 @@ def _run() -> None:
3600,
)
while not _stop_event.is_set():
tick_started = time.monotonic()
db = WorkerSessionLocal()
try:
pending_ids = _take_pending_task_ids(
@@ -260,6 +258,15 @@ def _run() -> None:
db.close()
if _stop_event.is_set():
break
min_interval = _env_int(
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS",
DEFAULT_MIN_INTERVAL_SECONDS,
1,
60,
)
remaining = min_interval - (time.monotonic() - tick_started)
if remaining > 0 and _stop_event.wait(remaining):
break
if _has_pending_task_ids():
continue
_wake_event.wait(interval)
@@ -14,6 +14,7 @@ from loguru import logger
from sqlalchemy import func, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from starlette.concurrency import run_in_threadpool
from ...models import (
Account,
@@ -2433,7 +2434,7 @@ def reset_task_binding(
return get_task_detail(db, task.id)
async def start_binding(
def start_binding_sync(
db: Session,
*,
task_id: int,
@@ -2536,6 +2537,23 @@ async def start_binding(
return get_task_detail(db, task.id)
async def start_binding(
db: Session,
*,
task_id: int,
current: User,
force_refresh: bool = False,
) -> dict[str, Any]:
"""Run the blocking DB/network dispatch path outside the ASGI event loop."""
return await run_in_threadpool(
start_binding_sync,
db,
task_id=task_id,
current=current,
force_refresh=force_refresh,
)
def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
task = _load_task(db, task_id)
if task.execution_status in {"completed", "cancelled"}:
@@ -2677,7 +2695,7 @@ def reconcile_binding_state_outbox(
}
async def confirm_binding(
def confirm_binding_sync(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
task = _load_task_for_update(db, task_id)
@@ -2754,6 +2772,17 @@ async def confirm_binding(
return get_task_detail(db, task.id)
async def confirm_binding(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
return await run_in_threadpool(
confirm_binding_sync,
db,
task_id=task_id,
current=current,
)
def refresh_exchange_eligibility(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
+77 -14
View File
@@ -19,6 +19,7 @@ from typing import Any
class TaskLane(StrEnum):
INTERACTIVE_BINDING = "interactive_binding"
BATCH_COORDINATOR = "batch_coordinator"
BINDING_POLL = "binding_poll"
LOGIN = "login"
ACCOUNT_CHECK = "account_check"
@@ -51,6 +52,7 @@ class LaneConfig:
_DEFAULT_WORKERS: dict[TaskLane, int] = {
TaskLane.INTERACTIVE_BINDING: 3,
TaskLane.BATCH_COORDINATOR: 2,
TaskLane.BINDING_POLL: 2,
TaskLane.LOGIN: 4,
TaskLane.ACCOUNT_CHECK: 4,
@@ -71,9 +73,19 @@ _DEFAULT_WORKERS: dict[TaskLane, int] = {
def _lane_config(lane: TaskLane) -> LaneConfig:
key = lane.value.upper()
workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane])
default_in_flight = (
workers
if lane
in {
TaskLane.BATCH_COORDINATOR,
TaskLane.BINDING_POLL,
TaskLane.PAYMENT_POLL,
}
else max(workers * 32, workers)
)
max_in_flight = _env_int(
f"TASK_LANE_{key}_MAX_IN_FLIGHT",
max(workers * 32, workers),
default_in_flight,
)
return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers))
@@ -107,6 +119,36 @@ def _runtime(lane: TaskLane) -> _LaneRuntime:
return runtime
def _submit[T](
lane: TaskLane,
fn: Callable[..., T],
/,
*args: Any,
stopped: Callable[[], bool] | None = None,
wait: bool,
**kwargs: Any,
) -> Future[T] | None:
runtime = _runtime(lane)
if stopped is not None and stopped():
return None
if wait:
while True:
if stopped is not None and stopped():
return None
if runtime.admission.acquire(timeout=0.2):
break
elif not runtime.admission.acquire(blocking=False):
return None
def run() -> T:
try:
return fn(*args, **kwargs)
finally:
runtime.admission.release()
return runtime.executor.submit(run)
def submit[T](
lane: TaskLane,
fn: Callable[..., T],
@@ -120,20 +162,39 @@ def submit[T](
Batch coordinators are cheap threads and are allowed to block here. Worker
threads themselves must not recursively submit to the same lane.
"""
return _submit(
lane,
fn,
*args,
stopped=stopped,
wait=True,
**kwargs,
)
def try_submit[T](
lane: TaskLane,
fn: Callable[..., T],
/,
*args: Any,
stopped: Callable[[], bool] | None = None,
**kwargs: Any,
) -> Future[T] | None:
"""Submit without blocking when the lane admission budget is full."""
return _submit(
lane,
fn,
*args,
stopped=stopped,
wait=False,
**kwargs,
)
def available_slots(lane: TaskLane) -> int:
"""Return the number of tasks that can currently enter a lane immediately."""
runtime = _runtime(lane)
while True:
if stopped is not None and stopped():
return None
if runtime.admission.acquire(timeout=0.2):
break
def run() -> T:
try:
return fn(*args, **kwargs)
finally:
runtime.admission.release()
return runtime.executor.submit(run)
return max(0, int(runtime.admission._value))
def lane_stats() -> dict[str, dict[str, int]]:
@@ -194,8 +255,10 @@ def douyu_lane_for_task(task_type: str) -> TaskLane:
__all__ = [
"TaskLane",
"available_slots",
"douyu_lane_for_task",
"lane_stats",
"shutdown",
"submit",
"try_submit",
]