fix(platform): 防止绑定任务压垮连接池和事件循环

This commit is contained in:
yml
2026-11-21 15:22:46 +08:00
parent d199e04d27
commit e1d979a296
13 changed files with 336 additions and 46 deletions
@@ -429,6 +429,10 @@
"20261121_0107_repair_douyu_active_binding_key.py": {
"revision": "20261121_0107",
"sha256": "29539b5fab9c6d8eaf7219b7d19376dfa6032d8b6cf02b76fc2a843c09891302"
},
"20261121_0108_account_pool_query_indexes.py": {
"revision": "20261121_0108",
"sha256": "b45a7f0c563e43e6516cf00ad12ce7a5fb78bb7bde6b3f8de02cb888ad761666"
}
}
}
@@ -0,0 +1,63 @@
"""为账号池自动选号增加组合索引。
自动选号会反复执行 accounts 候选扫描,并用 NOT EXISTS 排除活动协作单。
没有可用的组合索引时,生产库每次都要扫描约四万行账号再排序,造成 MySQL CPU
和请求延迟一起放大。
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261121_0108"
down_revision: str | None = "20261121_0107"
branch_labels: str | None = None
depends_on: str | None = None
ACCOUNT_INDEX = "ix_accounts_pool_candidate"
COLLAB_INDEX = "ix_douyu_external_orders_active_account"
def _index_exists(table: str, name: str) -> bool:
conn = op.get_bind()
return bool(
conn.execute(
sa.text(
"SELECT COUNT(*) FROM information_schema.statistics "
"WHERE table_schema = DATABASE() "
"AND table_name = :table AND index_name = :name"
),
{"table": table, "name": name},
).scalar()
)
def upgrade() -> None:
if not _index_exists("accounts", ACCOUNT_INDEX):
op.create_index(
ACCOUNT_INDEX,
"accounts",
[
"auto_managed",
"status",
"role_change_available_at",
"ck_check_status",
"points_expire_at",
"points",
"id",
],
)
if not _index_exists("douyu_external_orders", COLLAB_INDEX):
op.create_index(
COLLAB_INDEX,
"douyu_external_orders",
["account_id", "archived_at", "status"],
)
def downgrade() -> None:
if _index_exists("douyu_external_orders", COLLAB_INDEX):
op.drop_index(COLLAB_INDEX, table_name="douyu_external_orders")
if _index_exists("accounts", ACCOUNT_INDEX):
op.drop_index(ACCOUNT_INDEX, table_name="accounts")
+19
View File
@@ -270,6 +270,18 @@ class Account(Base):
Index("ix_accounts_status", "status"),
# 高价值三态的过期判定和待刷新筛选
Index("ix_accounts_high_value_checked_at", "high_value_checked_at"),
# 账号池选号:先按自动管理/生命周期过滤,再按 CK、积分和到期时间排序。
# 这条索引避免每次选号都回表扫描全部账号后再排序。
Index(
"ix_accounts_pool_candidate",
"auto_managed",
"status",
"role_change_available_at",
"ck_check_status",
"points_expire_at",
"points",
"id",
),
)
assigned_user: Mapped[User | None] = relationship(
@@ -800,6 +812,13 @@ class DouyuExternalOrder(Base):
"source_task_id",
unique=True,
),
# 账号池排除活动协作单的 NOT EXISTS 每轮都会执行。
Index(
"ix_douyu_external_orders_active_account",
"account_id",
"archived_at",
"status",
),
)
@@ -13,7 +13,7 @@ from sqlalchemy.orm import joinedload
from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .douyu_runner import DouyuBatchRunner
from .task_runtime import TaskLane, submit
from .task_runtime import TaskLane, available_slots, try_submit
DEFAULT_INTERVAL_SECONDS = 1
DEFAULT_BATCH_SIZE = 20
@@ -126,10 +126,13 @@ def _release_callback(_future: Future) -> None:
def run_binding_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
claimed = _claim_due_binding_tasks(max(1, limit))
capacity = available_slots(TaskLane.BINDING_POLL)
if capacity <= 0:
return 0
claimed = _claim_due_binding_tasks(min(max(1, limit), capacity))
submitted = 0
for task_id in claimed:
future = submit(
future = try_submit(
TaskLane.BINDING_POLL,
poll_binding_task_once,
task_id,
+42 -10
View File
@@ -7,12 +7,12 @@ from __future__ import annotations
import asyncio
import os
import threading
from web.backend.database import WorkerSessionLocal
from .douyu_concurrency import clamp_douyu_task_concurrency
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
from .task_runtime import TaskLane, submit
def _env_positive_int(name: str, default: int) -> int:
@@ -30,6 +30,33 @@ DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 10)
def _run_batch(
*,
batch_id: str,
task_type: str,
payload: dict,
concurrency: int,
loop: asyncio.AbstractEventLoop | None,
log_queue: asyncio.Queue | None,
) -> None:
"""Run one coordinator with a short-lived session owned by the worker."""
thread_db = WorkerSessionLocal()
try:
runner = DouyuBatchRunner(
db=thread_db,
batch_id=batch_id,
task_type=task_type,
payload=payload,
log_queue=log_queue,
loop=loop,
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
)
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
runner.run()
finally:
thread_db.close()
def start_douyu_batch(
*,
batch_id: str,
@@ -39,8 +66,14 @@ def start_douyu_batch(
loop: asyncio.AbstractEventLoop | None = None,
log_queue: asyncio.Queue | None = None,
capture_event_loop: bool = True,
) -> None:
"""Create the batch coordinator; websocket logging is optional."""
) -> bool:
"""Queue a batch coordinator without creating a thread per order.
The coordinator uses the dedicated bounded lane and opens its database
session only after a worker slot is available. Callers may therefore
safely invoke this from a request thread without holding a database
connection while waiting in the queue.
"""
if loop is None and capture_event_loop:
try:
loop = asyncio.get_running_loop()
@@ -48,18 +81,17 @@ def start_douyu_batch(
loop = None
if log_queue is None and loop is not None:
log_queue = asyncio.Queue()
thread_db = WorkerSessionLocal()
runner = DouyuBatchRunner(
db=thread_db,
future = submit(
TaskLane.BATCH_COORDINATOR,
_run_batch,
batch_id=batch_id,
task_type=task_type,
payload=payload,
log_queue=log_queue,
concurrency=concurrency,
loop=loop,
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
log_queue=log_queue,
)
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
threading.Thread(target=runner.run, daemon=True).start()
return future is not None
def stop_douyu_batches(batch_ids: list[str]) -> int:
@@ -14,7 +14,7 @@ from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .douyu_runner import DouyuBatchRunner
from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL
from .task_runtime import TaskLane, submit
from .task_runtime import TaskLane, available_slots, try_submit
DEFAULT_INTERVAL_SECONDS = 1
DEFAULT_BATCH_SIZE = 30
@@ -182,10 +182,13 @@ def _release_callback(future: Future) -> None:
def run_payment_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
claimed = _claim_due_payment_tasks(max(1, limit))
capacity = available_slots(TaskLane.PAYMENT_POLL)
if capacity <= 0:
return 0
claimed = _claim_due_payment_tasks(min(max(1, limit), capacity))
submitted = 0
for task_id in claimed:
future = submit(
future = try_submit(
TaskLane.PAYMENT_POLL,
poll_payment_task_once,
task_id,
@@ -2,9 +2,9 @@
from __future__ import annotations
import asyncio
import os
import threading
import time
from collections.abc import Iterable
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -18,6 +18,7 @@ from ...models import PlatformFulfillmentTask
from . import service
DEFAULT_INTERVAL_SECONDS = 20
DEFAULT_MIN_INTERVAL_SECONDS = 5
DEFAULT_BATCH_SIZE = 2
DEFAULT_RETRY_SECONDS = 45
@@ -144,12 +145,10 @@ def run_auto_allocation_once(
allocated_task.next_allocate_at = None
db.commit()
try:
asyncio.run(
service.start_binding(
db,
task_id=candidate_task_id,
current=actor,
)
service.start_binding_sync(
db,
task_id=candidate_task_id,
current=actor,
)
except Exception as exc: # noqa: BLE001 - allocation remains valid
db.rollback()
@@ -211,12 +210,10 @@ def start_allocated_bindings_once(
started = 0
for task in rows:
try:
asyncio.run(
service.start_binding(
db,
task_id=int(task.id),
current=actor,
)
service.start_binding_sync(
db,
task_id=int(task.id),
current=actor,
)
started += 1
except Exception as exc: # noqa: BLE001 - one session cannot block the rest
@@ -237,6 +234,7 @@ def _run() -> None:
3600,
)
while not _stop_event.is_set():
tick_started = time.monotonic()
db = WorkerSessionLocal()
try:
pending_ids = _take_pending_task_ids(
@@ -260,6 +258,15 @@ def _run() -> None:
db.close()
if _stop_event.is_set():
break
min_interval = _env_int(
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS",
DEFAULT_MIN_INTERVAL_SECONDS,
1,
60,
)
remaining = min_interval - (time.monotonic() - tick_started)
if remaining > 0 and _stop_event.wait(remaining):
break
if _has_pending_task_ids():
continue
_wake_event.wait(interval)
@@ -14,6 +14,7 @@ from loguru import logger
from sqlalchemy import func, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from starlette.concurrency import run_in_threadpool
from ...models import (
Account,
@@ -2433,7 +2434,7 @@ def reset_task_binding(
return get_task_detail(db, task.id)
async def start_binding(
def start_binding_sync(
db: Session,
*,
task_id: int,
@@ -2536,6 +2537,23 @@ async def start_binding(
return get_task_detail(db, task.id)
async def start_binding(
db: Session,
*,
task_id: int,
current: User,
force_refresh: bool = False,
) -> dict[str, Any]:
"""Run the blocking DB/network dispatch path outside the ASGI event loop."""
return await run_in_threadpool(
start_binding_sync,
db,
task_id=task_id,
current=current,
force_refresh=force_refresh,
)
def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
task = _load_task(db, task_id)
if task.execution_status in {"completed", "cancelled"}:
@@ -2677,7 +2695,7 @@ def reconcile_binding_state_outbox(
}
async def confirm_binding(
def confirm_binding_sync(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
task = _load_task_for_update(db, task_id)
@@ -2754,6 +2772,17 @@ async def confirm_binding(
return get_task_detail(db, task.id)
async def confirm_binding(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
return await run_in_threadpool(
confirm_binding_sync,
db,
task_id=task_id,
current=current,
)
def refresh_exchange_eligibility(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
+77 -14
View File
@@ -19,6 +19,7 @@ from typing import Any
class TaskLane(StrEnum):
INTERACTIVE_BINDING = "interactive_binding"
BATCH_COORDINATOR = "batch_coordinator"
BINDING_POLL = "binding_poll"
LOGIN = "login"
ACCOUNT_CHECK = "account_check"
@@ -51,6 +52,7 @@ class LaneConfig:
_DEFAULT_WORKERS: dict[TaskLane, int] = {
TaskLane.INTERACTIVE_BINDING: 3,
TaskLane.BATCH_COORDINATOR: 2,
TaskLane.BINDING_POLL: 2,
TaskLane.LOGIN: 4,
TaskLane.ACCOUNT_CHECK: 4,
@@ -71,9 +73,19 @@ _DEFAULT_WORKERS: dict[TaskLane, int] = {
def _lane_config(lane: TaskLane) -> LaneConfig:
key = lane.value.upper()
workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane])
default_in_flight = (
workers
if lane
in {
TaskLane.BATCH_COORDINATOR,
TaskLane.BINDING_POLL,
TaskLane.PAYMENT_POLL,
}
else max(workers * 32, workers)
)
max_in_flight = _env_int(
f"TASK_LANE_{key}_MAX_IN_FLIGHT",
max(workers * 32, workers),
default_in_flight,
)
return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers))
@@ -107,6 +119,36 @@ def _runtime(lane: TaskLane) -> _LaneRuntime:
return runtime
def _submit[T](
lane: TaskLane,
fn: Callable[..., T],
/,
*args: Any,
stopped: Callable[[], bool] | None = None,
wait: bool,
**kwargs: Any,
) -> Future[T] | None:
runtime = _runtime(lane)
if stopped is not None and stopped():
return None
if wait:
while True:
if stopped is not None and stopped():
return None
if runtime.admission.acquire(timeout=0.2):
break
elif not runtime.admission.acquire(blocking=False):
return None
def run() -> T:
try:
return fn(*args, **kwargs)
finally:
runtime.admission.release()
return runtime.executor.submit(run)
def submit[T](
lane: TaskLane,
fn: Callable[..., T],
@@ -120,20 +162,39 @@ def submit[T](
Batch coordinators are cheap threads and are allowed to block here. Worker
threads themselves must not recursively submit to the same lane.
"""
return _submit(
lane,
fn,
*args,
stopped=stopped,
wait=True,
**kwargs,
)
def try_submit[T](
lane: TaskLane,
fn: Callable[..., T],
/,
*args: Any,
stopped: Callable[[], bool] | None = None,
**kwargs: Any,
) -> Future[T] | None:
"""Submit without blocking when the lane admission budget is full."""
return _submit(
lane,
fn,
*args,
stopped=stopped,
wait=False,
**kwargs,
)
def available_slots(lane: TaskLane) -> int:
"""Return the number of tasks that can currently enter a lane immediately."""
runtime = _runtime(lane)
while True:
if stopped is not None and stopped():
return None
if runtime.admission.acquire(timeout=0.2):
break
def run() -> T:
try:
return fn(*args, **kwargs)
finally:
runtime.admission.release()
return runtime.executor.submit(run)
return max(0, int(runtime.admission._value))
def lane_stats() -> dict[str, dict[str, int]]:
@@ -194,8 +255,10 @@ def douyu_lane_for_task(task_type: str) -> TaskLane:
__all__ = [
"TaskLane",
"available_slots",
"douyu_lane_for_task",
"lane_stats",
"shutdown",
"submit",
"try_submit",
]