fix(platform): 防止绑定任务压垮连接池和事件循环

This commit is contained in:
yml
2026-11-21 15:22:46 +08:00
parent d199e04d27
commit e1d979a296
13 changed files with 336 additions and 46 deletions
@@ -13,7 +13,7 @@ from sqlalchemy.orm import joinedload
from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .douyu_runner import DouyuBatchRunner
from .task_runtime import TaskLane, submit
from .task_runtime import TaskLane, available_slots, try_submit
DEFAULT_INTERVAL_SECONDS = 1
DEFAULT_BATCH_SIZE = 20
@@ -126,10 +126,13 @@ def _release_callback(_future: Future) -> None:
def run_binding_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
claimed = _claim_due_binding_tasks(max(1, limit))
capacity = available_slots(TaskLane.BINDING_POLL)
if capacity <= 0:
return 0
claimed = _claim_due_binding_tasks(min(max(1, limit), capacity))
submitted = 0
for task_id in claimed:
future = submit(
future = try_submit(
TaskLane.BINDING_POLL,
poll_binding_task_once,
task_id,
+42 -10
View File
@@ -7,12 +7,12 @@ from __future__ import annotations
import asyncio
import os
import threading
from web.backend.database import WorkerSessionLocal
from .douyu_concurrency import clamp_douyu_task_concurrency
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
from .task_runtime import TaskLane, submit
def _env_positive_int(name: str, default: int) -> int:
@@ -30,6 +30,33 @@ DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 10)
def _run_batch(
*,
batch_id: str,
task_type: str,
payload: dict,
concurrency: int,
loop: asyncio.AbstractEventLoop | None,
log_queue: asyncio.Queue | None,
) -> None:
"""Run one coordinator with a short-lived session owned by the worker."""
thread_db = WorkerSessionLocal()
try:
runner = DouyuBatchRunner(
db=thread_db,
batch_id=batch_id,
task_type=task_type,
payload=payload,
log_queue=log_queue,
loop=loop,
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
)
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
runner.run()
finally:
thread_db.close()
def start_douyu_batch(
*,
batch_id: str,
@@ -39,8 +66,14 @@ def start_douyu_batch(
loop: asyncio.AbstractEventLoop | None = None,
log_queue: asyncio.Queue | None = None,
capture_event_loop: bool = True,
) -> None:
"""Create the batch coordinator; websocket logging is optional."""
) -> bool:
"""Queue a batch coordinator without creating a thread per order.
The coordinator uses the dedicated bounded lane and opens its database
session only after a worker slot is available. Callers may therefore
safely invoke this from a request thread without holding a database
connection while waiting in the queue.
"""
if loop is None and capture_event_loop:
try:
loop = asyncio.get_running_loop()
@@ -48,18 +81,17 @@ def start_douyu_batch(
loop = None
if log_queue is None and loop is not None:
log_queue = asyncio.Queue()
thread_db = WorkerSessionLocal()
runner = DouyuBatchRunner(
db=thread_db,
future = submit(
TaskLane.BATCH_COORDINATOR,
_run_batch,
batch_id=batch_id,
task_type=task_type,
payload=payload,
log_queue=log_queue,
concurrency=concurrency,
loop=loop,
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
log_queue=log_queue,
)
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
threading.Thread(target=runner.run, daemon=True).start()
return future is not None
def stop_douyu_batches(batch_ids: list[str]) -> int:
@@ -14,7 +14,7 @@ from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .douyu_runner import DouyuBatchRunner
from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL
from .task_runtime import TaskLane, submit
from .task_runtime import TaskLane, available_slots, try_submit
DEFAULT_INTERVAL_SECONDS = 1
DEFAULT_BATCH_SIZE = 30
@@ -182,10 +182,13 @@ def _release_callback(future: Future) -> None:
def run_payment_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
claimed = _claim_due_payment_tasks(max(1, limit))
capacity = available_slots(TaskLane.PAYMENT_POLL)
if capacity <= 0:
return 0
claimed = _claim_due_payment_tasks(min(max(1, limit), capacity))
submitted = 0
for task_id in claimed:
future = submit(
future = try_submit(
TaskLane.PAYMENT_POLL,
poll_payment_task_once,
task_id,
@@ -2,9 +2,9 @@
from __future__ import annotations
import asyncio
import os
import threading
import time
from collections.abc import Iterable
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -18,6 +18,7 @@ from ...models import PlatformFulfillmentTask
from . import service
DEFAULT_INTERVAL_SECONDS = 20
DEFAULT_MIN_INTERVAL_SECONDS = 5
DEFAULT_BATCH_SIZE = 2
DEFAULT_RETRY_SECONDS = 45
@@ -144,12 +145,10 @@ def run_auto_allocation_once(
allocated_task.next_allocate_at = None
db.commit()
try:
asyncio.run(
service.start_binding(
db,
task_id=candidate_task_id,
current=actor,
)
service.start_binding_sync(
db,
task_id=candidate_task_id,
current=actor,
)
except Exception as exc: # noqa: BLE001 - allocation remains valid
db.rollback()
@@ -211,12 +210,10 @@ def start_allocated_bindings_once(
started = 0
for task in rows:
try:
asyncio.run(
service.start_binding(
db,
task_id=int(task.id),
current=actor,
)
service.start_binding_sync(
db,
task_id=int(task.id),
current=actor,
)
started += 1
except Exception as exc: # noqa: BLE001 - one session cannot block the rest
@@ -237,6 +234,7 @@ def _run() -> None:
3600,
)
while not _stop_event.is_set():
tick_started = time.monotonic()
db = WorkerSessionLocal()
try:
pending_ids = _take_pending_task_ids(
@@ -260,6 +258,15 @@ def _run() -> None:
db.close()
if _stop_event.is_set():
break
min_interval = _env_int(
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS",
DEFAULT_MIN_INTERVAL_SECONDS,
1,
60,
)
remaining = min_interval - (time.monotonic() - tick_started)
if remaining > 0 and _stop_event.wait(remaining):
break
if _has_pending_task_ids():
continue
_wake_event.wait(interval)
@@ -14,6 +14,7 @@ from loguru import logger
from sqlalchemy import func, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from starlette.concurrency import run_in_threadpool
from ...models import (
Account,
@@ -2433,7 +2434,7 @@ def reset_task_binding(
return get_task_detail(db, task.id)
async def start_binding(
def start_binding_sync(
db: Session,
*,
task_id: int,
@@ -2536,6 +2537,23 @@ async def start_binding(
return get_task_detail(db, task.id)
async def start_binding(
db: Session,
*,
task_id: int,
current: User,
force_refresh: bool = False,
) -> dict[str, Any]:
"""Run the blocking DB/network dispatch path outside the ASGI event loop."""
return await run_in_threadpool(
start_binding_sync,
db,
task_id=task_id,
current=current,
force_refresh=force_refresh,
)
def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
task = _load_task(db, task_id)
if task.execution_status in {"completed", "cancelled"}:
@@ -2677,7 +2695,7 @@ def reconcile_binding_state_outbox(
}
async def confirm_binding(
def confirm_binding_sync(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
task = _load_task_for_update(db, task_id)
@@ -2754,6 +2772,17 @@ async def confirm_binding(
return get_task_detail(db, task.id)
async def confirm_binding(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
return await run_in_threadpool(
confirm_binding_sync,
db,
task_id=task_id,
current=current,
)
def refresh_exchange_eligibility(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
+77 -14
View File
@@ -19,6 +19,7 @@ from typing import Any
class TaskLane(StrEnum):
INTERACTIVE_BINDING = "interactive_binding"
BATCH_COORDINATOR = "batch_coordinator"
BINDING_POLL = "binding_poll"
LOGIN = "login"
ACCOUNT_CHECK = "account_check"
@@ -51,6 +52,7 @@ class LaneConfig:
_DEFAULT_WORKERS: dict[TaskLane, int] = {
TaskLane.INTERACTIVE_BINDING: 3,
TaskLane.BATCH_COORDINATOR: 2,
TaskLane.BINDING_POLL: 2,
TaskLane.LOGIN: 4,
TaskLane.ACCOUNT_CHECK: 4,
@@ -71,9 +73,19 @@ _DEFAULT_WORKERS: dict[TaskLane, int] = {
def _lane_config(lane: TaskLane) -> LaneConfig:
key = lane.value.upper()
workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane])
default_in_flight = (
workers
if lane
in {
TaskLane.BATCH_COORDINATOR,
TaskLane.BINDING_POLL,
TaskLane.PAYMENT_POLL,
}
else max(workers * 32, workers)
)
max_in_flight = _env_int(
f"TASK_LANE_{key}_MAX_IN_FLIGHT",
max(workers * 32, workers),
default_in_flight,
)
return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers))
@@ -107,6 +119,36 @@ def _runtime(lane: TaskLane) -> _LaneRuntime:
return runtime
def _submit[T](
lane: TaskLane,
fn: Callable[..., T],
/,
*args: Any,
stopped: Callable[[], bool] | None = None,
wait: bool,
**kwargs: Any,
) -> Future[T] | None:
runtime = _runtime(lane)
if stopped is not None and stopped():
return None
if wait:
while True:
if stopped is not None and stopped():
return None
if runtime.admission.acquire(timeout=0.2):
break
elif not runtime.admission.acquire(blocking=False):
return None
def run() -> T:
try:
return fn(*args, **kwargs)
finally:
runtime.admission.release()
return runtime.executor.submit(run)
def submit[T](
lane: TaskLane,
fn: Callable[..., T],
@@ -120,20 +162,39 @@ def submit[T](
Batch coordinators are cheap threads and are allowed to block here. Worker
threads themselves must not recursively submit to the same lane.
"""
return _submit(
lane,
fn,
*args,
stopped=stopped,
wait=True,
**kwargs,
)
def try_submit[T](
lane: TaskLane,
fn: Callable[..., T],
/,
*args: Any,
stopped: Callable[[], bool] | None = None,
**kwargs: Any,
) -> Future[T] | None:
"""Submit without blocking when the lane admission budget is full."""
return _submit(
lane,
fn,
*args,
stopped=stopped,
wait=False,
**kwargs,
)
def available_slots(lane: TaskLane) -> int:
"""Return the number of tasks that can currently enter a lane immediately."""
runtime = _runtime(lane)
while True:
if stopped is not None and stopped():
return None
if runtime.admission.acquire(timeout=0.2):
break
def run() -> T:
try:
return fn(*args, **kwargs)
finally:
runtime.admission.release()
return runtime.executor.submit(run)
return max(0, int(runtime.admission._value))
def lane_stats() -> dict[str, dict[str, int]]:
@@ -194,8 +255,10 @@ def douyu_lane_for_task(task_type: str) -> TaskLane:
__all__ = [
"TaskLane",
"available_slots",
"douyu_lane_for_task",
"lane_stats",
"shutdown",
"submit",
"try_submit",
]