fix(platform): 防止绑定任务压垮连接池和事件循环
This commit is contained in:
@@ -13,7 +13,7 @@ from sqlalchemy.orm import joinedload
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .douyu_runner import DouyuBatchRunner
|
||||
from .task_runtime import TaskLane, submit
|
||||
from .task_runtime import TaskLane, available_slots, try_submit
|
||||
|
||||
DEFAULT_INTERVAL_SECONDS = 1
|
||||
DEFAULT_BATCH_SIZE = 20
|
||||
@@ -126,10 +126,13 @@ def _release_callback(_future: Future) -> None:
|
||||
|
||||
|
||||
def run_binding_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
|
||||
claimed = _claim_due_binding_tasks(max(1, limit))
|
||||
capacity = available_slots(TaskLane.BINDING_POLL)
|
||||
if capacity <= 0:
|
||||
return 0
|
||||
claimed = _claim_due_binding_tasks(min(max(1, limit), capacity))
|
||||
submitted = 0
|
||||
for task_id in claimed:
|
||||
future = submit(
|
||||
future = try_submit(
|
||||
TaskLane.BINDING_POLL,
|
||||
poll_binding_task_once,
|
||||
task_id,
|
||||
|
||||
@@ -7,12 +7,12 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
|
||||
from web.backend.database import WorkerSessionLocal
|
||||
|
||||
from .douyu_concurrency import clamp_douyu_task_concurrency
|
||||
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||
from .task_runtime import TaskLane, submit
|
||||
|
||||
|
||||
def _env_positive_int(name: str, default: int) -> int:
|
||||
@@ -30,6 +30,33 @@ DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
|
||||
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 10)
|
||||
|
||||
|
||||
def _run_batch(
|
||||
*,
|
||||
batch_id: str,
|
||||
task_type: str,
|
||||
payload: dict,
|
||||
concurrency: int,
|
||||
loop: asyncio.AbstractEventLoop | None,
|
||||
log_queue: asyncio.Queue | None,
|
||||
) -> None:
|
||||
"""Run one coordinator with a short-lived session owned by the worker."""
|
||||
thread_db = WorkerSessionLocal()
|
||||
try:
|
||||
runner = DouyuBatchRunner(
|
||||
db=thread_db,
|
||||
batch_id=batch_id,
|
||||
task_type=task_type,
|
||||
payload=payload,
|
||||
log_queue=log_queue,
|
||||
loop=loop,
|
||||
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
|
||||
)
|
||||
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
|
||||
runner.run()
|
||||
finally:
|
||||
thread_db.close()
|
||||
|
||||
|
||||
def start_douyu_batch(
|
||||
*,
|
||||
batch_id: str,
|
||||
@@ -39,8 +66,14 @@ def start_douyu_batch(
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
log_queue: asyncio.Queue | None = None,
|
||||
capture_event_loop: bool = True,
|
||||
) -> None:
|
||||
"""Create the batch coordinator; websocket logging is optional."""
|
||||
) -> bool:
|
||||
"""Queue a batch coordinator without creating a thread per order.
|
||||
|
||||
The coordinator uses the dedicated bounded lane and opens its database
|
||||
session only after a worker slot is available. Callers may therefore
|
||||
safely invoke this from a request thread without holding a database
|
||||
connection while waiting in the queue.
|
||||
"""
|
||||
if loop is None and capture_event_loop:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -48,18 +81,17 @@ def start_douyu_batch(
|
||||
loop = None
|
||||
if log_queue is None and loop is not None:
|
||||
log_queue = asyncio.Queue()
|
||||
thread_db = WorkerSessionLocal()
|
||||
runner = DouyuBatchRunner(
|
||||
db=thread_db,
|
||||
future = submit(
|
||||
TaskLane.BATCH_COORDINATOR,
|
||||
_run_batch,
|
||||
batch_id=batch_id,
|
||||
task_type=task_type,
|
||||
payload=payload,
|
||||
log_queue=log_queue,
|
||||
concurrency=concurrency,
|
||||
loop=loop,
|
||||
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
|
||||
log_queue=log_queue,
|
||||
)
|
||||
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
|
||||
threading.Thread(target=runner.run, daemon=True).start()
|
||||
return future is not None
|
||||
|
||||
|
||||
def stop_douyu_batches(batch_ids: list[str]) -> int:
|
||||
|
||||
@@ -14,7 +14,7 @@ from ..database import WorkerSessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .douyu_runner import DouyuBatchRunner
|
||||
from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL
|
||||
from .task_runtime import TaskLane, submit
|
||||
from .task_runtime import TaskLane, available_slots, try_submit
|
||||
|
||||
DEFAULT_INTERVAL_SECONDS = 1
|
||||
DEFAULT_BATCH_SIZE = 30
|
||||
@@ -182,10 +182,13 @@ def _release_callback(future: Future) -> None:
|
||||
|
||||
|
||||
def run_payment_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
|
||||
claimed = _claim_due_payment_tasks(max(1, limit))
|
||||
capacity = available_slots(TaskLane.PAYMENT_POLL)
|
||||
if capacity <= 0:
|
||||
return 0
|
||||
claimed = _claim_due_payment_tasks(min(max(1, limit), capacity))
|
||||
submitted = 0
|
||||
for task_id in claimed:
|
||||
future = submit(
|
||||
future = try_submit(
|
||||
TaskLane.PAYMENT_POLL,
|
||||
poll_payment_task_once,
|
||||
task_id,
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
@@ -18,6 +18,7 @@ from ...models import PlatformFulfillmentTask
|
||||
from . import service
|
||||
|
||||
DEFAULT_INTERVAL_SECONDS = 20
|
||||
DEFAULT_MIN_INTERVAL_SECONDS = 5
|
||||
DEFAULT_BATCH_SIZE = 2
|
||||
DEFAULT_RETRY_SECONDS = 45
|
||||
|
||||
@@ -144,12 +145,10 @@ def run_auto_allocation_once(
|
||||
allocated_task.next_allocate_at = None
|
||||
db.commit()
|
||||
try:
|
||||
asyncio.run(
|
||||
service.start_binding(
|
||||
db,
|
||||
task_id=candidate_task_id,
|
||||
current=actor,
|
||||
)
|
||||
service.start_binding_sync(
|
||||
db,
|
||||
task_id=candidate_task_id,
|
||||
current=actor,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - allocation remains valid
|
||||
db.rollback()
|
||||
@@ -211,12 +210,10 @@ def start_allocated_bindings_once(
|
||||
started = 0
|
||||
for task in rows:
|
||||
try:
|
||||
asyncio.run(
|
||||
service.start_binding(
|
||||
db,
|
||||
task_id=int(task.id),
|
||||
current=actor,
|
||||
)
|
||||
service.start_binding_sync(
|
||||
db,
|
||||
task_id=int(task.id),
|
||||
current=actor,
|
||||
)
|
||||
started += 1
|
||||
except Exception as exc: # noqa: BLE001 - one session cannot block the rest
|
||||
@@ -237,6 +234,7 @@ def _run() -> None:
|
||||
3600,
|
||||
)
|
||||
while not _stop_event.is_set():
|
||||
tick_started = time.monotonic()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
pending_ids = _take_pending_task_ids(
|
||||
@@ -260,6 +258,15 @@ def _run() -> None:
|
||||
db.close()
|
||||
if _stop_event.is_set():
|
||||
break
|
||||
min_interval = _env_int(
|
||||
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS",
|
||||
DEFAULT_MIN_INTERVAL_SECONDS,
|
||||
1,
|
||||
60,
|
||||
)
|
||||
remaining = min_interval - (time.monotonic() - tick_started)
|
||||
if remaining > 0 and _stop_event.wait(remaining):
|
||||
break
|
||||
if _has_pending_task_ids():
|
||||
continue
|
||||
_wake_event.wait(interval)
|
||||
|
||||
@@ -14,6 +14,7 @@ from loguru import logger
|
||||
from sqlalchemy import func, or_
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from ...models import (
|
||||
Account,
|
||||
@@ -2433,7 +2434,7 @@ def reset_task_binding(
|
||||
return get_task_detail(db, task.id)
|
||||
|
||||
|
||||
async def start_binding(
|
||||
def start_binding_sync(
|
||||
db: Session,
|
||||
*,
|
||||
task_id: int,
|
||||
@@ -2536,6 +2537,23 @@ async def start_binding(
|
||||
return get_task_detail(db, task.id)
|
||||
|
||||
|
||||
async def start_binding(
|
||||
db: Session,
|
||||
*,
|
||||
task_id: int,
|
||||
current: User,
|
||||
force_refresh: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the blocking DB/network dispatch path outside the ASGI event loop."""
|
||||
return await run_in_threadpool(
|
||||
start_binding_sync,
|
||||
db,
|
||||
task_id=task_id,
|
||||
current=current,
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
|
||||
|
||||
def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
|
||||
task = _load_task(db, task_id)
|
||||
if task.execution_status in {"completed", "cancelled"}:
|
||||
@@ -2677,7 +2695,7 @@ def reconcile_binding_state_outbox(
|
||||
}
|
||||
|
||||
|
||||
async def confirm_binding(
|
||||
def confirm_binding_sync(
|
||||
db: Session, *, task_id: int, current: User
|
||||
) -> dict[str, Any]:
|
||||
task = _load_task_for_update(db, task_id)
|
||||
@@ -2754,6 +2772,17 @@ async def confirm_binding(
|
||||
return get_task_detail(db, task.id)
|
||||
|
||||
|
||||
async def confirm_binding(
|
||||
db: Session, *, task_id: int, current: User
|
||||
) -> dict[str, Any]:
|
||||
return await run_in_threadpool(
|
||||
confirm_binding_sync,
|
||||
db,
|
||||
task_id=task_id,
|
||||
current=current,
|
||||
)
|
||||
|
||||
|
||||
def refresh_exchange_eligibility(
|
||||
db: Session, *, task_id: int, current: User
|
||||
) -> dict[str, Any]:
|
||||
|
||||
@@ -19,6 +19,7 @@ from typing import Any
|
||||
|
||||
class TaskLane(StrEnum):
|
||||
INTERACTIVE_BINDING = "interactive_binding"
|
||||
BATCH_COORDINATOR = "batch_coordinator"
|
||||
BINDING_POLL = "binding_poll"
|
||||
LOGIN = "login"
|
||||
ACCOUNT_CHECK = "account_check"
|
||||
@@ -51,6 +52,7 @@ class LaneConfig:
|
||||
|
||||
_DEFAULT_WORKERS: dict[TaskLane, int] = {
|
||||
TaskLane.INTERACTIVE_BINDING: 3,
|
||||
TaskLane.BATCH_COORDINATOR: 2,
|
||||
TaskLane.BINDING_POLL: 2,
|
||||
TaskLane.LOGIN: 4,
|
||||
TaskLane.ACCOUNT_CHECK: 4,
|
||||
@@ -71,9 +73,19 @@ _DEFAULT_WORKERS: dict[TaskLane, int] = {
|
||||
def _lane_config(lane: TaskLane) -> LaneConfig:
|
||||
key = lane.value.upper()
|
||||
workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane])
|
||||
default_in_flight = (
|
||||
workers
|
||||
if lane
|
||||
in {
|
||||
TaskLane.BATCH_COORDINATOR,
|
||||
TaskLane.BINDING_POLL,
|
||||
TaskLane.PAYMENT_POLL,
|
||||
}
|
||||
else max(workers * 32, workers)
|
||||
)
|
||||
max_in_flight = _env_int(
|
||||
f"TASK_LANE_{key}_MAX_IN_FLIGHT",
|
||||
max(workers * 32, workers),
|
||||
default_in_flight,
|
||||
)
|
||||
return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers))
|
||||
|
||||
@@ -107,6 +119,36 @@ def _runtime(lane: TaskLane) -> _LaneRuntime:
|
||||
return runtime
|
||||
|
||||
|
||||
def _submit[T](
|
||||
lane: TaskLane,
|
||||
fn: Callable[..., T],
|
||||
/,
|
||||
*args: Any,
|
||||
stopped: Callable[[], bool] | None = None,
|
||||
wait: bool,
|
||||
**kwargs: Any,
|
||||
) -> Future[T] | None:
|
||||
runtime = _runtime(lane)
|
||||
if stopped is not None and stopped():
|
||||
return None
|
||||
if wait:
|
||||
while True:
|
||||
if stopped is not None and stopped():
|
||||
return None
|
||||
if runtime.admission.acquire(timeout=0.2):
|
||||
break
|
||||
elif not runtime.admission.acquire(blocking=False):
|
||||
return None
|
||||
|
||||
def run() -> T:
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
runtime.admission.release()
|
||||
|
||||
return runtime.executor.submit(run)
|
||||
|
||||
|
||||
def submit[T](
|
||||
lane: TaskLane,
|
||||
fn: Callable[..., T],
|
||||
@@ -120,20 +162,39 @@ def submit[T](
|
||||
Batch coordinators are cheap threads and are allowed to block here. Worker
|
||||
threads themselves must not recursively submit to the same lane.
|
||||
"""
|
||||
return _submit(
|
||||
lane,
|
||||
fn,
|
||||
*args,
|
||||
stopped=stopped,
|
||||
wait=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def try_submit[T](
|
||||
lane: TaskLane,
|
||||
fn: Callable[..., T],
|
||||
/,
|
||||
*args: Any,
|
||||
stopped: Callable[[], bool] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Future[T] | None:
|
||||
"""Submit without blocking when the lane admission budget is full."""
|
||||
return _submit(
|
||||
lane,
|
||||
fn,
|
||||
*args,
|
||||
stopped=stopped,
|
||||
wait=False,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def available_slots(lane: TaskLane) -> int:
|
||||
"""Return the number of tasks that can currently enter a lane immediately."""
|
||||
runtime = _runtime(lane)
|
||||
while True:
|
||||
if stopped is not None and stopped():
|
||||
return None
|
||||
if runtime.admission.acquire(timeout=0.2):
|
||||
break
|
||||
|
||||
def run() -> T:
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
runtime.admission.release()
|
||||
|
||||
return runtime.executor.submit(run)
|
||||
return max(0, int(runtime.admission._value))
|
||||
|
||||
|
||||
def lane_stats() -> dict[str, dict[str, int]]:
|
||||
@@ -194,8 +255,10 @@ def douyu_lane_for_task(task_type: str) -> TaskLane:
|
||||
|
||||
__all__ = [
|
||||
"TaskLane",
|
||||
"available_slots",
|
||||
"douyu_lane_for_task",
|
||||
"lane_stats",
|
||||
"shutdown",
|
||||
"submit",
|
||||
"try_submit",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user