fix(platform): 防止绑定任务压垮连接池和事件循环
This commit is contained in:
@@ -82,14 +82,18 @@ DOUYU_MAX_ACTIVE_BATCHES=10
|
||||
# 一键查询使用独立任务槽;普通查询仍为 3 个 worker。
|
||||
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS=5
|
||||
# 业务域任务槽。支付出码与到账轮询分离后,各自按实际耗时独立限流。
|
||||
TASK_LANE_BATCH_COORDINATOR_WORKERS=2
|
||||
TASK_LANE_BATCH_COORDINATOR_MAX_IN_FLIGHT=2
|
||||
TASK_LANE_INTERACTIVE_BINDING_WORKERS=3
|
||||
TASK_LANE_BINDING_POLL_WORKERS=2
|
||||
TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT=2
|
||||
TASK_LANE_LOGIN_WORKERS=4
|
||||
TASK_LANE_ACCOUNT_CHECK_WORKERS=4
|
||||
TASK_LANE_OVERVIEW_QUERY_WORKERS=5
|
||||
TASK_LANE_QUERY_BATCH_WORKERS=3
|
||||
TASK_LANE_PAYMENT_QR_WORKERS=10
|
||||
TASK_LANE_PAYMENT_POLL_WORKERS=3
|
||||
TASK_LANE_PAYMENT_POLL_MAX_IN_FLIGHT=3
|
||||
TASK_LANE_OPS_WRITE_WORKERS=2
|
||||
DOUYU_MAX_ACTIVE_TASKS=250
|
||||
DOUYU_PROBE_CACHE_SECONDS=30
|
||||
@@ -191,6 +195,8 @@ PLATFORM_FULFILLMENT_RECONCILE_SECONDS=900
|
||||
ACCOUNT_COMMODITY_STATE_TTL_SECONDS=900
|
||||
# legacy=旧实时扫号 / hybrid=能力索引优先、失败回退旧逻辑 / capability=只读能力索引
|
||||
PLATFORM_FULFILLMENT_MATCH_MODE=hybrid
|
||||
# 订单 Webhook 连续唤醒时保持最小选号间隔,避免忙轮询压满数据库。
|
||||
PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS=5
|
||||
# 客户绑定二维码有效期
|
||||
PLATFORM_BINDING_QR_TTL_SECONDS=600
|
||||
|
||||
|
||||
@@ -49,14 +49,18 @@ services:
|
||||
- DOUYU_PAYMENT_QR_CONCURRENCY=${DOUYU_PAYMENT_QR_CONCURRENCY:-10}
|
||||
- DOUYU_MAX_ACTIVE_BATCHES=${DOUYU_MAX_ACTIVE_BATCHES:-10}
|
||||
- DOUYU_MAX_ACTIVE_OVERVIEW_TASKS=${DOUYU_MAX_ACTIVE_OVERVIEW_TASKS:-5}
|
||||
- TASK_LANE_BATCH_COORDINATOR_WORKERS=${TASK_LANE_BATCH_COORDINATOR_WORKERS:-2}
|
||||
- TASK_LANE_BATCH_COORDINATOR_MAX_IN_FLIGHT=${TASK_LANE_BATCH_COORDINATOR_MAX_IN_FLIGHT:-2}
|
||||
- TASK_LANE_INTERACTIVE_BINDING_WORKERS=${TASK_LANE_INTERACTIVE_BINDING_WORKERS:-3}
|
||||
- TASK_LANE_BINDING_POLL_WORKERS=${TASK_LANE_BINDING_POLL_WORKERS:-2}
|
||||
- TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT=${TASK_LANE_BINDING_POLL_MAX_IN_FLIGHT:-2}
|
||||
- TASK_LANE_LOGIN_WORKERS=${TASK_LANE_LOGIN_WORKERS:-4}
|
||||
- TASK_LANE_ACCOUNT_CHECK_WORKERS=${TASK_LANE_ACCOUNT_CHECK_WORKERS:-4}
|
||||
- TASK_LANE_OVERVIEW_QUERY_WORKERS=${TASK_LANE_OVERVIEW_QUERY_WORKERS:-5}
|
||||
- TASK_LANE_QUERY_BATCH_WORKERS=${TASK_LANE_QUERY_BATCH_WORKERS:-3}
|
||||
- TASK_LANE_PAYMENT_QR_WORKERS=${TASK_LANE_PAYMENT_QR_WORKERS:-10}
|
||||
- TASK_LANE_PAYMENT_POLL_WORKERS=${TASK_LANE_PAYMENT_POLL_WORKERS:-3}
|
||||
- TASK_LANE_PAYMENT_POLL_MAX_IN_FLIGHT=${TASK_LANE_PAYMENT_POLL_MAX_IN_FLIGHT:-3}
|
||||
- TASK_LANE_OPS_WRITE_WORKERS=${TASK_LANE_OPS_WRITE_WORKERS:-2}
|
||||
- DOUYU_MAX_ACTIVE_TASKS=${DOUYU_MAX_ACTIVE_TASKS:-250}
|
||||
- DOUYU_PROBE_CACHE_SECONDS=${DOUYU_PROBE_CACHE_SECONDS:-30}
|
||||
@@ -112,6 +116,7 @@ services:
|
||||
- PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID=${PLATFORM_FULFILLMENT_INTEGRATION_ACTOR_ID:-}
|
||||
- ACCOUNT_COMMODITY_STATE_TTL_SECONDS=${ACCOUNT_COMMODITY_STATE_TTL_SECONDS:-900}
|
||||
- PLATFORM_FULFILLMENT_MATCH_MODE=${PLATFORM_FULFILLMENT_MATCH_MODE:-hybrid}
|
||||
- PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS=${PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS:-5}
|
||||
- PLATFORM_BINDING_QR_TTL_SECONDS=${PLATFORM_BINDING_QR_TTL_SECONDS:-600}
|
||||
# 用户上传图片的桶(在线协作 / P2 资料收集共用;生产用内网 endpoint)。
|
||||
# OSS_MODE=oss 才走对象存储,否则落容器内 ./data/file_storage。
|
||||
|
||||
@@ -15,6 +15,7 @@ from web.backend.schemas import DouyuTaskBatchRequest
|
||||
from web.backend.services import (
|
||||
binding_poll_scheduler,
|
||||
business_events,
|
||||
douyu_batch_launcher,
|
||||
douyu_concurrency,
|
||||
douyu_service,
|
||||
payment_poll_scheduler,
|
||||
@@ -225,6 +226,61 @@ def test_binding_poll_scheduler_claims_due_task_without_worker_wait(db):
|
||||
binding_poll_scheduler._in_flight.discard(task.id)
|
||||
|
||||
|
||||
def test_binding_poll_dispatch_skips_claim_when_lane_is_saturated(monkeypatch):
|
||||
claimed_limits: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
binding_poll_scheduler,
|
||||
"available_slots",
|
||||
lambda _lane: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
binding_poll_scheduler,
|
||||
"_claim_due_binding_tasks",
|
||||
lambda limit: claimed_limits.append(limit) or [],
|
||||
)
|
||||
|
||||
submitted = binding_poll_scheduler.run_binding_poll_dispatch_once(20)
|
||||
|
||||
assert submitted == 0
|
||||
assert claimed_limits == []
|
||||
|
||||
|
||||
def test_batch_launcher_defers_database_session_to_worker(monkeypatch):
|
||||
submitted: dict[str, object] = {}
|
||||
|
||||
def fake_submit(lane, fn, /, *args, **kwargs):
|
||||
submitted.update(
|
||||
{
|
||||
"lane": lane,
|
||||
"fn": fn,
|
||||
"args": args,
|
||||
"kwargs": kwargs,
|
||||
}
|
||||
)
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(
|
||||
douyu_batch_launcher,
|
||||
"WorkerSessionLocal",
|
||||
lambda: (_ for _ in ()).throw(
|
||||
AssertionError("队列阶段不能创建数据库 Session")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(douyu_batch_launcher, "submit", fake_submit)
|
||||
|
||||
queued = douyu_batch_launcher.start_douyu_batch(
|
||||
batch_id="deferred-session",
|
||||
task_type="get_bind_qr",
|
||||
payload={},
|
||||
concurrency=1,
|
||||
capture_event_loop=False,
|
||||
)
|
||||
|
||||
assert queued is True
|
||||
assert submitted["lane"] == TaskLane.BATCH_COORDINATOR
|
||||
assert submitted["fn"] is douyu_batch_launcher._run_batch
|
||||
|
||||
|
||||
def test_payment_poll_scheduler_claims_due_task_without_worker_wait(db):
|
||||
_, _, task = _fixtures(db, status="running")
|
||||
due_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
|
||||
@@ -42,7 +42,7 @@ TESTS_DIR = Path(__file__).resolve().parent
|
||||
PROJECT_ROOT = TESTS_DIR.parent
|
||||
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
|
||||
|
||||
HEAD_REVISION = "20261121_0107"
|
||||
HEAD_REVISION = "20261121_0108"
|
||||
|
||||
# 迁移新增、但不声明在模型里的复合查询索引。
|
||||
EXTRA_INDEXES = (
|
||||
|
||||
@@ -429,6 +429,10 @@
|
||||
"20261121_0107_repair_douyu_active_binding_key.py": {
|
||||
"revision": "20261121_0107",
|
||||
"sha256": "29539b5fab9c6d8eaf7219b7d19376dfa6032d8b6cf02b76fc2a843c09891302"
|
||||
},
|
||||
"20261121_0108_account_pool_query_indexes.py": {
|
||||
"revision": "20261121_0108",
|
||||
"sha256": "b45a7f0c563e43e6516cf00ad12ce7a5fb78bb7bde6b3f8de02cb888ad761666"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""为账号池自动选号增加组合索引。
|
||||
|
||||
自动选号会反复执行 accounts 候选扫描,并用 NOT EXISTS 排除活动协作单。
|
||||
没有可用的组合索引时,生产库每次都要扫描约四万行账号再排序,造成 MySQL CPU
|
||||
和请求延迟一起放大。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20261121_0108"
|
||||
down_revision: str | None = "20261121_0107"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
ACCOUNT_INDEX = "ix_accounts_pool_candidate"
|
||||
COLLAB_INDEX = "ix_douyu_external_orders_active_account"
|
||||
|
||||
|
||||
def _index_exists(table: str, name: str) -> bool:
|
||||
conn = op.get_bind()
|
||||
return bool(
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"SELECT COUNT(*) FROM information_schema.statistics "
|
||||
"WHERE table_schema = DATABASE() "
|
||||
"AND table_name = :table AND index_name = :name"
|
||||
),
|
||||
{"table": table, "name": name},
|
||||
).scalar()
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _index_exists("accounts", ACCOUNT_INDEX):
|
||||
op.create_index(
|
||||
ACCOUNT_INDEX,
|
||||
"accounts",
|
||||
[
|
||||
"auto_managed",
|
||||
"status",
|
||||
"role_change_available_at",
|
||||
"ck_check_status",
|
||||
"points_expire_at",
|
||||
"points",
|
||||
"id",
|
||||
],
|
||||
)
|
||||
if not _index_exists("douyu_external_orders", COLLAB_INDEX):
|
||||
op.create_index(
|
||||
COLLAB_INDEX,
|
||||
"douyu_external_orders",
|
||||
["account_id", "archived_at", "status"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _index_exists("douyu_external_orders", COLLAB_INDEX):
|
||||
op.drop_index(COLLAB_INDEX, table_name="douyu_external_orders")
|
||||
if _index_exists("accounts", ACCOUNT_INDEX):
|
||||
op.drop_index(ACCOUNT_INDEX, table_name="accounts")
|
||||
@@ -270,6 +270,18 @@ class Account(Base):
|
||||
Index("ix_accounts_status", "status"),
|
||||
# 高价值三态的过期判定和待刷新筛选
|
||||
Index("ix_accounts_high_value_checked_at", "high_value_checked_at"),
|
||||
# 账号池选号:先按自动管理/生命周期过滤,再按 CK、积分和到期时间排序。
|
||||
# 这条索引避免每次选号都回表扫描全部账号后再排序。
|
||||
Index(
|
||||
"ix_accounts_pool_candidate",
|
||||
"auto_managed",
|
||||
"status",
|
||||
"role_change_available_at",
|
||||
"ck_check_status",
|
||||
"points_expire_at",
|
||||
"points",
|
||||
"id",
|
||||
),
|
||||
)
|
||||
|
||||
assigned_user: Mapped[User | None] = relationship(
|
||||
@@ -800,6 +812,13 @@ class DouyuExternalOrder(Base):
|
||||
"source_task_id",
|
||||
unique=True,
|
||||
),
|
||||
# 账号池排除活动协作单的 NOT EXISTS 每轮都会执行。
|
||||
Index(
|
||||
"ix_douyu_external_orders_active_account",
|
||||
"account_id",
|
||||
"archived_at",
|
||||
"status",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from sqlalchemy.orm import joinedload
|
||||
from ..database import WorkerSessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .douyu_runner import DouyuBatchRunner
|
||||
from .task_runtime import TaskLane, submit
|
||||
from .task_runtime import TaskLane, available_slots, try_submit
|
||||
|
||||
DEFAULT_INTERVAL_SECONDS = 1
|
||||
DEFAULT_BATCH_SIZE = 20
|
||||
@@ -126,10 +126,13 @@ def _release_callback(_future: Future) -> None:
|
||||
|
||||
|
||||
def run_binding_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
|
||||
claimed = _claim_due_binding_tasks(max(1, limit))
|
||||
capacity = available_slots(TaskLane.BINDING_POLL)
|
||||
if capacity <= 0:
|
||||
return 0
|
||||
claimed = _claim_due_binding_tasks(min(max(1, limit), capacity))
|
||||
submitted = 0
|
||||
for task_id in claimed:
|
||||
future = submit(
|
||||
future = try_submit(
|
||||
TaskLane.BINDING_POLL,
|
||||
poll_binding_task_once,
|
||||
task_id,
|
||||
|
||||
@@ -7,12 +7,12 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
|
||||
from web.backend.database import WorkerSessionLocal
|
||||
|
||||
from .douyu_concurrency import clamp_douyu_task_concurrency
|
||||
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||
from .task_runtime import TaskLane, submit
|
||||
|
||||
|
||||
def _env_positive_int(name: str, default: int) -> int:
|
||||
@@ -30,6 +30,33 @@ DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
|
||||
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 10)
|
||||
|
||||
|
||||
def _run_batch(
|
||||
*,
|
||||
batch_id: str,
|
||||
task_type: str,
|
||||
payload: dict,
|
||||
concurrency: int,
|
||||
loop: asyncio.AbstractEventLoop | None,
|
||||
log_queue: asyncio.Queue | None,
|
||||
) -> None:
|
||||
"""Run one coordinator with a short-lived session owned by the worker."""
|
||||
thread_db = WorkerSessionLocal()
|
||||
try:
|
||||
runner = DouyuBatchRunner(
|
||||
db=thread_db,
|
||||
batch_id=batch_id,
|
||||
task_type=task_type,
|
||||
payload=payload,
|
||||
log_queue=log_queue,
|
||||
loop=loop,
|
||||
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
|
||||
)
|
||||
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
|
||||
runner.run()
|
||||
finally:
|
||||
thread_db.close()
|
||||
|
||||
|
||||
def start_douyu_batch(
|
||||
*,
|
||||
batch_id: str,
|
||||
@@ -39,8 +66,14 @@ def start_douyu_batch(
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
log_queue: asyncio.Queue | None = None,
|
||||
capture_event_loop: bool = True,
|
||||
) -> None:
|
||||
"""Create the batch coordinator; websocket logging is optional."""
|
||||
) -> bool:
|
||||
"""Queue a batch coordinator without creating a thread per order.
|
||||
|
||||
The coordinator uses the dedicated bounded lane and opens its database
|
||||
session only after a worker slot is available. Callers may therefore
|
||||
safely invoke this from a request thread without holding a database
|
||||
connection while waiting in the queue.
|
||||
"""
|
||||
if loop is None and capture_event_loop:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -48,18 +81,17 @@ def start_douyu_batch(
|
||||
loop = None
|
||||
if log_queue is None and loop is not None:
|
||||
log_queue = asyncio.Queue()
|
||||
thread_db = WorkerSessionLocal()
|
||||
runner = DouyuBatchRunner(
|
||||
db=thread_db,
|
||||
future = submit(
|
||||
TaskLane.BATCH_COORDINATOR,
|
||||
_run_batch,
|
||||
batch_id=batch_id,
|
||||
task_type=task_type,
|
||||
payload=payload,
|
||||
log_queue=log_queue,
|
||||
concurrency=concurrency,
|
||||
loop=loop,
|
||||
concurrency=clamp_douyu_task_concurrency(task_type, concurrency),
|
||||
log_queue=log_queue,
|
||||
)
|
||||
douyu_batch_registry.register(batch_id, log_queue, loop, runner)
|
||||
threading.Thread(target=runner.run, daemon=True).start()
|
||||
return future is not None
|
||||
|
||||
|
||||
def stop_douyu_batches(batch_ids: list[str]) -> int:
|
||||
|
||||
@@ -14,7 +14,7 @@ from ..database import WorkerSessionLocal
|
||||
from ..models import DouyuTask
|
||||
from .douyu_runner import DouyuBatchRunner
|
||||
from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL
|
||||
from .task_runtime import TaskLane, submit
|
||||
from .task_runtime import TaskLane, available_slots, try_submit
|
||||
|
||||
DEFAULT_INTERVAL_SECONDS = 1
|
||||
DEFAULT_BATCH_SIZE = 30
|
||||
@@ -182,10 +182,13 @@ def _release_callback(future: Future) -> None:
|
||||
|
||||
|
||||
def run_payment_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
|
||||
claimed = _claim_due_payment_tasks(max(1, limit))
|
||||
capacity = available_slots(TaskLane.PAYMENT_POLL)
|
||||
if capacity <= 0:
|
||||
return 0
|
||||
claimed = _claim_due_payment_tasks(min(max(1, limit), capacity))
|
||||
submitted = 0
|
||||
for task_id in claimed:
|
||||
future = submit(
|
||||
future = try_submit(
|
||||
TaskLane.PAYMENT_POLL,
|
||||
poll_payment_task_once,
|
||||
task_id,
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
@@ -18,6 +18,7 @@ from ...models import PlatformFulfillmentTask
|
||||
from . import service
|
||||
|
||||
DEFAULT_INTERVAL_SECONDS = 20
|
||||
DEFAULT_MIN_INTERVAL_SECONDS = 5
|
||||
DEFAULT_BATCH_SIZE = 2
|
||||
DEFAULT_RETRY_SECONDS = 45
|
||||
|
||||
@@ -144,12 +145,10 @@ def run_auto_allocation_once(
|
||||
allocated_task.next_allocate_at = None
|
||||
db.commit()
|
||||
try:
|
||||
asyncio.run(
|
||||
service.start_binding(
|
||||
db,
|
||||
task_id=candidate_task_id,
|
||||
current=actor,
|
||||
)
|
||||
service.start_binding_sync(
|
||||
db,
|
||||
task_id=candidate_task_id,
|
||||
current=actor,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - allocation remains valid
|
||||
db.rollback()
|
||||
@@ -211,12 +210,10 @@ def start_allocated_bindings_once(
|
||||
started = 0
|
||||
for task in rows:
|
||||
try:
|
||||
asyncio.run(
|
||||
service.start_binding(
|
||||
db,
|
||||
task_id=int(task.id),
|
||||
current=actor,
|
||||
)
|
||||
service.start_binding_sync(
|
||||
db,
|
||||
task_id=int(task.id),
|
||||
current=actor,
|
||||
)
|
||||
started += 1
|
||||
except Exception as exc: # noqa: BLE001 - one session cannot block the rest
|
||||
@@ -237,6 +234,7 @@ def _run() -> None:
|
||||
3600,
|
||||
)
|
||||
while not _stop_event.is_set():
|
||||
tick_started = time.monotonic()
|
||||
db = WorkerSessionLocal()
|
||||
try:
|
||||
pending_ids = _take_pending_task_ids(
|
||||
@@ -260,6 +258,15 @@ def _run() -> None:
|
||||
db.close()
|
||||
if _stop_event.is_set():
|
||||
break
|
||||
min_interval = _env_int(
|
||||
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_MIN_INTERVAL_SECONDS",
|
||||
DEFAULT_MIN_INTERVAL_SECONDS,
|
||||
1,
|
||||
60,
|
||||
)
|
||||
remaining = min_interval - (time.monotonic() - tick_started)
|
||||
if remaining > 0 and _stop_event.wait(remaining):
|
||||
break
|
||||
if _has_pending_task_ids():
|
||||
continue
|
||||
_wake_event.wait(interval)
|
||||
|
||||
@@ -14,6 +14,7 @@ from loguru import logger
|
||||
from sqlalchemy import func, or_
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from ...models import (
|
||||
Account,
|
||||
@@ -2433,7 +2434,7 @@ def reset_task_binding(
|
||||
return get_task_detail(db, task.id)
|
||||
|
||||
|
||||
async def start_binding(
|
||||
def start_binding_sync(
|
||||
db: Session,
|
||||
*,
|
||||
task_id: int,
|
||||
@@ -2536,6 +2537,23 @@ async def start_binding(
|
||||
return get_task_detail(db, task.id)
|
||||
|
||||
|
||||
async def start_binding(
|
||||
db: Session,
|
||||
*,
|
||||
task_id: int,
|
||||
current: User,
|
||||
force_refresh: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the blocking DB/network dispatch path outside the ASGI event loop."""
|
||||
return await run_in_threadpool(
|
||||
start_binding_sync,
|
||||
db,
|
||||
task_id=task_id,
|
||||
current=current,
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
|
||||
|
||||
def refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
|
||||
task = _load_task(db, task_id)
|
||||
if task.execution_status in {"completed", "cancelled"}:
|
||||
@@ -2677,7 +2695,7 @@ def reconcile_binding_state_outbox(
|
||||
}
|
||||
|
||||
|
||||
async def confirm_binding(
|
||||
def confirm_binding_sync(
|
||||
db: Session, *, task_id: int, current: User
|
||||
) -> dict[str, Any]:
|
||||
task = _load_task_for_update(db, task_id)
|
||||
@@ -2754,6 +2772,17 @@ async def confirm_binding(
|
||||
return get_task_detail(db, task.id)
|
||||
|
||||
|
||||
async def confirm_binding(
|
||||
db: Session, *, task_id: int, current: User
|
||||
) -> dict[str, Any]:
|
||||
return await run_in_threadpool(
|
||||
confirm_binding_sync,
|
||||
db,
|
||||
task_id=task_id,
|
||||
current=current,
|
||||
)
|
||||
|
||||
|
||||
def refresh_exchange_eligibility(
|
||||
db: Session, *, task_id: int, current: User
|
||||
) -> dict[str, Any]:
|
||||
|
||||
@@ -19,6 +19,7 @@ from typing import Any
|
||||
|
||||
class TaskLane(StrEnum):
|
||||
INTERACTIVE_BINDING = "interactive_binding"
|
||||
BATCH_COORDINATOR = "batch_coordinator"
|
||||
BINDING_POLL = "binding_poll"
|
||||
LOGIN = "login"
|
||||
ACCOUNT_CHECK = "account_check"
|
||||
@@ -51,6 +52,7 @@ class LaneConfig:
|
||||
|
||||
_DEFAULT_WORKERS: dict[TaskLane, int] = {
|
||||
TaskLane.INTERACTIVE_BINDING: 3,
|
||||
TaskLane.BATCH_COORDINATOR: 2,
|
||||
TaskLane.BINDING_POLL: 2,
|
||||
TaskLane.LOGIN: 4,
|
||||
TaskLane.ACCOUNT_CHECK: 4,
|
||||
@@ -71,9 +73,19 @@ _DEFAULT_WORKERS: dict[TaskLane, int] = {
|
||||
def _lane_config(lane: TaskLane) -> LaneConfig:
|
||||
key = lane.value.upper()
|
||||
workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane])
|
||||
default_in_flight = (
|
||||
workers
|
||||
if lane
|
||||
in {
|
||||
TaskLane.BATCH_COORDINATOR,
|
||||
TaskLane.BINDING_POLL,
|
||||
TaskLane.PAYMENT_POLL,
|
||||
}
|
||||
else max(workers * 32, workers)
|
||||
)
|
||||
max_in_flight = _env_int(
|
||||
f"TASK_LANE_{key}_MAX_IN_FLIGHT",
|
||||
max(workers * 32, workers),
|
||||
default_in_flight,
|
||||
)
|
||||
return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers))
|
||||
|
||||
@@ -107,6 +119,36 @@ def _runtime(lane: TaskLane) -> _LaneRuntime:
|
||||
return runtime
|
||||
|
||||
|
||||
def _submit[T](
|
||||
lane: TaskLane,
|
||||
fn: Callable[..., T],
|
||||
/,
|
||||
*args: Any,
|
||||
stopped: Callable[[], bool] | None = None,
|
||||
wait: bool,
|
||||
**kwargs: Any,
|
||||
) -> Future[T] | None:
|
||||
runtime = _runtime(lane)
|
||||
if stopped is not None and stopped():
|
||||
return None
|
||||
if wait:
|
||||
while True:
|
||||
if stopped is not None and stopped():
|
||||
return None
|
||||
if runtime.admission.acquire(timeout=0.2):
|
||||
break
|
||||
elif not runtime.admission.acquire(blocking=False):
|
||||
return None
|
||||
|
||||
def run() -> T:
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
runtime.admission.release()
|
||||
|
||||
return runtime.executor.submit(run)
|
||||
|
||||
|
||||
def submit[T](
|
||||
lane: TaskLane,
|
||||
fn: Callable[..., T],
|
||||
@@ -120,20 +162,39 @@ def submit[T](
|
||||
Batch coordinators are cheap threads and are allowed to block here. Worker
|
||||
threads themselves must not recursively submit to the same lane.
|
||||
"""
|
||||
return _submit(
|
||||
lane,
|
||||
fn,
|
||||
*args,
|
||||
stopped=stopped,
|
||||
wait=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def try_submit[T](
|
||||
lane: TaskLane,
|
||||
fn: Callable[..., T],
|
||||
/,
|
||||
*args: Any,
|
||||
stopped: Callable[[], bool] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Future[T] | None:
|
||||
"""Submit without blocking when the lane admission budget is full."""
|
||||
return _submit(
|
||||
lane,
|
||||
fn,
|
||||
*args,
|
||||
stopped=stopped,
|
||||
wait=False,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def available_slots(lane: TaskLane) -> int:
|
||||
"""Return the number of tasks that can currently enter a lane immediately."""
|
||||
runtime = _runtime(lane)
|
||||
while True:
|
||||
if stopped is not None and stopped():
|
||||
return None
|
||||
if runtime.admission.acquire(timeout=0.2):
|
||||
break
|
||||
|
||||
def run() -> T:
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
runtime.admission.release()
|
||||
|
||||
return runtime.executor.submit(run)
|
||||
return max(0, int(runtime.admission._value))
|
||||
|
||||
|
||||
def lane_stats() -> dict[str, dict[str, int]]:
|
||||
@@ -194,8 +255,10 @@ def douyu_lane_for_task(task_type: str) -> TaskLane:
|
||||
|
||||
__all__ = [
|
||||
"TaskLane",
|
||||
"available_slots",
|
||||
"douyu_lane_for_task",
|
||||
"lane_stats",
|
||||
"shutdown",
|
||||
"submit",
|
||||
"try_submit",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user