perf(douyu): 限制任务并发并释放长事务连接

This commit is contained in:
yml
2026-11-20 20:16:53 +08:00
parent 5d44ab4ca1
commit 18fd9840f7
16 changed files with 331 additions and 27 deletions
+43 -1
View File
@@ -1,7 +1,10 @@
import inspect
import threading
import time
import pytest
from fastapi import HTTPException
from pydantic import ValidationError
from sqlalchemy import text
from tests.support.db import make_test_session
@@ -9,7 +12,8 @@ from web.backend.database import engine as app_engine
from web.backend.main import health
from web.backend.models import Account, DouyuTask, User
from web.backend.routers import douyu as douyu_router
from web.backend.services import business_events, douyu_service
from web.backend.schemas import DouyuTaskBatchRequest
from web.backend.services import business_events, douyu_concurrency, douyu_service
from web.backend.services.douyu_runner_core import (
DouyuBatchRunnerCore,
slim_douyu_task_result_for_storage,
@@ -193,6 +197,44 @@ def test_active_batch_budget_returns_429(monkeypatch):
assert exc_info.value.status_code == 429
def test_douyu_batch_request_rejects_more_than_five_workers():
with pytest.raises(ValidationError):
DouyuTaskBatchRequest(
account_ids=[1],
task_type="query_points",
handbook_scope="elite",
concurrency=6,
)
def test_batch_task_concurrency_is_capped_at_five():
assert douyu_concurrency.clamp_batch_task_concurrency(20) == 5
def test_overview_limiter_blocks_extra_workers(monkeypatch):
slots = threading.BoundedSemaphore(1)
monkeypatch.setattr(douyu_concurrency, "_overview_slots", slots)
acquired: list[bool] = []
started = threading.Event()
assert douyu_concurrency.acquire_overview_slot(lambda: False) is True
def worker():
started.set()
acquired.append(douyu_concurrency.acquire_overview_slot(lambda: False))
thread = threading.Thread(target=worker)
thread.start()
started.wait(timeout=1)
time.sleep(0.05)
assert acquired == []
douyu_concurrency.release_overview_slot()
thread.join(timeout=1)
assert acquired == [True]
douyu_concurrency.release_overview_slot()
def test_slim_task_result_removes_debug_payloads():
result = slim_douyu_task_result_for_storage(
{