Files
live-hub-py/tests/test_douyu_task_concurrency.py
T

270 lines
7.5 KiB
Python

import inspect
import threading
import time
import pytest
from fastapi import HTTPException
from pydantic import ValidationError
from sqlalchemy import text
from tests.support.db import make_test_session
from web.backend.database import engine as app_engine
from web.backend.main import health
from web.backend.models import Account, DouyuTask, User
from web.backend.routers import douyu as douyu_router
from web.backend.schemas import DouyuTaskBatchRequest
from web.backend.services import business_events, douyu_concurrency, douyu_service
from web.backend.services.douyu_runner_core import (
DouyuBatchRunnerCore,
slim_douyu_task_result_for_storage,
)
def _fixtures(db, *, status: str = "planned") -> tuple[User, Account, DouyuTask]:
user = User(username="concurrency-user", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="concurrency-account",
password="password",
email="concurrency@example.com",
email_password="mail-password",
cookie="acf_uid=1",
)
db.add(account)
db.flush()
task = DouyuTask(
batch_id="concurrency-batch",
account_id=account.id,
account_assigned_to=user.id,
task_type="get_bind_qr",
handbook_scope="elite",
status=status,
created_by=user.id,
)
db.add(task)
db.commit()
return user, account, task
def test_task_rows_commit_before_started_events(db, monkeypatch):
user = User(username="event-order", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="event-order-account",
password="password",
email="event-order@example.com",
email_password="mail-password",
cookie="acf_uid=2",
)
db.add(account)
db.commit()
observed: list[int] = []
def fake_emit_bulk_read_committed(_db, events):
other = make_test_session()
try:
observed.append(
other.query(DouyuTask).filter(DouyuTask.status == "planned").count()
)
finally:
other.close()
return len(events)
monkeypatch.setattr(douyu_service, "emit_bulk", fake_emit_bulk_read_committed)
batch_id, count = douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_points",
"elite",
user.id,
)
assert batch_id
assert count == 1
assert observed == [1]
def test_mark_task_commits_result_before_account_event(db, monkeypatch):
_, account, task = _fixtures(db, status="running")
task.account = account
runner = DouyuBatchRunnerCore(
db,
task.batch_id,
task.task_type,
close_db=False,
)
observed: list[tuple[str, object]] = []
def fake_record(_db, _account, **kwargs):
other = make_test_session()
try:
row = other.get(DouyuTask, task.id)
observed.append((row.status, row.result))
finally:
other.close()
return True
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
runner._mark_task(
db,
task,
"success",
"done",
{
"url": "https://example.test/qr",
"raw": {"large": "payload"},
"before_bind_info": {
"role_name": "role-a",
"raw": {"large": "nested"},
},
},
)
assert observed == [
(
"success",
{
"url": "https://example.test/qr",
"before_bind_info": {"role_name": "role-a"},
},
)
]
def test_cleanup_commits_stopped_status_before_account_event(db, monkeypatch):
_, _, task = _fixtures(db, status="running")
observed: list[str] = []
def fake_record(_db, _account, **kwargs):
other = make_test_session()
try:
observed.append(other.get(DouyuTask, task.id).status)
finally:
other.close()
return True
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
cleaned = douyu_service.cleanup_orphan_douyu_tasks(
db,
active_batch_ids=set(),
statuses=("running",),
)
assert cleaned == 1
assert observed == ["stopped"]
def test_active_task_budget_rejects_new_batch(db, monkeypatch):
user = User(username="budget-user", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="budget-account",
password="password",
email="budget@example.com",
email_password="mail-password",
cookie="acf_uid=3",
)
db.add(account)
db.commit()
monkeypatch.setattr(douyu_service, "DOUYU_MAX_ACTIVE_TASKS", 1)
douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_points",
"elite",
user.id,
)
with pytest.raises(douyu_service.DouyuTaskBackpressureError, match="上限 1"):
douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_game_name",
"elite",
user.id,
)
def test_active_batch_budget_returns_429(monkeypatch):
monkeypatch.setattr(douyu_router, "DOUYU_MAX_ACTIVE_BATCHES", 1)
with pytest.raises(HTTPException) as exc_info:
douyu_router._ensure_active_batch_budget(1)
assert exc_info.value.status_code == 429
def test_douyu_batch_request_rejects_more_than_five_workers():
with pytest.raises(ValidationError):
DouyuTaskBatchRequest(
account_ids=[1],
task_type="query_points",
handbook_scope="elite",
concurrency=6,
)
def test_batch_task_concurrency_is_capped_at_five():
assert douyu_concurrency.clamp_batch_task_concurrency(20) == 5
def test_overview_limiter_blocks_extra_workers(monkeypatch):
slots = threading.BoundedSemaphore(1)
monkeypatch.setattr(douyu_concurrency, "_overview_slots", slots)
acquired: list[bool] = []
started = threading.Event()
assert douyu_concurrency.acquire_overview_slot(lambda: False) is True
def worker():
started.set()
acquired.append(douyu_concurrency.acquire_overview_slot(lambda: False))
thread = threading.Thread(target=worker)
thread.start()
started.wait(timeout=1)
time.sleep(0.05)
assert acquired == []
douyu_concurrency.release_overview_slot()
thread.join(timeout=1)
assert acquired == [True]
douyu_concurrency.release_overview_slot()
def test_slim_task_result_removes_debug_payloads():
result = slim_douyu_task_result_for_storage(
{
"url": "https://example.test/qr",
"raw": {"debug": True},
"query_params": {"token": "secret"},
"before_bind_info": {
"role_name": "role-a",
"raw": {"debug": True},
},
"items": [{"name": "kept", "raw": {"debug": True}}],
}
)
assert result == {
"url": "https://example.test/qr",
"before_bind_info": {"role_name": "role-a"},
"items": [{"name": "kept"}],
}
def test_health_endpoint_does_not_use_db_threadpool():
assert inspect.iscoroutinefunction(health)
def test_mysql_lock_wait_timeout_is_bounded():
with app_engine.connect() as connection:
value = connection.execute(
text("SELECT @@SESSION.innodb_lock_wait_timeout")
).scalar()
assert value is not None
assert int(value) <= 10