perf(douyu): 消除任务锁竞争并增加创建背压

This commit is contained in:
yml
2026-11-19 22:01:21 +08:00
parent 62cb0a41ec
commit 09cdf3f2eb
11 changed files with 555 additions and 95 deletions
+227
View File
@@ -0,0 +1,227 @@
import inspect
import pytest
from fastapi import HTTPException
from sqlalchemy import text
from tests.support.db import make_test_session
from web.backend.database import engine as app_engine
from web.backend.main import health
from web.backend.models import Account, DouyuTask, User
from web.backend.routers import douyu as douyu_router
from web.backend.services import business_events, douyu_service
from web.backend.services.douyu_runner_core import (
DouyuBatchRunnerCore,
slim_douyu_task_result_for_storage,
)
def _fixtures(db, *, status: str = "planned") -> tuple[User, Account, DouyuTask]:
user = User(username="concurrency-user", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="concurrency-account",
password="password",
email="concurrency@example.com",
email_password="mail-password",
cookie="acf_uid=1",
)
db.add(account)
db.flush()
task = DouyuTask(
batch_id="concurrency-batch",
account_id=account.id,
account_assigned_to=user.id,
task_type="get_bind_qr",
handbook_scope="elite",
status=status,
created_by=user.id,
)
db.add(task)
db.commit()
return user, account, task
def test_task_rows_commit_before_started_events(db, monkeypatch):
user = User(username="event-order", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="event-order-account",
password="password",
email="event-order@example.com",
email_password="mail-password",
cookie="acf_uid=2",
)
db.add(account)
db.commit()
observed: list[int] = []
def fake_emit_bulk_read_committed(_db, events):
other = make_test_session()
try:
observed.append(
other.query(DouyuTask).filter(DouyuTask.status == "planned").count()
)
finally:
other.close()
return len(events)
monkeypatch.setattr(douyu_service, "emit_bulk", fake_emit_bulk_read_committed)
batch_id, count = douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_points",
"elite",
user.id,
)
assert batch_id
assert count == 1
assert observed == [1]
def test_mark_task_commits_result_before_account_event(db, monkeypatch):
_, account, task = _fixtures(db, status="running")
task.account = account
runner = DouyuBatchRunnerCore(
db,
task.batch_id,
task.task_type,
close_db=False,
)
observed: list[tuple[str, object]] = []
def fake_record(_db, _account, **kwargs):
other = make_test_session()
try:
row = other.get(DouyuTask, task.id)
observed.append((row.status, row.result))
finally:
other.close()
return True
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
runner._mark_task(
db,
task,
"success",
"done",
{
"url": "https://example.test/qr",
"raw": {"large": "payload"},
"before_bind_info": {
"role_name": "role-a",
"raw": {"large": "nested"},
},
},
)
assert observed == [
(
"success",
{
"url": "https://example.test/qr",
"before_bind_info": {"role_name": "role-a"},
},
)
]
def test_cleanup_commits_stopped_status_before_account_event(db, monkeypatch):
_, _, task = _fixtures(db, status="running")
observed: list[str] = []
def fake_record(_db, _account, **kwargs):
other = make_test_session()
try:
observed.append(other.get(DouyuTask, task.id).status)
finally:
other.close()
return True
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
cleaned = douyu_service.cleanup_orphan_douyu_tasks(
db,
active_batch_ids=set(),
statuses=("running",),
)
assert cleaned == 1
assert observed == ["stopped"]
def test_active_task_budget_rejects_new_batch(db, monkeypatch):
user = User(username="budget-user", password_hash="hash", role="operation")
db.add(user)
db.flush()
account = Account(
username="budget-account",
password="password",
email="budget@example.com",
email_password="mail-password",
cookie="acf_uid=3",
)
db.add(account)
db.commit()
monkeypatch.setattr(douyu_service, "DOUYU_MAX_ACTIVE_TASKS", 1)
douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_points",
"elite",
user.id,
)
with pytest.raises(douyu_service.DouyuTaskBackpressureError, match="上限 1"):
douyu_service.create_douyu_planned_tasks(
db,
[account.id],
"query_game_name",
"elite",
user.id,
)
def test_active_batch_budget_returns_429(monkeypatch):
monkeypatch.setattr(douyu_router, "DOUYU_MAX_ACTIVE_BATCHES", 1)
with pytest.raises(HTTPException) as exc_info:
douyu_router._ensure_active_batch_budget(1)
assert exc_info.value.status_code == 429
def test_slim_task_result_removes_debug_payloads():
result = slim_douyu_task_result_for_storage(
{
"url": "https://example.test/qr",
"raw": {"debug": True},
"query_params": {"token": "secret"},
"before_bind_info": {
"role_name": "role-a",
"raw": {"debug": True},
},
"items": [{"name": "kept", "raw": {"debug": True}}],
}
)
assert result == {
"url": "https://example.test/qr",
"before_bind_info": {"role_name": "role-a"},
"items": [{"name": "kept"}],
}
def test_health_endpoint_does_not_use_db_threadpool():
assert inspect.iscoroutinefunction(health)
def test_mysql_lock_wait_timeout_is_bounded():
with app_engine.connect() as connection:
value = connection.execute(
text("SELECT @@SESSION.innodb_lock_wait_timeout")
).scalar()
assert value is not None
assert int(value) <= 10
+5 -1
View File
@@ -142,7 +142,11 @@ class TestLoginFailureEvents(_Base):
self.session.commit()
_mark_attempt_failed(task, "极验验证失败: slide")
self.session.commit()
assert len(self.events(account.id)) == 1
events = self.events(account.id)
assert len(events) == 1, [
(event.source_ref, event.occurred_at) for event in events
]
assert events[0].occurred_at.microsecond == 0
def test_event_does_not_break_when_task_has_no_session(self):
"""纯内存构造的任务(没有 session)不能因为写事件而炸掉主流程。"""