455 lines
13 KiB
Python
455 lines
13 KiB
Python
import inspect
|
|
import threading
|
|
import time
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
from pydantic import ValidationError
|
|
from sqlalchemy import text
|
|
|
|
from tests.support.db import make_test_session
|
|
from web.backend.database import engine as app_engine
|
|
from web.backend.main import health
|
|
from web.backend.models import Account, DouyuTask, User
|
|
from web.backend.routers import douyu as douyu_router
|
|
from web.backend.schemas import DouyuTaskBatchRequest
|
|
from web.backend.services import (
|
|
binding_poll_scheduler,
|
|
business_events,
|
|
douyu_concurrency,
|
|
douyu_service,
|
|
)
|
|
from web.backend.services.douyu_runner import DouyuBatchRunner
|
|
from web.backend.services.douyu_runner_core import (
|
|
DouyuBatchRunnerCore,
|
|
slim_douyu_task_result_for_storage,
|
|
)
|
|
from web.backend.services.task_runtime import TaskLane, douyu_lane_for_task
|
|
|
|
|
|
def _fixtures(db, *, status: str = "planned") -> tuple[User, Account, DouyuTask]:
|
|
user = User(username="concurrency-user", password_hash="hash", role="operation")
|
|
db.add(user)
|
|
db.flush()
|
|
account = Account(
|
|
username="concurrency-account",
|
|
password="password",
|
|
email="concurrency@example.com",
|
|
email_password="mail-password",
|
|
cookie="acf_uid=1",
|
|
)
|
|
db.add(account)
|
|
db.flush()
|
|
task = DouyuTask(
|
|
batch_id="concurrency-batch",
|
|
account_id=account.id,
|
|
account_assigned_to=user.id,
|
|
task_type="get_bind_qr",
|
|
handbook_scope="elite",
|
|
status=status,
|
|
created_by=user.id,
|
|
)
|
|
db.add(task)
|
|
db.commit()
|
|
return user, account, task
|
|
|
|
|
|
def test_task_rows_commit_before_started_events(db, monkeypatch):
|
|
user = User(username="event-order", password_hash="hash", role="operation")
|
|
db.add(user)
|
|
db.flush()
|
|
account = Account(
|
|
username="event-order-account",
|
|
password="password",
|
|
email="event-order@example.com",
|
|
email_password="mail-password",
|
|
cookie="acf_uid=2",
|
|
)
|
|
db.add(account)
|
|
db.commit()
|
|
observed: list[int] = []
|
|
|
|
def fake_emit_bulk_read_committed(_db, events):
|
|
other = make_test_session()
|
|
try:
|
|
observed.append(
|
|
other.query(DouyuTask).filter(DouyuTask.status == "planned").count()
|
|
)
|
|
finally:
|
|
other.close()
|
|
return len(events)
|
|
|
|
monkeypatch.setattr(douyu_service, "emit_bulk", fake_emit_bulk_read_committed)
|
|
batch_id, count = douyu_service.create_douyu_planned_tasks(
|
|
db,
|
|
[account.id],
|
|
"query_points",
|
|
"elite",
|
|
user.id,
|
|
)
|
|
|
|
assert batch_id
|
|
assert count == 1
|
|
assert observed == [1]
|
|
|
|
|
|
def test_mark_task_commits_result_before_account_event(db, monkeypatch):
|
|
_, account, task = _fixtures(db, status="running")
|
|
task.account = account
|
|
runner = DouyuBatchRunnerCore(
|
|
db,
|
|
task.batch_id,
|
|
task.task_type,
|
|
close_db=False,
|
|
)
|
|
observed: list[tuple[str, object]] = []
|
|
|
|
def fake_record(_db, _account, **kwargs):
|
|
other = make_test_session()
|
|
try:
|
|
row = other.get(DouyuTask, task.id)
|
|
observed.append((row.status, row.result))
|
|
finally:
|
|
other.close()
|
|
return True
|
|
|
|
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
|
|
runner._mark_task(
|
|
db,
|
|
task,
|
|
"success",
|
|
"done",
|
|
{
|
|
"url": "https://example.test/qr",
|
|
"raw": {"large": "payload"},
|
|
"before_bind_info": {
|
|
"role_name": "role-a",
|
|
"raw": {"large": "nested"},
|
|
},
|
|
},
|
|
)
|
|
|
|
assert observed == [
|
|
(
|
|
"success",
|
|
{
|
|
"url": "https://example.test/qr",
|
|
"before_bind_info": {"role_name": "role-a"},
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_cleanup_commits_stopped_status_before_account_event(db, monkeypatch):
|
|
_, _, task = _fixtures(db, status="running")
|
|
observed: list[str] = []
|
|
|
|
def fake_record(_db, _account, **kwargs):
|
|
other = make_test_session()
|
|
try:
|
|
observed.append(other.get(DouyuTask, task.id).status)
|
|
finally:
|
|
other.close()
|
|
return True
|
|
|
|
monkeypatch.setattr(business_events, "record_douyu_task_result", fake_record)
|
|
cleaned = douyu_service.cleanup_orphan_douyu_tasks(
|
|
db,
|
|
active_batch_ids=set(),
|
|
statuses=("running",),
|
|
)
|
|
|
|
assert cleaned == 1
|
|
assert observed == ["stopped"]
|
|
|
|
|
|
def test_cleanup_preserves_recoverable_binding_poll(db):
|
|
_, _, task = _fixtures(db, status="running")
|
|
task.queue_lane = "binding_poll"
|
|
task.next_run_at = datetime.now(UTC) + timedelta(seconds=5)
|
|
task.run_deadline_at = datetime.now(UTC) + timedelta(minutes=10)
|
|
db.commit()
|
|
|
|
cleaned = douyu_service.cleanup_orphan_douyu_tasks(
|
|
db,
|
|
active_batch_ids=set(),
|
|
statuses=("running",),
|
|
)
|
|
|
|
db.refresh(task)
|
|
assert cleaned == 0
|
|
assert task.status == "running"
|
|
assert task.queue_lane == "binding_poll"
|
|
|
|
|
|
def test_binding_poll_scheduler_claims_due_task_without_worker_wait(db):
|
|
_, _, task = _fixtures(db, status="running")
|
|
due_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
task.queue_lane = "binding_poll"
|
|
task.next_run_at = due_at
|
|
task.run_deadline_at = datetime.now(UTC) + timedelta(minutes=10)
|
|
db.commit()
|
|
|
|
try:
|
|
claimed = binding_poll_scheduler._claim_due_binding_tasks(10)
|
|
db.refresh(task)
|
|
assert claimed == [task.id]
|
|
assert task.next_run_at is not None
|
|
assert task.next_run_at > due_at
|
|
finally:
|
|
with binding_poll_scheduler._in_flight_lock:
|
|
binding_poll_scheduler._in_flight.discard(task.id)
|
|
|
|
|
|
def test_bind_qr_schedules_poll_and_releases_worker(db, monkeypatch):
|
|
_, account, task = _fixtures(db, status="running")
|
|
runner = DouyuBatchRunner(
|
|
db,
|
|
task.batch_id,
|
|
"get_bind_qr",
|
|
close_db=False,
|
|
)
|
|
info = {
|
|
"act_alias": "bind-act",
|
|
"role_name": "old-role",
|
|
"is_bound_act": True,
|
|
}
|
|
snapshot = {
|
|
"role_name": "old-role",
|
|
"area_name": "",
|
|
"plat_name": "",
|
|
"game_id": "",
|
|
"game_name": "",
|
|
"game_channel": "",
|
|
"change_role_wait_time": None,
|
|
"can_change_time": None,
|
|
"can_change_role": True,
|
|
"is_bound_act": True,
|
|
"is_bound_role": True,
|
|
"is_bound_account": False,
|
|
"need_bind_act": False,
|
|
"need_bind_role": False,
|
|
}
|
|
client = type(
|
|
"Client",
|
|
(),
|
|
{"get_bind_qr": lambda self, _alias: {"url": "https://example.test/qr"}},
|
|
)()
|
|
monkeypatch.setattr(runner, "_client", lambda _cookie: client)
|
|
monkeypatch.setattr(runner, "_fetch_bind_info_candidates", lambda *_args: [info])
|
|
monkeypatch.setattr(runner, "_pick_baseline_bind_info", lambda *_args: info)
|
|
monkeypatch.setattr(runner, "_pick_change_wait_bind_info", lambda *_args: None)
|
|
monkeypatch.setattr(runner, "_pick_bind_info", lambda *_args, **_kwargs: info)
|
|
monkeypatch.setattr(runner, "_bind_snapshot", lambda _info: snapshot)
|
|
monkeypatch.setattr(
|
|
runner,
|
|
"_wait_bind_role_result",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
AssertionError("scan wait must not block the worker")
|
|
),
|
|
)
|
|
|
|
runner._execute_get_bind_qr(
|
|
db,
|
|
task,
|
|
account,
|
|
"PHPSESSID=x",
|
|
{
|
|
"bind_act_alias": "bind-act",
|
|
"legacy_act_alias": "legacy-act",
|
|
},
|
|
)
|
|
|
|
db.refresh(task)
|
|
assert task.status == "running"
|
|
assert task.queue_lane == "binding_poll"
|
|
assert task.next_run_at is not None
|
|
assert task.run_deadline_at is not None
|
|
assert task.result["url"] == "https://example.test/qr"
|
|
|
|
|
|
def test_binding_poll_once_finishes_when_role_is_found(db, monkeypatch):
|
|
_, account, task = _fixtures(db, status="running")
|
|
task.queue_lane = "binding_poll"
|
|
task.next_run_at = datetime.now(UTC)
|
|
task.run_deadline_at = datetime.now(UTC) + timedelta(minutes=10)
|
|
task.result = {
|
|
"url": "https://example.test/qr",
|
|
"query_act_aliases": ["bind-act"],
|
|
"baseline_role_name": "old-role",
|
|
"bind_polling": True,
|
|
}
|
|
db.commit()
|
|
runner = DouyuBatchRunner(
|
|
db,
|
|
task.batch_id,
|
|
"get_bind_qr",
|
|
close_db=False,
|
|
)
|
|
info = {"act_alias": "bind-act", "role_name": "new-role"}
|
|
snapshot = {
|
|
"role_name": "new-role",
|
|
"area_name": "微信",
|
|
"plat_name": "安卓",
|
|
"game_id": "",
|
|
"game_name": "",
|
|
"game_channel": "",
|
|
"change_role_wait_time": None,
|
|
"can_change_time": None,
|
|
"can_change_role": True,
|
|
"is_bound_act": False,
|
|
"is_bound_role": True,
|
|
"is_bound_account": False,
|
|
"need_bind_act": True,
|
|
"need_bind_role": True,
|
|
}
|
|
client = type("Client", (), {})()
|
|
monkeypatch.setattr(runner, "_client", lambda _cookie: client)
|
|
monkeypatch.setattr(runner, "_fetch_bind_info_candidates", lambda *_args: [info])
|
|
monkeypatch.setattr(runner, "_pick_bind_info", lambda *_args, **_kwargs: info)
|
|
monkeypatch.setattr(runner, "_bind_snapshot", lambda _info: snapshot)
|
|
monkeypatch.setattr(runner, "_is_pending_role", lambda *_args, **_kwargs: True)
|
|
monkeypatch.setattr(
|
|
runner,
|
|
"_format_bind_summary",
|
|
lambda *_args, **_kwargs: "role=new-role",
|
|
)
|
|
|
|
runner.poll_bind_role_once(
|
|
db,
|
|
task,
|
|
account,
|
|
{
|
|
"bind_act_alias": "bind-act",
|
|
"legacy_act_alias": "legacy-act",
|
|
},
|
|
)
|
|
|
|
db.refresh(task)
|
|
assert task.status == "success"
|
|
assert task.queue_lane == ""
|
|
assert task.next_run_at is None
|
|
assert task.result["role_name"] == "new-role"
|
|
assert task.result["bind_ready_for_confirm"] is True
|
|
|
|
|
|
def test_active_task_budget_rejects_new_batch(db, monkeypatch):
|
|
user = User(username="budget-user", password_hash="hash", role="operation")
|
|
db.add(user)
|
|
db.flush()
|
|
account = Account(
|
|
username="budget-account",
|
|
password="password",
|
|
email="budget@example.com",
|
|
email_password="mail-password",
|
|
cookie="acf_uid=3",
|
|
)
|
|
db.add(account)
|
|
db.commit()
|
|
monkeypatch.setattr(douyu_service, "DOUYU_MAX_ACTIVE_TASKS", 1)
|
|
|
|
douyu_service.create_douyu_planned_tasks(
|
|
db,
|
|
[account.id],
|
|
"query_points",
|
|
"elite",
|
|
user.id,
|
|
)
|
|
with pytest.raises(douyu_service.DouyuTaskBackpressureError, match="上限 1"):
|
|
douyu_service.create_douyu_planned_tasks(
|
|
db,
|
|
[account.id],
|
|
"query_game_name",
|
|
"elite",
|
|
user.id,
|
|
)
|
|
|
|
|
|
def test_active_batch_budget_returns_429(monkeypatch):
|
|
monkeypatch.setattr(douyu_router, "DOUYU_MAX_ACTIVE_BATCHES", 1)
|
|
|
|
with pytest.raises(HTTPException) as exc_info:
|
|
douyu_router._ensure_active_batch_budget(1)
|
|
|
|
assert exc_info.value.status_code == 429
|
|
|
|
|
|
def test_douyu_batch_request_rejects_more_than_five_workers():
|
|
with pytest.raises(ValidationError):
|
|
DouyuTaskBatchRequest(
|
|
account_ids=[1],
|
|
task_type="query_points",
|
|
handbook_scope="elite",
|
|
concurrency=6,
|
|
)
|
|
|
|
|
|
def test_batch_task_concurrency_is_capped_at_five():
|
|
assert douyu_concurrency.clamp_batch_task_concurrency(20) == 5
|
|
|
|
|
|
def test_douyu_lanes_separate_interactive_query_payment_and_writes():
|
|
assert douyu_lane_for_task("get_bind_qr") == TaskLane.INTERACTIVE_BINDING
|
|
assert douyu_lane_for_task("query_account_overview") == TaskLane.QUERY_BATCH
|
|
assert douyu_lane_for_task("create_gold_qr") == TaskLane.PAYMENT_WAIT
|
|
assert douyu_lane_for_task("exchange_goods") == TaskLane.OPS_WRITE
|
|
|
|
|
|
def test_overview_limiter_blocks_extra_workers(monkeypatch):
|
|
slots = threading.BoundedSemaphore(1)
|
|
monkeypatch.setattr(douyu_concurrency, "_overview_slots", slots)
|
|
acquired: list[bool] = []
|
|
started = threading.Event()
|
|
|
|
assert douyu_concurrency.acquire_overview_slot(lambda: False) is True
|
|
|
|
def worker():
|
|
started.set()
|
|
acquired.append(douyu_concurrency.acquire_overview_slot(lambda: False))
|
|
|
|
thread = threading.Thread(target=worker)
|
|
thread.start()
|
|
started.wait(timeout=1)
|
|
time.sleep(0.05)
|
|
assert acquired == []
|
|
|
|
douyu_concurrency.release_overview_slot()
|
|
thread.join(timeout=1)
|
|
assert acquired == [True]
|
|
douyu_concurrency.release_overview_slot()
|
|
|
|
|
|
def test_slim_task_result_removes_debug_payloads():
|
|
result = slim_douyu_task_result_for_storage(
|
|
{
|
|
"url": "https://example.test/qr",
|
|
"raw": {"debug": True},
|
|
"query_params": {"token": "secret"},
|
|
"before_bind_info": {
|
|
"role_name": "role-a",
|
|
"raw": {"debug": True},
|
|
},
|
|
"items": [{"name": "kept", "raw": {"debug": True}}],
|
|
}
|
|
)
|
|
|
|
assert result == {
|
|
"url": "https://example.test/qr",
|
|
"before_bind_info": {"role_name": "role-a"},
|
|
"items": [{"name": "kept"}],
|
|
}
|
|
|
|
|
|
def test_health_endpoint_does_not_use_db_threadpool():
|
|
assert inspect.iscoroutinefunction(health)
|
|
|
|
|
|
def test_mysql_lock_wait_timeout_is_bounded():
|
|
with app_engine.connect() as connection:
|
|
value = connection.execute(
|
|
text("SELECT @@SESSION.innodb_lock_wait_timeout")
|
|
).scalar()
|
|
assert value is not None
|
|
assert int(value) <= 10
|