Files
live-hub-py/tests/test_automation_ck_health.py
T
yml2213 c3fd299f3a test(db): 测试改用真实 MySQL,方言差异不再漏到生产
生产是 MySQL 8、测试是 SQLite,两者的方言差异在本地完全测不出来。今天为此
付出了两次代价:

1. `ORDER BY ck_checked_at ASC NULLS FIRST` 在 SQLite 上完全正常,本地 258 条
   全绿,一部署到生产第一次巡检就 1064 语法错误(MySQL 不支持 NULLS FIRST);
2. P1b 自动重登的结算复用调度器事务读取,在 SQLite 下看不出问题,但在 MySQL 的
   REPEATABLE READ 下读到的是批次开始前的快照 —— 10 条实际成功的重登全部被记成
   失败。SQLite 的读一致性与 MySQL 不同,这类问题跑一万遍也复现不出来。

改动:
- 新增 tests/support/db.py:测试专用引擎/会话工具,连接串从 .env 读取,
  默认指向独立的 douyu_login_test 库;
- 新增 tests/conftest.py:会话级建好表结构,每个用例前后清空所有表做隔离;
- 根 conftest.py 接管连接串设置:它在 pytest 收集阶段最先执行,在这里把
  DATABASE_URL 指向测试库,应用模块首次 import 拿到就是 MySQL,不依赖任何
  import 顺序(原先这里是 sqlite:// 兜底,改造后会变成「先建 SQLite engine
  再打补丁换成 MySQL」,顺序一变就静默跑错库);
- 21 个测试文件的 setup 样板统一改用共享测试引擎;
- 两个迁移测试改为在临时库 douyu_login_mig_<随机> 上跑完整 alembic upgrade head,
  跑完 drop —— 迁移脚本本身也终于在 MySQL 上被验证;
- dev.sh 与 README 的过时说明一并更新(原先写着「不需要 MySQL / .env 配置」;
  「测试各自建 SQLite 内存库」)。

安全与易用:
- 测试库名与开发库同名时**直接拒绝运行**(clear_tables 会 DELETE 所有表,
  一旦 TEST_MYSQL_DATABASE 被指到 douyu_login 就是数据灾难);
- 连不上 MySQL 时给出可操作提示(先 ./dev.sh 起库 / 用 TEST_MYSQL_DATABASE 覆盖),
  而不是抛一段 SQLAlchemy 原始堆栈。

验收:
- 全量 301 条通过,连跑两次结果一致(隔离有效),耗时约 12.5 秒(原 SQLite 约 3.6 秒);
- 单文件独立运行通过(不依赖全量执行顺序);
- `grep -rn sqlite tests/ conftest.py` 已无实际用法;
- **方言确实生效**:临时把一条查询改回 `NULLS FIRST`,13 条测试立刻变红(含运行期
  失败),还原即恢复;这条由 test_generated_sql_is_mysql_compatible 长期守着。

代价(已在 README 与规划附录 C.2 记明):跑测试前必须先有 MySQL,pre-push 钩子
同理;CI 需要 service container。
2026-10-02 21:22:08 +08:00

585 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""P1a 回归:CK 自动巡检。
两块约束必须一起守住:
1. **调度器行为**:只跑到点的、打开开关的作业;作业抛异常只记 failed,不能
把调度线程带崩;跑完排下一轮时间;已有配置不能被重启重置。
2. **本作业绝不改 CK**:P1a 的定位是「只检测、只标记」,所以巡检后
``cookie`` / ``status`` / ``message`` / ``attempt_*`` 必须一字未动 ——
线上全靠这条保证「自动化上线了但人工流程照旧」,回归时一旦有人顺手写了
``status``,这里要立刻红。
"""
from datetime import UTC, datetime, timedelta
import pytest
from fastapi import HTTPException
from tests.support.db import get_test_engine, make_test_session
from web.backend.deps import require_permission
from web.backend.models import Account, AutomationJob, AutomationRun, LoginTask, User
from web.backend.services.automation import (
JobSpec,
register_default_jobs,
register_job,
reset_registry,
run_due_jobs,
run_job_now,
)
from web.backend.services.automation.jobs import ck_health
class _Base:
def setup_method(self):
self.engine = get_test_engine()
self.session = make_test_session(self.engine)
self.admin = User(
username="admin", password_hash="h", role="super_admin", is_active=True
)
self.support = User(
username="kf", password_hash="h", role="support", is_active=True
)
self.session.add_all([self.admin, self.support])
self.session.commit()
# 注册表是模块级的,不重置会被上一个用例注册的探针作业污染。
reset_registry()
register_default_jobs()
def teardown_method(self):
self.session.close()
def make_account(self, name="acc", *, auto_managed=True, assigned_to=None):
account = Account(
username=name,
password="p",
email=f"{name}@example.com",
email_password="ep",
assigned_to=assigned_to,
auto_managed=auto_managed,
)
self.session.add(account)
self.session.commit()
return account
def make_task(
self,
account,
*,
cookie="acf_uid=1",
status="success",
attempt_status="",
ck_checked_at=None,
ck_check_status="",
):
task = LoginTask(
batch_id="b1",
account_id=account.id,
created_by=self.admin.id,
status=status,
cookie=cookie,
attempt_status=attempt_status,
ck_checked_at=ck_checked_at,
ck_check_status=ck_check_status,
)
self.session.add(task)
self.session.commit()
return task
class TestCkHealthSelection(_Base):
def _selected_ids(self, job):
return [
t.id for t in ck_health._select_tasks(self.session, job, datetime.now(UTC))
]
def test_picks_only_rows_that_still_hold_cookie(self):
job = self._job()
with_cookie = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b"), cookie="")
assert self._selected_ids(job) == [with_cookie.id]
def test_skips_accounts_with_inflight_attempt(self):
job = self._job()
idle = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b"), attempt_status="running")
self.make_task(self.make_account("c"), attempt_status="pending")
assert self._selected_ids(job) == [idle.id]
def test_skips_accounts_opted_out_of_automation(self):
job = self._job()
managed = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b", auto_managed=False))
assert self._selected_ids(job) == [managed.id]
def test_respects_interval_and_prefers_stalest(self):
job = self._job()
fresh = self.make_task(
self.make_account("fresh"), ck_checked_at=datetime.now(UTC)
)
stale = self.make_task(
self.make_account("stale"),
ck_checked_at=datetime.now(UTC) - timedelta(hours=30),
)
never = self.make_task(self.make_account("never"), ck_checked_at=None)
ids = self._selected_ids(job)
assert fresh.id not in ids
# 从没检测过的排最前,其次是久未检测的
assert ids[0] == never.id
assert stale.id in ids
def test_generated_sql_is_mysql_compatible(self):
"""回归:选行 SQL 不能出现 MySQL 不认识的 NULLS FIRST。
生产是 MySQL、本地测试是 SQLite(SQLite 支持 NULLS FIRST,所以行为测试
发现不了),这里按 MySQL 方言编译一次断言。
"""
from sqlalchemy.dialects import mysql
query = ck_health.build_select_query(
self.session, self._job(), datetime.now(UTC)
)
sql = str(query.statement.compile(dialect=mysql.dialect())).upper()
assert "NULLS FIRST" not in sql and "NULLS LAST" not in sql
# 「没检测过的优先」要用可移植写法表达
assert "IS NULL DESC" in sql
def test_batch_size_caps_the_round(self):
job = self._job()
job.batch_size = 2
for index in range(5):
self.make_task(self.make_account(f"a{index}"))
assert len(self._selected_ids(job)) == 2
def _job(self):
return AutomationJob(
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
)
class TestCkHealthNeverTouchesCookie(_Base):
def test_patrol_only_writes_check_fields(self):
job = self._job()
account = self.make_account("acc")
task = self.make_task(account, attempt_status="failed", ck_check_status="")
before = (task.cookie, task.status, task.message, task.attempt_status)
ck_health.check_douyu_cookie = _fake_check(valid=True)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
self.session.refresh(task)
assert counters["scanned"] == 1 and counters["valid"] == 1
assert task.ck_check_status == "valid"
assert task.ck_checked_at is not None
assert task.ck_check_result["nickname"] == "测试"
# 核心断言:CK 本体与尝试状态一字未动
assert (task.cookie, task.status, task.message, task.attempt_status) == before
def test_invalid_result_marks_invalid_but_keeps_cookie(self):
job = self._job()
task = self.make_task(self.make_account("acc"))
# 必须是鉴权类失败(斗鱼说未登录)才会被标成 invalid
ck_health.check_douyu_cookie = _fake_check(
valid=False,
message="鱼丸接口: 账户登陆异常,请重新登陆;等级接口: 用户未登录",
)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
self.session.refresh(task)
assert counters["invalid"] == 1
assert task.ck_check_status == "invalid"
assert task.cookie == "acf_uid=1"
def test_detection_exception_does_not_abort_round(self):
"""单个账号检测抛异常不影响整轮,但也不能把它标成失效。"""
job = self._job()
first = self.make_task(self.make_account("a"), ck_check_status="valid")
second = self.make_task(self.make_account("b"), ck_check_status="valid")
ck_health.check_douyu_cookie = _fake_check(raises=True)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert counters["scanned"] == 2
# 查不到不等于失效:两行都保留原结论,只记录这次异常
assert counters["invalid"] == 0
assert counters["skipped"] == 2
for task in (first, second):
self.session.refresh(task)
assert task.ck_check_status == "valid"
assert task.ck_check_result["verdict"] == "error"
def _job(self):
return AutomationJob(
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
)
class TestScheduler(_Base):
def _register_probe(self, calls):
def run(db, job):
calls.append(job.key)
return {"scanned": 3, "valid": 2, "invalid": 1, "message": "ok"}
register_job(
JobSpec(
key="probe",
title="探针",
interval_seconds=3600,
batch_size=1,
concurrency=1,
run=run,
)
)
def test_only_enabled_jobs_run(self):
from web.backend.services.automation import ensure_jobs
calls: list[str] = []
self._register_probe(calls)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
{"enabled": True}
)
self.session.commit()
run_due_jobs(self.session)
assert calls == ["probe"]
calls.clear()
run_due_jobs(self.session)
# 上一轮已排了 next_run_at,未到点不再跑
assert calls == []
def test_job_failure_is_recorded_not_raised(self):
from web.backend.services.automation import ensure_jobs
def boom(db, job):
raise RuntimeError("接口挂了")
register_job(
JobSpec(
key="boom",
title="会炸的作业",
interval_seconds=3600,
batch_size=1,
concurrency=1,
run=boom,
)
)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "boom").update(
{"enabled": True}
)
self.session.commit()
outcomes = run_due_jobs(self.session)
assert outcomes[0]["status"] == "failed"
run = self.session.query(AutomationRun).one()
assert run.status == "failed" and "接口挂了" in run.message
def test_next_run_follows_tick_and_counters_persist(self):
from web.backend.services.automation import ensure_jobs
calls: list[str] = []
self._register_probe(calls)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
{"enabled": True}
)
self.session.commit()
run_due_jobs(self.session)
job = (
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").one()
)
assert job.last_status == "success"
# 下一轮在一个 tick 之后(作业每个 tick 都跑),与 interval_seconds 无关
from web.backend.services.automation.scheduler import TICK_SECONDS
assert _aware(job.next_run_at) - _aware(job.last_run_at) == timedelta(
seconds=TICK_SECONDS
)
run = self.session.query(AutomationRun).one()
assert (run.scanned, run.valid, run.invalid) == (3, 2, 1)
def test_ensure_jobs_keeps_operator_settings(self):
from web.backend.services.automation import ensure_jobs
ensure_jobs(self.session)
job = (
self.session.query(AutomationJob)
.filter(AutomationJob.key == ck_health.JOB_KEY)
.one()
)
job.enabled = True
job.batch_size = 7
self.session.commit()
ensure_jobs(self.session) # 模拟重启
self.session.refresh(job)
assert job.enabled is True and job.batch_size == 7
def test_run_job_now_ignores_switch(self):
from web.backend.services.automation import ensure_jobs
ensure_jobs(self.session)
outcome = run_job_now(self.session, ck_health.JOB_KEY)
assert outcome["job_key"] == ck_health.JOB_KEY
assert self.session.query(AutomationRun).count() == 1
def test_run_job_now_unknown_key(self):
with pytest.raises(KeyError):
run_job_now(self.session, "nope")
class TestAutomationPermissions(_Base):
def test_view_allows_operation_denies_support(self):
from web.backend.permissions import ROLE_PERMISSIONS
assert "automation:view" in ROLE_PERMISSIONS["operation"]
assert "automation:view" not in ROLE_PERMISSIONS["support"]
assert "automation:manage" in ROLE_PERMISSIONS["super_admin"]
assert "automation:manage" not in ROLE_PERMISSIONS["operation"]
def test_manage_dependency_rejects_operator(self):
operator = User(
username="op", password_hash="h", role="operation", is_active=True
)
checker = require_permission("automation:manage")
with pytest.raises(HTTPException) as exc:
checker(current_user=operator)
assert exc.value.status_code == 403
def test_view_dependency_accepts_operator(self):
operator = User(
username="op", password_hash="h", role="operation", is_active=True
)
checker = require_permission("automation:view")
assert checker(current_user=operator).id == operator.id
def _aware(value):
"""库里读出的是 naive UTC,比较前补时区。"""
return value if value.tzinfo else value.replace(tzinfo=UTC)
_REAL_CHECK = ck_health.check_douyu_cookie
def _fake_check(*, valid=True, raises=False, message=None):
def fake(cookie: str) -> dict:
if raises:
raise RuntimeError("网络异常")
return {
"valid": valid,
"message": message
if message is not None
else ("ok" if valid else "Cookie 已失效"),
"fish_ball": 1,
"nickname": "测试",
"level": 30,
"checked_at": datetime.now(UTC),
}
return fake
class TestAutomationApi(_Base):
"""接口层:序列化字段、开关落库、手动触发不受开关限制。"""
def _api(self):
from web.backend.routers import automation as api
return api
def test_list_jobs_exposes_scheduler_and_job_state(self):
api = self._api()
payload = api.list_jobs(db=self.session, current=self.admin)
assert payload["scheduler"]["tick_seconds"] == 300
assert payload["scheduler"]["running"] is False
keys = [item["key"] for item in payload["jobs"]]
assert ck_health.JOB_KEY in keys
job = next(item for item in payload["jobs"] if item["key"] == ck_health.JOB_KEY)
assert job["enabled"] is False
assert job["interval_seconds"] == 6 * 3600
assert job["batch_size"] == 200
assert job["latest_run"] is None
def test_update_job_persists_and_schedules_next_run(self):
api = self._api()
api.update_job(
ck_health.JOB_KEY,
api.JobUpdate(
enabled=True, interval_seconds=3600, batch_size=5, concurrency=2
),
db=self.session,
current=self.admin,
)
job = (
self.session.query(AutomationJob)
.filter(AutomationJob.key == ck_health.JOB_KEY)
.one()
)
assert job.enabled is True
assert job.interval_seconds == 3600
assert job.batch_size == 5
assert job.next_run_at is not None # 打开开关就排了一轮
# 审计留痕
from web.backend.models import AuditLog
entry = (
self.session.query(AuditLog)
.filter(AuditLog.action == "automation:update")
.one()
)
assert entry.target == ck_health.JOB_KEY
def test_run_endpoint_records_run_and_audit(self):
api = self._api()
self.make_task(self.make_account("a"))
ck_health.check_douyu_cookie = _fake_check(valid=True)
try:
outcome = api.run_job(
ck_health.JOB_KEY, db=self.session, current=self.admin
)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert outcome["status"] == "success"
assert outcome["scanned"] == 1
assert self.session.query(AutomationRun).count() == 1
from web.backend.models import AuditLog
assert (
self.session.query(AuditLog)
.filter(AuditLog.action == "automation:run")
.count()
== 1
)
def test_run_endpoint_unknown_job_returns_404(self):
api = self._api()
with pytest.raises(HTTPException) as exc:
api.run_job("nope", db=self.session, current=self.admin)
assert exc.value.status_code == 404
def test_runs_endpoint_filters_by_job_key(self):
api = self._api()
self.session.add(
AutomationRun(job_key=ck_health.JOB_KEY, status="success", scanned=1)
)
self.session.add(AutomationRun(job_key="other", status="failed"))
self.session.commit()
payload = api.list_runs(
job_key=ck_health.JOB_KEY, limit=50, db=self.session, current=self.admin
)
assert len(payload["items"]) == 1
assert payload["items"][0]["job_key"] == ck_health.JOB_KEY
class TestFailureClassification(_Base):
"""「查不到」不等于「失效」:非鉴权类失败不能覆盖原来的结论。
斗鱼接口抖动/被风控时,如果不加区分地写 invalid,一小时就能把上千个好账号
标成失效,客服会误判成 CK 大面积挂掉。这里盯三件事:分类正确、error 不改
结论、失败占比过高时自动暂停作业。
"""
def test_auth_failure_message_counts_as_invalid(self):
assert (
ck_health.classify_result(
{
"valid": False,
"message": "鱼丸接口: 账户登陆异常,请重新登陆;等级接口: 用户未登录",
}
)
== "invalid"
)
def test_network_failure_counts_as_error_not_invalid(self):
assert (
ck_health.classify_result(
{
"valid": False,
"message": "鱼丸接口: 请求失败: timeout;等级接口: 请求失败: timeout",
}
)
== "error"
)
assert (
ck_health.classify_result({"valid": False, "message": "响应异常"})
== "error"
)
def test_valid_still_valid(self):
assert ck_health.classify_result({"valid": True, "message": "有效"}) == "valid"
def test_error_keeps_previous_verdict_but_marks_checked(self):
job = self._job()
task = self.make_task(self.make_account("acc"), ck_check_status="valid")
ck_health.check_douyu_cookie = _fake_check(
valid=False, message="请求失败: timeout"
)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
self.session.refresh(task)
assert counters["valid"] == 0 and counters["invalid"] == 0
assert counters["skipped"] == 1 # error 计入 skipped
assert task.ck_check_status == "valid" # 原结论保留
assert task.ck_check_result["verdict"] == "error" # 但记录了这次查不到
assert task.ck_checked_at is not None
def test_mass_network_failure_pauses_the_job(self):
job = self._job()
job.enabled = True
job.batch_size = 40
for index in range(40):
self.make_task(self.make_account(f"a{index}"))
ck_health.check_douyu_cookie = _fake_check(
valid=False, message="请求失败: Connection refused"
)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert counters["skipped"] == 40
assert counters["detail"]["paused"] is True
assert job.enabled is False # 自动暂停,不再继续污染
assert "已自动暂停" in counters["message"]
def test_normal_invalid_does_not_pause(self):
"""真的都是死号时不能暂停 —— 首轮 100% invalid 是合法且预期的情况。"""
job = self._job()
job.enabled = True
job.batch_size = 40
for index in range(40):
self.make_task(self.make_account(f"a{index}"))
ck_health.check_douyu_cookie = _fake_check(
valid=False,
message="鱼丸接口: 账户登陆异常,请重新登陆;等级接口: 用户未登录",
)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert counters["invalid"] == 40
assert counters["skipped"] == 0
assert job.enabled is True
assert counters["detail"]["paused"] is False
def _job(self):
return AutomationJob(
key=ck_health.JOB_KEY,
interval_seconds=6 * 3600,
batch_size=200,
concurrency=3,
)