yml2213
c3fd299f3a
生产是 MySQL 8、测试是 SQLite，两者的方言差异在本地完全测不出来。今天为此 付出了两次代价： 1. `ORDER BY ck_checked_at ASC NULLS FIRST` 在 SQLite 上完全正常，本地 258 条 全绿，一部署到生产第一次巡检就 1064 语法错误（MySQL 不支持 NULLS FIRST）； 2. P1b 自动重登的结算复用调度器事务读取，在 SQLite 下看不出问题，但在 MySQL 的 REPEATABLE READ 下读到的是批次开始前的快照 —— 10 条实际成功的重登全部被记成 失败。SQLite 的读一致性与 MySQL 不同，这类问题跑一万遍也复现不出来。 改动： - 新增 tests/support/db.py：测试专用引擎/会话工具，连接串从 .env 读取， 默认指向独立的 douyu_login_test 库； - 新增 tests/conftest.py：会话级建好表结构，每个用例前后清空所有表做隔离； - 根 conftest.py 接管连接串设置：它在 pytest 收集阶段最先执行，在这里把 DATABASE_URL 指向测试库，应用模块首次 import 拿到就是 MySQL，不依赖任何 import 顺序（原先这里是 sqlite:// 兜底，改造后会变成「先建 SQLite engine 再打补丁换成 MySQL」，顺序一变就静默跑错库）； - 21 个测试文件的 setup 样板统一改用共享测试引擎； - 两个迁移测试改为在临时库 douyu_login_mig_<随机> 上跑完整 alembic upgrade head， 跑完 drop —— 迁移脚本本身也终于在 MySQL 上被验证； - dev.sh 与 README 的过时说明一并更新（原先写着「不需要 MySQL / .env 配置」； 「测试各自建 SQLite 内存库」）。 安全与易用： - 测试库名与开发库同名时**直接拒绝运行**（clear_tables 会 DELETE 所有表， 一旦 TEST_MYSQL_DATABASE 被指到 douyu_login 就是数据灾难）； - 连不上 MySQL 时给出可操作提示（先 ./dev.sh 起库 / 用 TEST_MYSQL_DATABASE 覆盖）， 而不是抛一段 SQLAlchemy 原始堆栈。 验收： - 全量 301 条通过，连跑两次结果一致（隔离有效），耗时约 12.5 秒（原 SQLite 约 3.6 秒）； - 单文件独立运行通过（不依赖全量执行顺序）； - `grep -rn sqlite tests/ conftest.py` 已无实际用法； - **方言确实生效**：临时把一条查询改回 `NULLS FIRST`，13 条测试立刻变红（含运行期 失败），还原即恢复；这条由 test_generated_sql_is_mysql_compatible 长期守着。 代价（已在 README 与规划附录 C.2 记明）：跑测试前必须先有 MySQL，pre-push 钩子 同理；CI 需要 service container。
471 lines
18 KiB
Python
471 lines
18 KiB
Python
"""P1b 回归：CK 自动重登的候选筛选、失败分型与结算。
|
||
|
||
两件事必须一起守住：
|
||
|
||
1. **分型正确**：线上 415 个重登失败里只有 35%（极验）真的需要人工，另外 64%
|
||
（取码失败 + 重启中断）其实可自动重试。分错方向就是白扔可救名额，或者让
|
||
极验账号反复撞风控。
|
||
2. **选行安全**：只挑「巡检判定失效 + 允许自动管理 + 没有在途尝试 + 退避到期 +
|
||
没转人工」的行；转人工的类型绝不能被再次自动重登。
|
||
|
||
结算逻辑（``settle_results``）与选行查询（``build_select_query``）都刻意抽成了
|
||
纯函数/可编译对象，这里不碰真实登录链路。
|
||
"""
|
||
|
||
from datetime import UTC, datetime, timedelta
|
||
|
||
from tests.support.db import get_test_engine, make_test_session
|
||
from web.backend.models import Account, AutomationJob, LoginTask, User
|
||
from web.backend.services.automation import register_default_jobs, reset_registry
|
||
from web.backend.services.automation.jobs import ck_relogin
|
||
|
||
|
||
class _Base:
|
||
def setup_method(self):
|
||
self.engine = get_test_engine()
|
||
self.session = make_test_session(self.engine)
|
||
# 作业注册表是模块级全局，测试之间会互相污染；重置成生产启动时的状态，
|
||
# 否则 list_jobs() 里的 ensure_jobs() 建不出作业行。
|
||
reset_registry()
|
||
register_default_jobs()
|
||
self.admin = User(
|
||
username="admin", password_hash="h", role="super_admin", is_active=True
|
||
)
|
||
self.session.add(self.admin)
|
||
self.session.commit()
|
||
|
||
def teardown_method(self):
|
||
self.session.close()
|
||
|
||
def make_task(
|
||
self,
|
||
name="acc",
|
||
*,
|
||
ck_check_status="invalid",
|
||
cookie="acf_uid=1",
|
||
attempt_status="",
|
||
attempt_message="",
|
||
relogin_attempts=0,
|
||
relogin_next_at=None,
|
||
relogin_failure_kind="",
|
||
relogin_last_error="",
|
||
auto_managed=True,
|
||
):
|
||
account = Account(
|
||
username=name,
|
||
password="p",
|
||
email=f"{name}@e.com",
|
||
email_password="ep",
|
||
auto_managed=auto_managed,
|
||
)
|
||
self.session.add(account)
|
||
self.session.commit()
|
||
task = LoginTask(
|
||
batch_id="b",
|
||
account_id=account.id,
|
||
created_by=self.admin.id,
|
||
status="success",
|
||
cookie=cookie,
|
||
ck_check_status=ck_check_status,
|
||
attempt_status=attempt_status,
|
||
attempt_message=attempt_message,
|
||
relogin_attempts=relogin_attempts,
|
||
relogin_next_at=relogin_next_at,
|
||
relogin_failure_kind=relogin_failure_kind,
|
||
relogin_last_error=relogin_last_error,
|
||
)
|
||
self.session.add(task)
|
||
self.session.commit()
|
||
return task
|
||
|
||
def job(self, batch_size=10):
|
||
return AutomationJob(
|
||
key=ck_relogin.JOB_KEY,
|
||
interval_seconds=6 * 3600,
|
||
batch_size=batch_size,
|
||
concurrency=3,
|
||
)
|
||
|
||
|
||
class TestFailureClassification(_Base):
|
||
def test_quota_wins_over_code(self):
|
||
"""「发送次数已达上限」既含验证码又含上限，必须优先判配额。
|
||
|
||
判成 email_code 会按小时反复撞配额；判成 email_quota 才退避到次日。
|
||
"""
|
||
msg = "重新登录失败: 提交验证码失败: 您的邮箱验证码发送次数已达上限，请改天再试！（旧 Cookie 已保留）"
|
||
assert ck_relogin.classify_failure(msg) == "email_quota"
|
||
|
||
def test_captcha_is_manual(self):
|
||
assert (
|
||
ck_relogin.classify_failure(
|
||
"重新登录失败: 极验验证失败: slide（旧 Cookie 已保留）"
|
||
)
|
||
== "captcha"
|
||
)
|
||
assert "captcha" in ck_relogin.MANUAL_KINDS
|
||
|
||
def test_email_code_is_retryable(self):
|
||
assert (
|
||
ck_relogin.classify_failure(
|
||
"重新登录失败: 提交验证码失败: 邮箱验证码错误，请重新输入"
|
||
)
|
||
== "email_code"
|
||
)
|
||
|
||
def test_interrupted_is_retryable(self):
|
||
assert (
|
||
ck_relogin.classify_failure("登录已中断（服务重启），旧 Cookie 已保留")
|
||
== "interrupted"
|
||
)
|
||
assert (
|
||
ck_relogin.classify_failure("重新登录已停止，旧 Cookie 已保留")
|
||
== "interrupted"
|
||
)
|
||
|
||
def test_credential_is_manual(self):
|
||
assert (
|
||
ck_relogin.classify_failure("重新登录失败: 账号缺少密码或邮箱")
|
||
== "credential"
|
||
)
|
||
assert "credential" in ck_relogin.MANUAL_KINDS
|
||
|
||
def test_unknown_defaults_to_retryable(self):
|
||
"""未知文案按可重试处理：有每日上限兜底，别把可能只是抖动的账号推给人工。"""
|
||
assert ck_relogin.classify_failure("某种没见过的报错") == "network"
|
||
assert ck_relogin.classify_failure("") == "network"
|
||
|
||
|
||
class TestBackoff(_Base):
|
||
def test_ladder_is_escalating(self):
|
||
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
|
||
first = ck_relogin._next_retry_at("email_code", 1, now) - now
|
||
second = ck_relogin._next_retry_at("email_code", 2, now) - now
|
||
third = ck_relogin._next_retry_at("email_code", 3, now) - now
|
||
assert first == timedelta(minutes=60)
|
||
assert second == timedelta(minutes=180)
|
||
assert third == timedelta(minutes=720)
|
||
|
||
def test_quota_goes_to_next_day_with_jitter(self):
|
||
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
|
||
nxt = ck_relogin._next_retry_at("email_quota", 1, now)
|
||
assert nxt.date() == (now + timedelta(days=1)).date()
|
||
assert nxt.hour == 0
|
||
assert 0 <= nxt.minute <= ck_relogin.QUOTA_JITTER_MINUTES
|
||
|
||
|
||
class TestSelection(_Base):
|
||
def _ids(self, job):
|
||
now = datetime.now(UTC)
|
||
return [
|
||
t.id
|
||
for t in ck_relogin.build_select_query(
|
||
self.session, job, now, max_attempts=2
|
||
).all()
|
||
]
|
||
|
||
def test_picks_invalid_with_expired_backoff(self):
|
||
job = self.job()
|
||
ready = self.make_task("ready")
|
||
assert self._ids(job) == [ready.id]
|
||
|
||
def test_skips_valid_and_empty_cookie_and_opted_out(self):
|
||
job = self.job()
|
||
keep = self.make_task("keep")
|
||
self.make_task("valid", ck_check_status="valid")
|
||
self.make_task("nocookie", cookie="")
|
||
self.make_task("optedout", auto_managed=False)
|
||
assert self._ids(job) == [keep.id]
|
||
|
||
def test_skips_inflight_attempt(self):
|
||
job = self.job()
|
||
keep = self.make_task("keep")
|
||
self.make_task("running", attempt_status="running")
|
||
self.make_task("pending", attempt_status="pending")
|
||
assert self._ids(job) == [keep.id]
|
||
|
||
def test_respects_backoff_window(self):
|
||
job = self.job()
|
||
due = self.make_task(
|
||
"due", relogin_next_at=datetime.now(UTC) - timedelta(minutes=1)
|
||
)
|
||
self.make_task("later", relogin_next_at=datetime.now(UTC) + timedelta(hours=3))
|
||
assert self._ids(job) == [due.id]
|
||
|
||
def test_skips_when_attempts_exhausted(self):
|
||
job = self.job()
|
||
keep = self.make_task("keep")
|
||
self.make_task("exhausted", relogin_attempts=2)
|
||
assert self._ids(job) == [keep.id]
|
||
|
||
def test_never_retries_manual_kinds(self):
|
||
"""已判定极验/凭证问题的账号，绝不能再次自动重登。"""
|
||
job = self.job()
|
||
keep = self.make_task("keep")
|
||
self.make_task("captcha", relogin_failure_kind="captcha")
|
||
self.make_task("credential", relogin_failure_kind="credential")
|
||
assert self._ids(job) == [keep.id]
|
||
|
||
def test_retryable_kinds_stay_eligible(self):
|
||
job = self.job()
|
||
code = self.make_task("code", relogin_failure_kind="email_code")
|
||
net = self.make_task("net", relogin_failure_kind="network")
|
||
assert sorted(self._ids(job)) == sorted([code.id, net.id])
|
||
|
||
def test_generated_sql_is_mysql_compatible(self):
|
||
from sqlalchemy.dialects import mysql
|
||
|
||
query = ck_relogin.build_select_query(
|
||
self.session, self.job(), datetime.now(UTC), max_attempts=2
|
||
)
|
||
sql = str(query.statement.compile(dialect=mysql.dialect())).upper()
|
||
assert "NULLS FIRST" not in sql and "NULLS LAST" not in sql
|
||
|
||
def test_batch_size_caps_the_round(self):
|
||
job = self.job(batch_size=3)
|
||
for index in range(6):
|
||
self.make_task(f"a{index}")
|
||
assert len(self._ids(job)) == 3
|
||
|
||
|
||
class TestSettlement(_Base):
|
||
"""结算：成功清零；失败按分型决定重试还是转人工。"""
|
||
|
||
def _settle(self, tasks, *, max_attempts=2):
|
||
now = datetime.now(UTC)
|
||
fresh = {t.id: t for t in tasks}
|
||
counters = ck_relogin.settle_results(
|
||
fresh, [t.id for t in tasks], now=now, max_attempts=max_attempts
|
||
)
|
||
self.session.commit()
|
||
return counters, now
|
||
|
||
def test_success_resets_state(self):
|
||
task = self.make_task(
|
||
"ok",
|
||
attempt_status="success",
|
||
relogin_attempts=1,
|
||
relogin_failure_kind="email_code",
|
||
relogin_last_error="旧的报错",
|
||
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
|
||
)
|
||
counters, _ = self._settle([task])
|
||
self.session.refresh(task)
|
||
assert counters["valid"] == 1 and counters["invalid"] == 0
|
||
assert task.relogin_attempts == 0
|
||
assert task.relogin_next_at is None
|
||
assert task.relogin_failure_kind == ""
|
||
assert task.relogin_last_error == ""
|
||
|
||
def test_retryable_failure_schedules_next_attempt(self):
|
||
task = self.make_task(
|
||
"retry",
|
||
attempt_status="failed",
|
||
attempt_message="等待验证码超时（旧 Cookie 已保留）",
|
||
)
|
||
counters, now = self._settle([task])
|
||
self.session.refresh(task)
|
||
assert counters["invalid"] == 1 and counters["retry"] == 1
|
||
assert task.relogin_failure_kind == "email_code"
|
||
assert task.relogin_attempts == 1
|
||
assert task.relogin_next_at is not None and task.relogin_next_at > now
|
||
assert "旧 Cookie" in task.relogin_last_error
|
||
|
||
def test_captcha_goes_to_manual_without_schedule(self):
|
||
task = self.make_task(
|
||
"cap", attempt_status="failed", attempt_message="极验验证失败: slide"
|
||
)
|
||
counters, _ = self._settle([task])
|
||
self.session.refresh(task)
|
||
assert counters["manual"] == 1 and counters["retry"] == 0
|
||
assert task.relogin_failure_kind == "captcha"
|
||
assert task.relogin_next_at is None # 不再自动排期
|
||
|
||
def test_quota_backs_off_to_next_day(self):
|
||
task = self.make_task(
|
||
"quota",
|
||
attempt_status="failed",
|
||
attempt_message="您的邮箱验证码发送次数已达上限，请改天再试！",
|
||
)
|
||
counters, now = self._settle([task])
|
||
self.session.refresh(task)
|
||
assert counters["email_quota"] == 1 and counters["retry"] == 1
|
||
assert task.relogin_next_at.date() == (now + timedelta(days=1)).date()
|
||
|
||
def test_attempts_exhausted_stops_retrying(self):
|
||
task = self.make_task(
|
||
"used",
|
||
attempt_status="failed",
|
||
attempt_message="等待验证码超时",
|
||
relogin_attempts=1,
|
||
)
|
||
counters, _ = self._settle([task], max_attempts=2)
|
||
self.session.refresh(task)
|
||
assert counters["manual"] == 1 and counters["retry"] == 0
|
||
assert task.relogin_next_at is None
|
||
|
||
def test_settlement_never_touches_cookie(self):
|
||
task = self.make_task(
|
||
"keepck", attempt_status="failed", attempt_message="极验验证失败: slide"
|
||
)
|
||
before = (task.cookie, task.status)
|
||
self._settle([task])
|
||
self.session.refresh(task)
|
||
assert (task.cookie, task.status) == before
|
||
|
||
|
||
class TestManualQueueApi(_Base):
|
||
"""待人工处理队列：只列自动重登救不了的，按原因分组。"""
|
||
|
||
def _queue(self):
|
||
from web.backend.routers import automation as api
|
||
|
||
return api.manual_queue(limit=20, db=self.session, current=self.admin)
|
||
|
||
def test_groups_manual_kinds_only(self):
|
||
captcha = self.make_task(
|
||
"cap",
|
||
attempt_status="failed",
|
||
attempt_message="极验验证失败: slide",
|
||
relogin_failure_kind="captcha",
|
||
relogin_attempts=1,
|
||
relogin_last_error="极验验证失败: slide",
|
||
)
|
||
self.make_task(
|
||
"cred",
|
||
attempt_status="failed",
|
||
attempt_message="账号缺少密码或邮箱",
|
||
relogin_failure_kind="credential",
|
||
relogin_attempts=1,
|
||
relogin_last_error="账号缺少密码或邮箱",
|
||
)
|
||
# 可自动重试的不该出现在人工队列里
|
||
self.make_task(
|
||
"retryable",
|
||
attempt_status="failed",
|
||
attempt_message="等待验证码超时",
|
||
relogin_failure_kind="email_code",
|
||
relogin_attempts=1,
|
||
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
|
||
relogin_last_error="等待验证码超时",
|
||
)
|
||
payload = self._queue()
|
||
kinds = [group["kind"] for group in payload["groups"]]
|
||
assert sorted(kinds) == ["captcha", "credential"]
|
||
assert payload["total"] == 2
|
||
captcha_group = next(g for g in payload["groups"] if g["kind"] == "captcha")
|
||
assert captcha_group["count"] == 1
|
||
assert captcha_group["samples"][0]["task_id"] == captcha.id
|
||
assert "极验" in captcha_group["label"]
|
||
|
||
def test_exhausted_retries_show_up_as_manual(self):
|
||
"""次数用尽的（next_run_at 已清空）也要冒出来，否则会静默卡住。"""
|
||
self.make_task(
|
||
"used",
|
||
attempt_status="failed",
|
||
attempt_message="等待验证码超时",
|
||
relogin_failure_kind="email_code",
|
||
relogin_attempts=2,
|
||
relogin_next_at=None,
|
||
relogin_last_error="等待验证码超时",
|
||
)
|
||
payload = self._queue()
|
||
assert payload["total"] == 1
|
||
assert "重试次数用尽" in payload["groups"][0]["label"]
|
||
|
||
def test_empty_queue(self):
|
||
payload = self._queue()
|
||
assert payload["total"] == 0 and payload["groups"] == []
|
||
|
||
|
||
class TestLinkageWithHealthJob(_Base):
|
||
"""两个作业是上下游：jobs 接口要暴露「当前有多少条等着自动重登」。
|
||
|
||
页面靠这个数字把巡检 → 重登的关系显性化；没有它，两个开关看起来像互不相干。
|
||
"""
|
||
|
||
def test_pending_relogin_counts_only_actionable_invalid(self):
|
||
from web.backend.routers import automation as api
|
||
|
||
ready = self.make_task("ready")
|
||
self.make_task("valid", ck_check_status="valid")
|
||
self.make_task("nocookie", cookie="")
|
||
self.make_task("optedout", auto_managed=False)
|
||
|
||
payload = api.list_jobs(db=self.session, current=self.admin)
|
||
# 只有 ready 同时满足：invalid + 有 CK + 允许自动管理
|
||
assert payload["pending_relogin"] == 1
|
||
assert any(job["key"] == ck_relogin.JOB_KEY for job in payload["jobs"])
|
||
assert ready.id is not None
|
||
|
||
def test_pending_count_is_not_capped_by_batch_size(self):
|
||
"""积压数不能被批次上限截断。
|
||
|
||
一开始复用了作业的候选查询来取 count，而那个查询带 LIMIT batch_size，
|
||
页面上于是恒显示 10（批次上限），完全看不出真实积压 —— 线上就是这样
|
||
显示了「待重登 10」而实际有一千多。
|
||
"""
|
||
from web.backend.routers import automation as api
|
||
|
||
for index in range(25):
|
||
self.make_task(f"acc{index}")
|
||
payload = api.list_jobs(db=self.session, current=self.admin)
|
||
relogin = next(j for j in payload["jobs"] if j["key"] == ck_relogin.JOB_KEY)
|
||
assert relogin["batch_size"] == 10
|
||
assert payload["pending_relogin"] == 25 # 不被 limit 10 截断
|
||
|
||
|
||
class TestSettlementVisibility(_Base):
|
||
"""回归：结算必须用**新事务**读回 runner 提交的结果。
|
||
|
||
MySQL 默认隔离级别是 REPEATABLE READ：在同一个事务里重新查询只会看到事务
|
||
开始时的快照。第一版实现复用调度器传进来的 session 读结果，而那 10 条重登
|
||
**实际全部成功**，结算却按失败写回（加了退避、2 条还被标成极验进人工队列）。
|
||
|
||
SQLite 的读一致性不同，这条在 SQLite 上复现不出来 —— 属于「测试跑在 SQLite
|
||
就漏掉」的典型，现在测试跑在 MySQL 上才守得住。
|
||
"""
|
||
|
||
def test_settlement_sees_other_session_commit(self):
|
||
task = self.make_task("acc", attempt_status="pending")
|
||
|
||
# 调用方会话先开事务（模拟调度器挑完候选后的状态）。
|
||
stale = make_test_session(self.engine)
|
||
assert (
|
||
stale.query(LoginTask).filter(LoginTask.id == task.id).one().attempt_status
|
||
== "pending"
|
||
)
|
||
|
||
# runner 在另一个会话里写入成功并提交。
|
||
runner = make_test_session(self.engine)
|
||
runner.query(LoginTask).filter(
|
||
LoginTask.id == task.id
|
||
).one().attempt_status = "success"
|
||
runner.commit()
|
||
runner.close()
|
||
|
||
# 旧实现在 stale 那个事务里读，只会看到 pending —— 于是把成功写成失败。
|
||
assert (
|
||
stale.query(LoginTask).filter(LoginTask.id == task.id).one().attempt_status
|
||
== "pending"
|
||
), "REPEATABLE READ 下同一事务读不到别人的提交，这正是当初踩的坑"
|
||
|
||
counters = ck_relogin.settle_fresh_results(
|
||
[task.id], now=datetime.now(UTC), max_attempts=2
|
||
)
|
||
self.session.expire_all()
|
||
assert counters["valid"] == 1 and counters["invalid"] == 0
|
||
assert self.session.get(LoginTask, task.id).relogin_failure_kind == ""
|
||
stale.close()
|
||
|
||
def test_settlement_marks_real_success_as_valid(self):
|
||
"""结算看到 success 就必须清零退避，不能写成失败。"""
|
||
task = self.make_task("acc", attempt_status="success", relogin_attempts=1)
|
||
now = datetime.now(UTC)
|
||
counters = ck_relogin.settle_results(
|
||
{task.id: task}, [task.id], now=now, max_attempts=2
|
||
)
|
||
assert counters["valid"] == 1 and counters["invalid"] == 0
|
||
assert task.relogin_failure_kind == ""
|
||
assert task.relogin_next_at is None
|