yml2213
2c532df14f
页面上「待重登候选」显示 10 —— 那只是 ck_relogin 的 batch_size，不是真实积压。
根因是我用作业的候选查询取 count，而那个查询带 LIMIT batch_size，count 自然
被截断在批次上限；线上实际有一千多条待重登，页面却恒显示 10。
改为独立的 count_pending_relogin()：同一套资格条件（invalid + 有 CK + 允许自动
管理 + 未转人工 + 未超每日上限），但不带 LIMIT，也不按退避窗口过滤 —— 退避是
「什么时候试」的细节，不影响「还有多少要处理」。
新增测试：批次上限为 10 时，25 条积压必须报 25。
另：本次一并还原了 ck_health.py 里被外部改动引入的回归 —— 有人把选行排序改成
text("login_tasks.ck_checked_at ASC NULLS FIRST")，那是逐字透传给数据库的片段，
而 MySQL 不支持 NULLS FIRST（今天线上就是这么炸的 1064）。已恢复成可移植写法，
test_generated_sql_is_mysql_compatible 会拦住这类改动（本次正是它先报的红）。
全量 298 通过。
416 lines
16 KiB
Python
416 lines
16 KiB
Python
"""P1b 回归：CK 自动重登的候选筛选、失败分型与结算。
|
||
|
||
两件事必须一起守住：
|
||
|
||
1. **分型正确**：线上 415 个重登失败里只有 35%（极验）真的需要人工，另外 64%
|
||
（取码失败 + 重启中断）其实可自动重试。分错方向就是白扔可救名额，或者让
|
||
极验账号反复撞风控。
|
||
2. **选行安全**：只挑「巡检判定失效 + 允许自动管理 + 没有在途尝试 + 退避到期 +
|
||
没转人工」的行；转人工的类型绝不能被再次自动重登。
|
||
|
||
结算逻辑（``settle_results``）与选行查询（``build_select_query``）都刻意抽成了
|
||
纯函数/可编译对象，这里不碰真实登录链路。
|
||
"""
|
||
|
||
from datetime import UTC, datetime, timedelta
|
||
|
||
from tests.support.db import get_test_engine, make_test_session
|
||
from web.backend.models import Account, AutomationJob, LoginTask, User
|
||
from web.backend.services.automation import register_default_jobs, reset_registry
|
||
from web.backend.services.automation.jobs import ck_relogin
|
||
|
||
|
||
class _Base:
|
||
def setup_method(self):
|
||
self.engine = get_test_engine()
|
||
self.session = make_test_session(self.engine)
|
||
# 作业注册表是模块级全局，测试之间会互相污染；重置成生产启动时的状态，
|
||
# 否则 list_jobs() 里的 ensure_jobs() 建不出作业行。
|
||
reset_registry()
|
||
register_default_jobs()
|
||
self.admin = User(
|
||
username="admin", password_hash="h", role="super_admin", is_active=True
|
||
)
|
||
self.session.add(self.admin)
|
||
self.session.commit()
|
||
|
||
def teardown_method(self):
|
||
self.session.close()
|
||
|
||
def make_task(
|
||
self,
|
||
name="acc",
|
||
*,
|
||
ck_check_status="invalid",
|
||
cookie="acf_uid=1",
|
||
attempt_status="",
|
||
attempt_message="",
|
||
relogin_attempts=0,
|
||
relogin_next_at=None,
|
||
relogin_failure_kind="",
|
||
relogin_last_error="",
|
||
auto_managed=True,
|
||
):
|
||
account = Account(
|
||
username=name,
|
||
password="p",
|
||
email=f"{name}@e.com",
|
||
email_password="ep",
|
||
auto_managed=auto_managed,
|
||
)
|
||
self.session.add(account)
|
||
self.session.commit()
|
||
task = LoginTask(
|
||
batch_id="b",
|
||
account_id=account.id,
|
||
created_by=self.admin.id,
|
||
status="success",
|
||
cookie=cookie,
|
||
ck_check_status=ck_check_status,
|
||
attempt_status=attempt_status,
|
||
attempt_message=attempt_message,
|
||
relogin_attempts=relogin_attempts,
|
||
relogin_next_at=relogin_next_at,
|
||
relogin_failure_kind=relogin_failure_kind,
|
||
relogin_last_error=relogin_last_error,
|
||
)
|
||
self.session.add(task)
|
||
self.session.commit()
|
||
return task
|
||
|
||
def job(self, batch_size=10):
|
||
return AutomationJob(
|
||
key=ck_relogin.JOB_KEY,
|
||
interval_seconds=6 * 3600,
|
||
batch_size=batch_size,
|
||
concurrency=3,
|
||
)
|
||
|
||
|
||
class TestFailureClassification(_Base):
|
||
def test_quota_wins_over_code(self):
|
||
"""「发送次数已达上限」既含验证码又含上限，必须优先判配额。
|
||
|
||
判成 email_code 会按小时反复撞配额；判成 email_quota 才退避到次日。
|
||
"""
|
||
msg = "重新登录失败: 提交验证码失败: 您的邮箱验证码发送次数已达上限，请改天再试！（旧 Cookie 已保留）"
|
||
assert ck_relogin.classify_failure(msg) == "email_quota"
|
||
|
||
def test_captcha_is_manual(self):
|
||
assert (
|
||
ck_relogin.classify_failure(
|
||
"重新登录失败: 极验验证失败: slide（旧 Cookie 已保留）"
|
||
)
|
||
== "captcha"
|
||
)
|
||
assert "captcha" in ck_relogin.MANUAL_KINDS
|
||
|
||
def test_email_code_is_retryable(self):
|
||
assert (
|
||
ck_relogin.classify_failure(
|
||
"重新登录失败: 提交验证码失败: 邮箱验证码错误，请重新输入"
|
||
)
|
||
== "email_code"
|
||
)
|
||
|
||
def test_interrupted_is_retryable(self):
|
||
assert (
|
||
ck_relogin.classify_failure("登录已中断（服务重启），旧 Cookie 已保留")
|
||
== "interrupted"
|
||
)
|
||
assert (
|
||
ck_relogin.classify_failure("重新登录已停止，旧 Cookie 已保留")
|
||
== "interrupted"
|
||
)
|
||
|
||
def test_credential_is_manual(self):
|
||
assert (
|
||
ck_relogin.classify_failure("重新登录失败: 账号缺少密码或邮箱")
|
||
== "credential"
|
||
)
|
||
assert "credential" in ck_relogin.MANUAL_KINDS
|
||
|
||
def test_unknown_defaults_to_retryable(self):
|
||
"""未知文案按可重试处理：有每日上限兜底，别把可能只是抖动的账号推给人工。"""
|
||
assert ck_relogin.classify_failure("某种没见过的报错") == "network"
|
||
assert ck_relogin.classify_failure("") == "network"
|
||
|
||
|
||
class TestBackoff(_Base):
|
||
def test_ladder_is_escalating(self):
|
||
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
|
||
first = ck_relogin._next_retry_at("email_code", 1, now) - now
|
||
second = ck_relogin._next_retry_at("email_code", 2, now) - now
|
||
third = ck_relogin._next_retry_at("email_code", 3, now) - now
|
||
assert first == timedelta(minutes=60)
|
||
assert second == timedelta(minutes=180)
|
||
assert third == timedelta(minutes=720)
|
||
|
||
def test_quota_goes_to_next_day_with_jitter(self):
|
||
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
|
||
nxt = ck_relogin._next_retry_at("email_quota", 1, now)
|
||
assert nxt.date() == (now + timedelta(days=1)).date()
|
||
assert nxt.hour == 0
|
||
assert 0 <= nxt.minute <= ck_relogin.QUOTA_JITTER_MINUTES
|
||
|
||
|
||
class TestSelection(_Base):
|
||
def _ids(self, job):
|
||
now = datetime.now(UTC)
|
||
return [
|
||
t.id
|
||
for t in ck_relogin.build_select_query(
|
||
self.session, job, now, max_attempts=2
|
||
).all()
|
||
]
|
||
|
||
def test_picks_invalid_with_expired_backoff(self):
|
||
job = self.job()
|
||
ready = self.make_task("ready")
|
||
assert self._ids(job) == [ready.id]
|
||
|
||
def test_skips_valid_and_empty_cookie_and_opted_out(self):
|
||
job = self.job()
|
||
keep = self.make_task("keep")
|
||
self.make_task("valid", ck_check_status="valid")
|
||
self.make_task("nocookie", cookie="")
|
||
self.make_task("optedout", auto_managed=False)
|
||
assert self._ids(job) == [keep.id]
|
||
|
||
def test_skips_inflight_attempt(self):
|
||
job = self.job()
|
||
keep = self.make_task("keep")
|
||
self.make_task("running", attempt_status="running")
|
||
self.make_task("pending", attempt_status="pending")
|
||
assert self._ids(job) == [keep.id]
|
||
|
||
def test_respects_backoff_window(self):
|
||
job = self.job()
|
||
due = self.make_task(
|
||
"due", relogin_next_at=datetime.now(UTC) - timedelta(minutes=1)
|
||
)
|
||
self.make_task("later", relogin_next_at=datetime.now(UTC) + timedelta(hours=3))
|
||
assert self._ids(job) == [due.id]
|
||
|
||
def test_skips_when_attempts_exhausted(self):
|
||
job = self.job()
|
||
keep = self.make_task("keep")
|
||
self.make_task("exhausted", relogin_attempts=2)
|
||
assert self._ids(job) == [keep.id]
|
||
|
||
def test_never_retries_manual_kinds(self):
|
||
"""已判定极验/凭证问题的账号，绝不能再次自动重登。"""
|
||
job = self.job()
|
||
keep = self.make_task("keep")
|
||
self.make_task("captcha", relogin_failure_kind="captcha")
|
||
self.make_task("credential", relogin_failure_kind="credential")
|
||
assert self._ids(job) == [keep.id]
|
||
|
||
def test_retryable_kinds_stay_eligible(self):
|
||
job = self.job()
|
||
code = self.make_task("code", relogin_failure_kind="email_code")
|
||
net = self.make_task("net", relogin_failure_kind="network")
|
||
assert sorted(self._ids(job)) == sorted([code.id, net.id])
|
||
|
||
def test_generated_sql_is_mysql_compatible(self):
|
||
from sqlalchemy.dialects import mysql
|
||
|
||
query = ck_relogin.build_select_query(
|
||
self.session, self.job(), datetime.now(UTC), max_attempts=2
|
||
)
|
||
sql = str(query.statement.compile(dialect=mysql.dialect())).upper()
|
||
assert "NULLS FIRST" not in sql and "NULLS LAST" not in sql
|
||
|
||
def test_batch_size_caps_the_round(self):
|
||
job = self.job(batch_size=3)
|
||
for index in range(6):
|
||
self.make_task(f"a{index}")
|
||
assert len(self._ids(job)) == 3
|
||
|
||
|
||
class TestSettlement(_Base):
|
||
"""结算：成功清零；失败按分型决定重试还是转人工。"""
|
||
|
||
def _settle(self, tasks, *, max_attempts=2):
|
||
now = datetime.now(UTC)
|
||
fresh = {t.id: t for t in tasks}
|
||
counters = ck_relogin.settle_results(
|
||
fresh, [t.id for t in tasks], now=now, max_attempts=max_attempts
|
||
)
|
||
self.session.commit()
|
||
return counters, now
|
||
|
||
def test_success_resets_state(self):
|
||
task = self.make_task(
|
||
"ok",
|
||
attempt_status="success",
|
||
relogin_attempts=1,
|
||
relogin_failure_kind="email_code",
|
||
relogin_last_error="旧的报错",
|
||
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
|
||
)
|
||
counters, _ = self._settle([task])
|
||
self.session.refresh(task)
|
||
assert counters["valid"] == 1 and counters["invalid"] == 0
|
||
assert task.relogin_attempts == 0
|
||
assert task.relogin_next_at is None
|
||
assert task.relogin_failure_kind == ""
|
||
assert task.relogin_last_error == ""
|
||
|
||
def test_retryable_failure_schedules_next_attempt(self):
|
||
task = self.make_task(
|
||
"retry",
|
||
attempt_status="failed",
|
||
attempt_message="等待验证码超时（旧 Cookie 已保留）",
|
||
)
|
||
counters, now = self._settle([task])
|
||
self.session.refresh(task)
|
||
assert counters["invalid"] == 1 and counters["retry"] == 1
|
||
assert task.relogin_failure_kind == "email_code"
|
||
assert task.relogin_attempts == 1
|
||
assert task.relogin_next_at is not None and task.relogin_next_at > now
|
||
assert "旧 Cookie" in task.relogin_last_error
|
||
|
||
def test_captcha_goes_to_manual_without_schedule(self):
|
||
task = self.make_task(
|
||
"cap", attempt_status="failed", attempt_message="极验验证失败: slide"
|
||
)
|
||
counters, _ = self._settle([task])
|
||
self.session.refresh(task)
|
||
assert counters["manual"] == 1 and counters["retry"] == 0
|
||
assert task.relogin_failure_kind == "captcha"
|
||
assert task.relogin_next_at is None # 不再自动排期
|
||
|
||
def test_quota_backs_off_to_next_day(self):
|
||
task = self.make_task(
|
||
"quota",
|
||
attempt_status="failed",
|
||
attempt_message="您的邮箱验证码发送次数已达上限，请改天再试！",
|
||
)
|
||
counters, now = self._settle([task])
|
||
self.session.refresh(task)
|
||
assert counters["email_quota"] == 1 and counters["retry"] == 1
|
||
assert task.relogin_next_at.date() == (now + timedelta(days=1)).date()
|
||
|
||
def test_attempts_exhausted_stops_retrying(self):
|
||
task = self.make_task(
|
||
"used",
|
||
attempt_status="failed",
|
||
attempt_message="等待验证码超时",
|
||
relogin_attempts=1,
|
||
)
|
||
counters, _ = self._settle([task], max_attempts=2)
|
||
self.session.refresh(task)
|
||
assert counters["manual"] == 1 and counters["retry"] == 0
|
||
assert task.relogin_next_at is None
|
||
|
||
def test_settlement_never_touches_cookie(self):
|
||
task = self.make_task(
|
||
"keepck", attempt_status="failed", attempt_message="极验验证失败: slide"
|
||
)
|
||
before = (task.cookie, task.status)
|
||
self._settle([task])
|
||
self.session.refresh(task)
|
||
assert (task.cookie, task.status) == before
|
||
|
||
|
||
class TestManualQueueApi(_Base):
|
||
"""待人工处理队列：只列自动重登救不了的，按原因分组。"""
|
||
|
||
def _queue(self):
|
||
from web.backend.routers import automation as api
|
||
|
||
return api.manual_queue(limit=20, db=self.session, current=self.admin)
|
||
|
||
def test_groups_manual_kinds_only(self):
|
||
captcha = self.make_task(
|
||
"cap",
|
||
attempt_status="failed",
|
||
attempt_message="极验验证失败: slide",
|
||
relogin_failure_kind="captcha",
|
||
relogin_attempts=1,
|
||
relogin_last_error="极验验证失败: slide",
|
||
)
|
||
self.make_task(
|
||
"cred",
|
||
attempt_status="failed",
|
||
attempt_message="账号缺少密码或邮箱",
|
||
relogin_failure_kind="credential",
|
||
relogin_attempts=1,
|
||
relogin_last_error="账号缺少密码或邮箱",
|
||
)
|
||
# 可自动重试的不该出现在人工队列里
|
||
self.make_task(
|
||
"retryable",
|
||
attempt_status="failed",
|
||
attempt_message="等待验证码超时",
|
||
relogin_failure_kind="email_code",
|
||
relogin_attempts=1,
|
||
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
|
||
relogin_last_error="等待验证码超时",
|
||
)
|
||
payload = self._queue()
|
||
kinds = [group["kind"] for group in payload["groups"]]
|
||
assert sorted(kinds) == ["captcha", "credential"]
|
||
assert payload["total"] == 2
|
||
captcha_group = next(g for g in payload["groups"] if g["kind"] == "captcha")
|
||
assert captcha_group["count"] == 1
|
||
assert captcha_group["samples"][0]["task_id"] == captcha.id
|
||
assert "极验" in captcha_group["label"]
|
||
|
||
def test_exhausted_retries_show_up_as_manual(self):
|
||
"""次数用尽的（next_run_at 已清空）也要冒出来，否则会静默卡住。"""
|
||
self.make_task(
|
||
"used",
|
||
attempt_status="failed",
|
||
attempt_message="等待验证码超时",
|
||
relogin_failure_kind="email_code",
|
||
relogin_attempts=2,
|
||
relogin_next_at=None,
|
||
relogin_last_error="等待验证码超时",
|
||
)
|
||
payload = self._queue()
|
||
assert payload["total"] == 1
|
||
assert "重试次数用尽" in payload["groups"][0]["label"]
|
||
|
||
def test_empty_queue(self):
|
||
payload = self._queue()
|
||
assert payload["total"] == 0 and payload["groups"] == []
|
||
|
||
|
||
class TestLinkageWithHealthJob(_Base):
|
||
"""两个作业是上下游：jobs 接口要暴露「当前有多少条等着自动重登」。
|
||
|
||
页面靠这个数字把巡检 → 重登的关系显性化；没有它，两个开关看起来像互不相干。
|
||
"""
|
||
|
||
def test_pending_relogin_counts_only_actionable_invalid(self):
|
||
from web.backend.routers import automation as api
|
||
|
||
ready = self.make_task("ready")
|
||
self.make_task("valid", ck_check_status="valid")
|
||
self.make_task("nocookie", cookie="")
|
||
self.make_task("optedout", auto_managed=False)
|
||
|
||
payload = api.list_jobs(db=self.session, current=self.admin)
|
||
# 只有 ready 同时满足：invalid + 有 CK + 允许自动管理
|
||
assert payload["pending_relogin"] == 1
|
||
assert any(job["key"] == ck_relogin.JOB_KEY for job in payload["jobs"])
|
||
assert ready.id is not None
|
||
|
||
def test_pending_count_is_not_capped_by_batch_size(self):
|
||
"""积压数不能被批次上限截断。
|
||
|
||
一开始复用了作业的候选查询来取 count，而那个查询带 LIMIT batch_size，
|
||
页面上于是恒显示 10（批次上限），完全看不出真实积压 —— 线上就是这样
|
||
显示了「待重登 10」而实际有一千多。
|
||
"""
|
||
from web.backend.routers import automation as api
|
||
|
||
for index in range(25):
|
||
self.make_task(f"acc{index}")
|
||
payload = api.list_jobs(db=self.session, current=self.admin)
|
||
relogin = next(j for j in payload["jobs"] if j["key"] == ck_relogin.JOB_KEY)
|
||
assert relogin["batch_size"] == 10
|
||
assert payload["pending_relogin"] == 25 # 不被 limit 10 截断
|