Files
live-hub-py/tests/test_automation_ck_relogin.py
T

637 lines
25 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""P1b 回归:CK 自动重登的候选筛选、失败分型与结算。
两件事必须一起守住:
1. **分型正确**:线上 415 个重登失败里只有 35%(极验)真的需要人工,另外 64%
(取码失败 + 重启中断)其实可自动重试。分错方向就是白扔可救名额,或者让
极验账号反复撞风控。
2. **选行安全**:只挑「巡检判定失效 + 允许自动管理 + 没有在途尝试 + 退避到期 +
没转人工」的行;转人工的类型绝不能被再次自动重登。
结算逻辑(``settle_results``)与选行查询(``build_select_query``)都刻意抽成了
纯函数/可编译对象,这里不碰真实登录链路。
CK 解耦(2026-10-04)后,**选行与退避进度的单位是账号**:``ck_check_status`` 与
``relogin_*`` 都落到了 ``accounts`` 上,``login_tasks`` 退化为纯登录尝试历史
(本轮只读它的 ``attempt_status`` / ``attempt_message`` / ``attempt_batch_id``)。
所以这里建的是 Account;只有「在途」这类**某次尝试**的属性才补一条 LoginTask。
"""
from datetime import UTC, datetime, timedelta
from tests.support.db import get_test_engine, make_test_session
from web.backend.models import Account, AutomationJob, LoginTask, User
from web.backend.services.account_lifecycle import derive_status
from web.backend.services.automation import register_default_jobs, reset_registry
from web.backend.services.automation.jobs import ck_relogin
class _Base:
def setup_method(self):
self.engine = get_test_engine()
self.session = make_test_session(self.engine)
# 作业注册表是模块级全局,测试之间会互相污染;重置成生产启动时的状态,
# 否则 list_jobs() 里的 ensure_jobs() 建不出作业行。
reset_registry()
register_default_jobs()
self.admin = User(
username="admin", password_hash="h", role="super_admin", is_active=True
)
self.session.add(self.admin)
self.session.commit()
def teardown_method(self):
self.session.close()
def make_account(
self,
name="acc",
*,
ck_check_status="invalid",
cookie="acf_uid=1",
relogin_attempts=0,
relogin_next_at=None,
relogin_failure_kind="",
relogin_last_error="",
auto_managed=True,
):
"""建一个账号 —— CK 与重登进度都是账号自己的属性。
``login_tasks`` 的 ``cookie`` / ``ck_check_*`` / ``relogin_*`` 旧列已不再被
新代码读写,这里也不往里写值:留作回退的列不该继续被测试固化。
"""
account = Account(
username=name,
password="p",
email=f"{name}@e.com",
email_password="ep",
auto_managed=auto_managed,
cookie=cookie,
ck_check_status=ck_check_status,
relogin_attempts=relogin_attempts,
relogin_next_at=relogin_next_at,
relogin_failure_kind=relogin_failure_kind,
relogin_last_error=relogin_last_error,
)
# 状态由生产的推导函数给出(而不是测试手写):造的是底层判据,
# 「该是什么状态」由 account_lifecycle 决定 —— 顺带验证两者一致。
account.status = derive_status(account)
self.session.add(account)
self.session.commit()
return account
def make_attempt(
self,
account,
*,
attempt_status="",
attempt_message="",
attempt_batch_id="",
status="success",
):
"""建一条登录尝试历史(本轮重登的那条记录)。"""
task = LoginTask(
batch_id="b",
account_id=account.id,
created_by=self.admin.id,
status=status,
attempt_status=attempt_status,
attempt_message=attempt_message,
attempt_batch_id=attempt_batch_id,
)
self.session.add(task)
self.session.commit()
return task
def job(self, batch_size=10):
return AutomationJob(
key=ck_relogin.JOB_KEY,
interval_seconds=6 * 3600,
batch_size=batch_size,
concurrency=3,
)
class TestFailureClassification(_Base):
def test_quota_wins_over_code(self):
"""「发送次数已达上限」既含验证码又含上限,必须优先判配额。
判成 email_code 会按小时反复撞配额;判成 email_quota 才退避到次日。
"""
msg = "重新登录失败: 提交验证码失败: 您的邮箱验证码发送次数已达上限,请改天再试!(旧 Cookie 已保留)"
assert ck_relogin.classify_failure(msg) == "email_quota"
def test_captcha_is_manual(self):
assert (
ck_relogin.classify_failure(
"重新登录失败: 极验验证失败: slide(旧 Cookie 已保留)"
)
== "captcha"
)
assert "captcha" in ck_relogin.MANUAL_KINDS
def test_email_code_is_retryable(self):
assert (
ck_relogin.classify_failure(
"重新登录失败: 提交验证码失败: 邮箱验证码错误,请重新输入"
)
== "email_code"
)
def test_interrupted_is_retryable(self):
assert (
ck_relogin.classify_failure("登录已中断(服务重启),旧 Cookie 已保留")
== "interrupted"
)
assert (
ck_relogin.classify_failure("重新登录已停止,旧 Cookie 已保留")
== "interrupted"
)
def test_credential_is_manual(self):
assert (
ck_relogin.classify_failure("重新登录失败: 账号缺少密码或邮箱")
== "credential"
)
assert "credential" in ck_relogin.MANUAL_KINDS
def test_unknown_defaults_to_retryable(self):
"""未知文案按可重试处理:有每日上限兜底,别把可能只是抖动的账号推给人工。"""
assert ck_relogin.classify_failure("某种没见过的报错") == "network"
assert ck_relogin.classify_failure("") == "network"
class TestFailureCircuit(_Base):
def test_hard_failure_ratio_pauses_job(self):
paused, hard, ratio = ck_relogin.evaluate_failure_circuit(
{
"scanned": 10,
"invalid": 7,
"email_quota": 0,
},
{},
)
assert paused is True
assert hard == 7
assert ratio == 0.7
def test_email_quota_does_not_trigger_circuit(self):
paused, hard, ratio = ck_relogin.evaluate_failure_circuit(
{
"scanned": 10,
"invalid": 10,
"email_quota": 10,
},
{},
)
assert paused is False
assert hard == 0
assert ratio == 0
def test_minimum_sample_size_is_respected(self):
paused, _, _ = ck_relogin.evaluate_failure_circuit(
{
"scanned": 5,
"invalid": 5,
"email_quota": 0,
},
{"failure_pause_min_scanned": 10},
)
assert paused is False
class TestBackoff(_Base):
def test_ladder_is_escalating(self):
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
first = ck_relogin._next_retry_at("email_code", 1, now) - now
second = ck_relogin._next_retry_at("email_code", 2, now) - now
third = ck_relogin._next_retry_at("email_code", 3, now) - now
assert first == timedelta(minutes=60)
assert second == timedelta(minutes=180)
assert third == timedelta(minutes=720)
def test_quota_goes_to_next_day_with_jitter(self):
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
nxt = ck_relogin._next_retry_at("email_quota", 1, now)
assert nxt.date() == (now + timedelta(days=1)).date()
assert nxt.hour == 0
assert 0 <= nxt.minute <= ck_relogin.QUOTA_JITTER_MINUTES
class TestSelection(_Base):
"""选行:``build_select_query`` 现在按**账号**返回,断言取 ``account.id``。"""
def _ids(self, job):
now = datetime.now(UTC)
return [
account.id
for account in ck_relogin.build_select_query(
self.session, job, now, max_attempts=2
).all()
]
def test_picks_invalid_with_expired_backoff(self):
job = self.job()
ready = self.make_account("ready")
assert self._ids(job) == [ready.id]
def test_skips_valid_and_empty_cookie_and_opted_out(self):
job = self.job()
keep = self.make_account("keep")
self.make_account("valid", ck_check_status="valid")
self.make_account("nocookie", cookie="")
self.make_account("optedout", auto_managed=False)
assert self._ids(job) == [keep.id]
def test_skips_inflight_attempt(self):
"""「在途」是某次尝试的属性,仍然要靠 login_tasks 表达。"""
job = self.job()
keep = self.make_account("keep")
running = self.make_account("running")
self.make_attempt(running, attempt_status="running")
pending = self.make_account("pending")
self.make_attempt(pending, attempt_status="pending")
assert self._ids(job) == [keep.id]
def test_respects_backoff_window(self):
job = self.job()
due = self.make_account(
"due", relogin_next_at=datetime.now(UTC) - timedelta(minutes=1)
)
self.make_account(
"later", relogin_next_at=datetime.now(UTC) + timedelta(hours=3)
)
assert self._ids(job) == [due.id]
def test_skips_when_attempts_exhausted(self):
job = self.job()
keep = self.make_account("keep")
self.make_account("exhausted", relogin_attempts=2)
assert self._ids(job) == [keep.id]
def test_never_retries_manual_kinds(self):
"""已判定极验/凭证问题的账号,绝不能再次自动重登。"""
job = self.job()
keep = self.make_account("keep")
self.make_account("captcha", relogin_failure_kind="captcha")
self.make_account("credential", relogin_failure_kind="credential")
assert self._ids(job) == [keep.id]
def test_retryable_kinds_stay_eligible(self):
job = self.job()
code = self.make_account("code", relogin_failure_kind="email_code")
net = self.make_account("net", relogin_failure_kind="network")
assert sorted(self._ids(job)) == sorted([code.id, net.id])
def test_generated_sql_is_mysql_compatible(self):
from sqlalchemy.dialects import mysql
query = ck_relogin.build_select_query(
self.session, self.job(), datetime.now(UTC), max_attempts=2
)
sql = str(query.statement.compile(dialect=mysql.dialect())).upper()
assert "NULLS FIRST" not in sql and "NULLS LAST" not in sql
def test_batch_size_caps_the_round(self):
job = self.job(batch_size=3)
for index in range(6):
self.make_account(f"a{index}")
assert len(self._ids(job)) == 3
def test_high_value_is_picked_before_plain(self):
"""名额先给第 1 档,其次第 2 档,普通失效号排最后。"""
job = self.job(batch_size=3)
plain = self.make_account("plain")
one_suit = self.make_account("one-suit")
one_suit.overview_queried_at = datetime.now(UTC)
one_suit.high_value_bairimeng_status = "owned"
one_suit.high_value_mengtu_status = "available"
one_suit.high_value_checked_at = datetime.now(UTC)
both = self.make_account("both")
both.overview_queried_at = datetime.now(UTC)
both.high_value_bairimeng_status = "available"
both.high_value_mengtu_status = "available"
both.high_value_checked_at = datetime.now(UTC)
self.session.commit()
ids = self._ids(job)
assert ids == [both.id, one_suit.id, plain.id]
class TestSettlement(_Base):
"""结算:成功清零;失败按分型决定重试还是转人工。
解耦后决策看「账号 + 本轮的尝试记录」:成败读 ``LoginTask.attempt_status``,
退避进度写**账号**的 ``relogin_*``。
"""
def _settle(self, pairs, *, max_attempts=2):
now = datetime.now(UTC)
fresh = {account.id: (account, task) for account, task in pairs}
counters = ck_relogin.settle_results(
self.session,
fresh,
[account.id for account, _ in pairs],
now=now,
max_attempts=max_attempts,
)
self.session.commit()
return counters, now
def test_success_resets_state(self):
account = self.make_account(
"ok",
relogin_attempts=1,
relogin_failure_kind="email_code",
relogin_last_error="旧的报错",
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
)
task = self.make_attempt(account, attempt_status="success")
counters, _ = self._settle([(account, task)])
self.session.refresh(account)
assert counters["valid"] == 1 and counters["invalid"] == 0
assert account.relogin_attempts == 0
assert account.relogin_next_at is None
assert account.relogin_failure_kind == ""
assert account.relogin_last_error == ""
def test_retryable_failure_schedules_next_attempt(self):
account = self.make_account("retry")
task = self.make_attempt(
account,
attempt_status="failed",
attempt_message="等待验证码超时(旧 Cookie 已保留)",
)
counters, now = self._settle([(account, task)])
self.session.refresh(account)
assert counters["invalid"] == 1 and counters["retry"] == 1
assert account.relogin_failure_kind == "email_code"
assert account.relogin_attempts == 1
assert account.relogin_next_at is not None and account.relogin_next_at > now
assert "旧 Cookie" in account.relogin_last_error
def test_captcha_goes_to_manual_without_schedule(self):
account = self.make_account("cap")
task = self.make_attempt(
account, attempt_status="failed", attempt_message="极验验证失败: slide"
)
counters, _ = self._settle([(account, task)])
self.session.refresh(account)
assert counters["manual"] == 1 and counters["retry"] == 0
assert account.relogin_failure_kind == "captcha"
assert account.relogin_next_at is None # 不再自动排期
def test_quota_backs_off_to_next_day(self):
account = self.make_account("quota")
task = self.make_attempt(
account,
attempt_status="failed",
attempt_message="您的邮箱验证码发送次数已达上限,请改天再试!",
)
counters, now = self._settle([(account, task)])
self.session.refresh(account)
assert counters["email_quota"] == 1 and counters["retry"] == 1
assert account.relogin_next_at.date() == (now + timedelta(days=1)).date()
def test_attempts_exhausted_stops_retrying(self):
account = self.make_account("used", relogin_attempts=1)
task = self.make_attempt(
account,
attempt_status="failed",
attempt_message="等待验证码超时",
)
counters, _ = self._settle([(account, task)], max_attempts=2)
self.session.refresh(account)
assert counters["manual"] == 1 and counters["retry"] == 0
assert account.relogin_next_at is None
def test_settlement_never_touches_cookie(self):
"""结算推进状态与退避进度,但**绝不碰账号持有的 CK**。
S3 之前这里断言「cookie 与 status 都不变」;状态机上线后 status
**本来就应该**被结算推进(极验 → manual)——「不碰」的对象只剩 CK。
"""
account = self.make_account("keepck", cookie="acf_uid=1")
task = self.make_attempt(
account, attempt_status="failed", attempt_message="极验验证失败: slide"
)
self._settle([(account, task)])
self.session.refresh(account)
# CK 本体一字未动
assert account.cookie == "acf_uid=1"
# 顺带确认结算确实跑了:分型写上了、状态推进到「待人工」
assert account.relogin_failure_kind == "captcha"
assert account.status == "manual"
class TestManualQueueApi(_Base):
"""待人工处理队列:只列自动重登救不了的,按原因分组。"""
def _queue(self):
from web.backend.routers import automation as api
return api.manual_queue(limit=20, db=self.session, current=self.admin)
def test_groups_manual_kinds_only(self):
captcha = self.make_account(
"cap",
relogin_failure_kind="captcha",
relogin_attempts=1,
relogin_last_error="极验验证失败: slide",
)
self.make_attempt(
captcha, attempt_status="failed", attempt_message="极验验证失败: slide"
)
cred = self.make_account(
"cred",
relogin_failure_kind="credential",
relogin_attempts=1,
relogin_last_error="账号缺少密码或邮箱",
)
self.make_attempt(
cred, attempt_status="failed", attempt_message="账号缺少密码或邮箱"
)
# 可自动重试的不该出现在人工队列里
retryable = self.make_account(
"retryable",
relogin_failure_kind="email_code",
relogin_attempts=1,
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
relogin_last_error="等待验证码超时",
)
self.make_attempt(
retryable, attempt_status="failed", attempt_message="等待验证码超时"
)
payload = self._queue()
kinds = [group["kind"] for group in payload["groups"]]
assert sorted(kinds) == ["captcha", "credential"]
assert payload["total"] == 2
captcha_group = next(g for g in payload["groups"] if g["kind"] == "captcha")
assert captcha_group["count"] == 1
assert captcha_group["samples"][0]["account_id"] == captcha.id
assert "极验" in captcha_group["label"]
def test_exhausted_retries_show_up_as_manual(self):
"""次数用尽的(next_run_at 已清空)也要冒出来,否则会静默卡住。"""
used = self.make_account(
"used",
relogin_failure_kind="email_code",
relogin_attempts=2,
relogin_next_at=None,
relogin_last_error="等待验证码超时",
)
self.make_attempt(
used, attempt_status="failed", attempt_message="等待验证码超时"
)
payload = self._queue()
assert payload["total"] == 1
assert "重试次数用尽" in payload["groups"][0]["label"]
def test_sample_last_error_comes_from_account(self):
"""样本里的 last_error 取账号的 ``relogin_last_error``,不回退到尝试文案。"""
account = self.make_account(
"cap",
relogin_failure_kind="captcha",
relogin_attempts=1,
relogin_last_error="账号侧记录的失败原因",
)
self.make_attempt(
account,
attempt_status="failed",
attempt_message="这是尝试记录里的文案,不该被当成账号的最后错误",
)
payload = self._queue()
assert payload["total"] == 1
assert (
payload["groups"][0]["samples"][0]["last_error"] == "账号侧记录的失败原因"
)
def test_empty_queue(self):
payload = self._queue()
assert payload["total"] == 0 and payload["groups"] == []
class TestLinkageWithHealthJob(_Base):
"""两个作业是上下游:jobs 接口要暴露「当前有多少条等着自动重登」。
页面靠这个数字把巡检 → 重登的关系显性化;没有它,两个开关看起来像互不相干。
解耦后 ``count_pending_relogin`` 按**账号**计数。
"""
def test_pending_relogin_counts_only_actionable_invalid(self):
from web.backend.routers import automation as api
ready = self.make_account("ready")
self.make_account("valid", ck_check_status="valid")
self.make_account("nocookie", cookie="")
self.make_account("optedout", auto_managed=False)
payload = api.list_jobs(db=self.session, current=self.admin)
# 只有 ready 同时满足:invalid + 有 CK + 允许自动管理
assert payload["pending_relogin"] == 1
assert any(job["key"] == ck_relogin.JOB_KEY for job in payload["jobs"])
assert ready.id is not None
def test_pending_count_is_not_capped_by_batch_size(self):
"""积压数不能被批次上限截断。
一开始复用了作业的候选查询来取 count,而那个查询带 LIMIT batch_size,
页面上于是恒显示 10(批次上限),完全看不出真实积压 —— 线上就是这样
显示了「待重登 10」而实际有一千多。
"""
from web.backend.routers import automation as api
for index in range(25):
self.make_account(f"acc{index}")
payload = api.list_jobs(db=self.session, current=self.admin)
relogin = next(j for j in payload["jobs"] if j["key"] == ck_relogin.JOB_KEY)
assert relogin["batch_size"] == 10
assert payload["pending_relogin"] == 25 # 不被 limit 10 截断
class TestSettlementVisibility(_Base):
"""回归:结算必须用**新事务**读回 runner 提交的结果。
MySQL 默认隔离级别是 REPEATABLE READ:在同一个事务里重新查询只会看到事务
开始时的快照。第一版实现复用调度器传进来的 session 读结果,而那 10 条重登
**实际全部成功**,结算却按失败写回(加了退避、2 条还被标成极验进人工队列)。
SQLite 的读一致性不同,这条在 SQLite 上复现不出来 —— 属于「测试跑在 SQLite
就漏掉」的典型,现在测试跑在 MySQL 上才守得住。
解耦后本轮记录靠 ``LoginTask.attempt_batch_id`` 找回,退避进度落在账号上。
"""
def test_settlement_sees_other_session_commit(self):
account = self.make_account(
"acc",
relogin_attempts=1,
relogin_failure_kind="email_code",
relogin_last_error="上一次的失败",
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
)
task = self.make_attempt(
account, attempt_status="pending", attempt_batch_id="batch-visibility"
)
# 调用方会话先开事务(模拟调度器挑完候选后的状态)。
stale = make_test_session(self.engine)
assert (
stale.query(LoginTask).filter(LoginTask.id == task.id).one().attempt_status
== "pending"
)
# runner 在另一个会话里写入成功并提交。
runner = make_test_session(self.engine)
runner.query(LoginTask).filter(
LoginTask.id == task.id
).one().attempt_status = "success"
runner.commit()
runner.close()
# 旧实现在 stale 那个事务里读,只会看到 pending —— 于是把成功写成失败。
assert (
stale.query(LoginTask).filter(LoginTask.id == task.id).one().attempt_status
== "pending"
), "REPEATABLE READ 下同一事务读不到别人的提交,这正是当初踩的坑"
counters = ck_relogin.settle_fresh_results(
"batch-visibility", now=datetime.now(UTC), max_attempts=2
)
assert counters["valid"] == 1 and counters["invalid"] == 0
# 结算写在自己的事务里。这里必须再用**一个全新会话**读回:当前测试会话的
# 事务快照在上面读 ``task.id``(commit 后属性已过期,取用即触发刷新)时
# 就已经开了,复用它会看到结算前的旧值 —— 正是这条用例要守的坑。
checker = make_test_session(self.engine)
settled = checker.get(Account, account.id)
assert settled.relogin_failure_kind == ""
assert settled.relogin_attempts == 0
assert settled.relogin_next_at is None
checker.close()
stale.close()
def test_settlement_marks_real_success_as_valid(self):
"""结算看到 success 就必须清零退避,不能写成失败。"""
account = self.make_account(
"acc",
relogin_attempts=1,
relogin_failure_kind="email_code",
relogin_last_error="上一次的失败",
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
)
task = self.make_attempt(account, attempt_status="success")
now = datetime.now(UTC)
counters = ck_relogin.settle_results(
self.session,
{account.id: (account, task)},
[account.id],
now=now,
max_attempts=2,
)
assert counters["valid"] == 1 and counters["invalid"] == 0
assert account.relogin_failure_kind == ""
assert account.relogin_next_at is None