feat(automation): CK 自动重登(P1b)+ 失败分型与待人工队列

P1a 的巡检只产出「谁失效了」的清单,不兑现修复:1,356 个失效账号仍要人工逐个
选集→点重登→过滑块。线上实测 20 个失效账号重登 20/20 成功(2 分 13 秒、零极验),
证实失效是会话级过期而非凭证问题,可救量约 1,350。

后端:
- 迁移 20261002_0038:login_tasks 加 relogin_failure_kind(枚举),供作业决定
  「继续退避重试」还是「转人工」;不用 attempt_message 做字符串匹配(文案一改
  就失效,页面也要按类型筛选);
- jobs/ck_relogin.py:
  - 候选 = 巡检判定 invalid + 允许自动管理 + 无在途尝试 + 退避到期 + 未超每日上限
    + **未转人工**(极验/凭证问题不再自动重登);
  - 复用 LoginBatchRunner(mode="relogin"),成功才替换 CK(沿用既有安全语义)。
    自动化身份取最早的超管做 created_by,权限给 login:view_all 以免受归属限制;
    总时长上限压到 240 秒,避免长时间占住调度线程;
  - settle_results() 独立成纯函数便于单测:成功清空退避状态;失败按分型分流 ——
    captcha/credential 转人工(不排期)、email_quota 退避到次日零点+随机抖动、
    email_code/interrupted/network 走 1h→3h→12h→24h 阶梯;次数用尽也停下等人工;
  - 未知文案默认按可重试处理(有每日上限兜底),别把可能只是抖动的账号推给人工;
- 注册第二个作业「CK 自动重登」(batch 10 / 并发 3,**默认关闭**);
- 新增 GET /api/automation/manual-queue:按原因分组列出待人工处理的账号。可重试
  类型但已无下次排期的(次数用尽)单独成组 —— 否则会混进「验证码失败」里,客服
  看不出这个已经不会再自动试了。

前端:
- /automation 页新增「待人工处理」卡片,按分型分组展示(极验要过滑块、凭证要核对、
  次数用尽要人工确认),各带账号与最近失败原因。

测试:新增 26 条(分型顺序陷阱「上限」优先于「验证码」、退避阶梯、配额退避次日、
候选筛选、MySQL 方言编译、结算、待人工队列分组)。
提交前全量 297 通过、ruff 全绿、前端 tsc 通过。

注意:提交时另一路「测试改用 MySQL」的改造正在同一工作区进行,本次刻意只暂存
本功能相关文件,未触碰 tests/conftest.py、tests/support/ 及对方已转换的测试文件。
This commit is contained in:
yml2213
2026-10-02 20:51:19 +08:00
parent 5645f19102
commit 8fc8e623b9
11 changed files with 1000 additions and 65 deletions
+7 -3
View File
@@ -406,8 +406,8 @@ class TestAutomationApi(_Base):
assert payload["scheduler"]["tick_seconds"] == 300
assert payload["scheduler"]["running"] is False
keys = [item["key"] for item in payload["jobs"]]
assert keys == [ck_health.JOB_KEY]
job = payload["jobs"][0]
assert ck_health.JOB_KEY in keys
job = next(item for item in payload["jobs"] if item["key"] == ck_health.JOB_KEY)
assert job["enabled"] is False
assert job["interval_seconds"] == 6 * 3600
assert job["batch_size"] == 200
@@ -423,7 +423,11 @@ class TestAutomationApi(_Base):
db=self.session,
current=self.admin,
)
job = self.session.query(AutomationJob).one()
job = (
self.session.query(AutomationJob)
.filter(AutomationJob.key == ck_health.JOB_KEY)
.one()
)
assert job.enabled is True
assert job.interval_seconds == 3600
assert job.batch_size == 5
+377
View File
@@ -0,0 +1,377 @@
"""P1b 回归:CK 自动重登的候选筛选、失败分型与结算。
两件事必须一起守住:
1. **分型正确**:线上 415 个重登失败里只有 35%(极验)真的需要人工,另外 64%
(取码失败 + 重启中断)其实可自动重试。分错方向就是白扔可救名额,或者让
极验账号反复撞风控。
2. **选行安全**:只挑「巡检判定失效 + 允许自动管理 + 没有在途尝试 + 退避到期 +
没转人工」的行;转人工的类型绝不能被再次自动重登。
结算逻辑(``settle_results``)与选行查询(``build_select_query``)都刻意抽成了
纯函数/可编译对象,这里不碰真实登录链路。
"""
from datetime import UTC, datetime, timedelta
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from web.backend.database import Base
from web.backend.models import Account, AutomationJob, LoginTask, User
from web.backend.services.automation.jobs import ck_relogin
class _Base:
def setup_method(self):
self.engine = create_engine("sqlite://")
Base.metadata.create_all(self.engine)
self.session = sessionmaker(bind=self.engine)()
self.admin = User(
username="admin", password_hash="h", role="super_admin", is_active=True
)
self.session.add(self.admin)
self.session.commit()
def teardown_method(self):
self.session.close()
def make_task(
self,
name="acc",
*,
ck_check_status="invalid",
cookie="acf_uid=1",
attempt_status="",
attempt_message="",
relogin_attempts=0,
relogin_next_at=None,
relogin_failure_kind="",
relogin_last_error="",
auto_managed=True,
):
account = Account(
username=name,
password="p",
email=f"{name}@e.com",
email_password="ep",
auto_managed=auto_managed,
)
self.session.add(account)
self.session.commit()
task = LoginTask(
batch_id="b",
account_id=account.id,
created_by=self.admin.id,
status="success",
cookie=cookie,
ck_check_status=ck_check_status,
attempt_status=attempt_status,
attempt_message=attempt_message,
relogin_attempts=relogin_attempts,
relogin_next_at=relogin_next_at,
relogin_failure_kind=relogin_failure_kind,
relogin_last_error=relogin_last_error,
)
self.session.add(task)
self.session.commit()
return task
def job(self, batch_size=10):
return AutomationJob(
key=ck_relogin.JOB_KEY,
interval_seconds=6 * 3600,
batch_size=batch_size,
concurrency=3,
)
class TestFailureClassification(_Base):
def test_quota_wins_over_code(self):
"""「发送次数已达上限」既含验证码又含上限,必须优先判配额。
判成 email_code 会按小时反复撞配额;判成 email_quota 才退避到次日。
"""
msg = "重新登录失败: 提交验证码失败: 您的邮箱验证码发送次数已达上限,请改天再试!(旧 Cookie 已保留)"
assert ck_relogin.classify_failure(msg) == "email_quota"
def test_captcha_is_manual(self):
assert (
ck_relogin.classify_failure(
"重新登录失败: 极验验证失败: slide(旧 Cookie 已保留)"
)
== "captcha"
)
assert "captcha" in ck_relogin.MANUAL_KINDS
def test_email_code_is_retryable(self):
assert (
ck_relogin.classify_failure(
"重新登录失败: 提交验证码失败: 邮箱验证码错误,请重新输入"
)
== "email_code"
)
def test_interrupted_is_retryable(self):
assert (
ck_relogin.classify_failure("登录已中断(服务重启),旧 Cookie 已保留")
== "interrupted"
)
assert (
ck_relogin.classify_failure("重新登录已停止,旧 Cookie 已保留")
== "interrupted"
)
def test_credential_is_manual(self):
assert (
ck_relogin.classify_failure("重新登录失败: 账号缺少密码或邮箱")
== "credential"
)
assert "credential" in ck_relogin.MANUAL_KINDS
def test_unknown_defaults_to_retryable(self):
"""未知文案按可重试处理:有每日上限兜底,别把可能只是抖动的账号推给人工。"""
assert ck_relogin.classify_failure("某种没见过的报错") == "network"
assert ck_relogin.classify_failure("") == "network"
class TestBackoff(_Base):
def test_ladder_is_escalating(self):
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
first = ck_relogin._next_retry_at("email_code", 1, now) - now
second = ck_relogin._next_retry_at("email_code", 2, now) - now
third = ck_relogin._next_retry_at("email_code", 3, now) - now
assert first == timedelta(minutes=60)
assert second == timedelta(minutes=180)
assert third == timedelta(minutes=720)
def test_quota_goes_to_next_day_with_jitter(self):
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
nxt = ck_relogin._next_retry_at("email_quota", 1, now)
assert nxt.date() == (now + timedelta(days=1)).date()
assert nxt.hour == 0
assert 0 <= nxt.minute <= ck_relogin.QUOTA_JITTER_MINUTES
class TestSelection(_Base):
def _ids(self, job):
now = datetime.now(UTC)
return [
t.id
for t in ck_relogin.build_select_query(
self.session, job, now, max_attempts=2
).all()
]
def test_picks_invalid_with_expired_backoff(self):
job = self.job()
ready = self.make_task("ready")
assert self._ids(job) == [ready.id]
def test_skips_valid_and_empty_cookie_and_opted_out(self):
job = self.job()
keep = self.make_task("keep")
self.make_task("valid", ck_check_status="valid")
self.make_task("nocookie", cookie="")
self.make_task("optedout", auto_managed=False)
assert self._ids(job) == [keep.id]
def test_skips_inflight_attempt(self):
job = self.job()
keep = self.make_task("keep")
self.make_task("running", attempt_status="running")
self.make_task("pending", attempt_status="pending")
assert self._ids(job) == [keep.id]
def test_respects_backoff_window(self):
job = self.job()
due = self.make_task(
"due", relogin_next_at=datetime.now(UTC) - timedelta(minutes=1)
)
self.make_task("later", relogin_next_at=datetime.now(UTC) + timedelta(hours=3))
assert self._ids(job) == [due.id]
def test_skips_when_attempts_exhausted(self):
job = self.job()
keep = self.make_task("keep")
self.make_task("exhausted", relogin_attempts=2)
assert self._ids(job) == [keep.id]
def test_never_retries_manual_kinds(self):
"""已判定极验/凭证问题的账号,绝不能再次自动重登。"""
job = self.job()
keep = self.make_task("keep")
self.make_task("captcha", relogin_failure_kind="captcha")
self.make_task("credential", relogin_failure_kind="credential")
assert self._ids(job) == [keep.id]
def test_retryable_kinds_stay_eligible(self):
job = self.job()
code = self.make_task("code", relogin_failure_kind="email_code")
net = self.make_task("net", relogin_failure_kind="network")
assert sorted(self._ids(job)) == sorted([code.id, net.id])
def test_generated_sql_is_mysql_compatible(self):
from sqlalchemy.dialects import mysql
query = ck_relogin.build_select_query(
self.session, self.job(), datetime.now(UTC), max_attempts=2
)
sql = str(query.statement.compile(dialect=mysql.dialect())).upper()
assert "NULLS FIRST" not in sql and "NULLS LAST" not in sql
def test_batch_size_caps_the_round(self):
job = self.job(batch_size=3)
for index in range(6):
self.make_task(f"a{index}")
assert len(self._ids(job)) == 3
class TestSettlement(_Base):
"""结算:成功清零;失败按分型决定重试还是转人工。"""
def _settle(self, tasks, *, max_attempts=2):
now = datetime.now(UTC)
fresh = {t.id: t for t in tasks}
counters = ck_relogin.settle_results(
fresh, [t.id for t in tasks], now=now, max_attempts=max_attempts
)
self.session.commit()
return counters, now
def test_success_resets_state(self):
task = self.make_task(
"ok",
attempt_status="success",
relogin_attempts=1,
relogin_failure_kind="email_code",
relogin_last_error="旧的报错",
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
)
counters, _ = self._settle([task])
self.session.refresh(task)
assert counters["valid"] == 1 and counters["invalid"] == 0
assert task.relogin_attempts == 0
assert task.relogin_next_at is None
assert task.relogin_failure_kind == ""
assert task.relogin_last_error == ""
def test_retryable_failure_schedules_next_attempt(self):
task = self.make_task(
"retry",
attempt_status="failed",
attempt_message="等待验证码超时(旧 Cookie 已保留)",
)
counters, now = self._settle([task])
self.session.refresh(task)
assert counters["invalid"] == 1 and counters["retry"] == 1
assert task.relogin_failure_kind == "email_code"
assert task.relogin_attempts == 1
assert task.relogin_next_at is not None and task.relogin_next_at > now
assert "旧 Cookie" in task.relogin_last_error
def test_captcha_goes_to_manual_without_schedule(self):
task = self.make_task(
"cap", attempt_status="failed", attempt_message="极验验证失败: slide"
)
counters, _ = self._settle([task])
self.session.refresh(task)
assert counters["manual"] == 1 and counters["retry"] == 0
assert task.relogin_failure_kind == "captcha"
assert task.relogin_next_at is None # 不再自动排期
def test_quota_backs_off_to_next_day(self):
task = self.make_task(
"quota",
attempt_status="failed",
attempt_message="您的邮箱验证码发送次数已达上限,请改天再试!",
)
counters, now = self._settle([task])
self.session.refresh(task)
assert counters["email_quota"] == 1 and counters["retry"] == 1
assert task.relogin_next_at.date() == (now + timedelta(days=1)).date()
def test_attempts_exhausted_stops_retrying(self):
task = self.make_task(
"used",
attempt_status="failed",
attempt_message="等待验证码超时",
relogin_attempts=1,
)
counters, _ = self._settle([task], max_attempts=2)
self.session.refresh(task)
assert counters["manual"] == 1 and counters["retry"] == 0
assert task.relogin_next_at is None
def test_settlement_never_touches_cookie(self):
task = self.make_task(
"keepck", attempt_status="failed", attempt_message="极验验证失败: slide"
)
before = (task.cookie, task.status)
self._settle([task])
self.session.refresh(task)
assert (task.cookie, task.status) == before
class TestManualQueueApi(_Base):
"""待人工处理队列:只列自动重登救不了的,按原因分组。"""
def _queue(self):
from web.backend.routers import automation as api
return api.manual_queue(limit=20, db=self.session, current=self.admin)
def test_groups_manual_kinds_only(self):
captcha = self.make_task(
"cap",
attempt_status="failed",
attempt_message="极验验证失败: slide",
relogin_failure_kind="captcha",
relogin_attempts=1,
relogin_last_error="极验验证失败: slide",
)
self.make_task(
"cred",
attempt_status="failed",
attempt_message="账号缺少密码或邮箱",
relogin_failure_kind="credential",
relogin_attempts=1,
relogin_last_error="账号缺少密码或邮箱",
)
# 可自动重试的不该出现在人工队列里
self.make_task(
"retryable",
attempt_status="failed",
attempt_message="等待验证码超时",
relogin_failure_kind="email_code",
relogin_attempts=1,
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
relogin_last_error="等待验证码超时",
)
payload = self._queue()
kinds = [group["kind"] for group in payload["groups"]]
assert sorted(kinds) == ["captcha", "credential"]
assert payload["total"] == 2
captcha_group = next(g for g in payload["groups"] if g["kind"] == "captcha")
assert captcha_group["count"] == 1
assert captcha_group["samples"][0]["task_id"] == captcha.id
assert "极验" in captcha_group["label"]
def test_exhausted_retries_show_up_as_manual(self):
"""次数用尽的(next_run_at 已清空)也要冒出来,否则会静默卡住。"""
self.make_task(
"used",
attempt_status="failed",
attempt_message="等待验证码超时",
relogin_failure_kind="email_code",
relogin_attempts=2,
relogin_next_at=None,
relogin_last_error="等待验证码超时",
)
payload = self._queue()
assert payload["total"] == 1
assert "重试次数用尽" in payload["groups"][0]["label"]
def test_empty_queue(self):
payload = self._queue()
assert payload["total"] == 0 and payload["groups"] == []
+1 -1
View File
@@ -22,7 +22,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261002_0037"
HEAD_REVISION = "20261002_0038"
# 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = (