Files
live-hub-py/tests/test_automation_ck_health.py
T
yml2213 8fc8e623b9 feat(automation): CK 自动重登(P1b)+ 失败分型与待人工队列
P1a 的巡检只产出「谁失效了」的清单,不兑现修复:1,356 个失效账号仍要人工逐个
选集→点重登→过滑块。线上实测 20 个失效账号重登 20/20 成功(2 分 13 秒、零极验),
证实失效是会话级过期而非凭证问题,可救量约 1,350。

后端:
- 迁移 20261002_0038:login_tasks 加 relogin_failure_kind(枚举),供作业决定
  「继续退避重试」还是「转人工」;不用 attempt_message 做字符串匹配(文案一改
  就失效,页面也要按类型筛选);
- jobs/ck_relogin.py:
  - 候选 = 巡检判定 invalid + 允许自动管理 + 无在途尝试 + 退避到期 + 未超每日上限
    + **未转人工**(极验/凭证问题不再自动重登);
  - 复用 LoginBatchRunner(mode="relogin"),成功才替换 CK(沿用既有安全语义)。
    自动化身份取最早的超管做 created_by,权限给 login:view_all 以免受归属限制;
    总时长上限压到 240 秒,避免长时间占住调度线程;
  - settle_results() 独立成纯函数便于单测:成功清空退避状态;失败按分型分流 ——
    captcha/credential 转人工(不排期)、email_quota 退避到次日零点+随机抖动、
    email_code/interrupted/network 走 1h→3h→12h→24h 阶梯;次数用尽也停下等人工;
  - 未知文案默认按可重试处理(有每日上限兜底),别把可能只是抖动的账号推给人工;
- 注册第二个作业「CK 自动重登」(batch 10 / 并发 3,**默认关闭**);
- 新增 GET /api/automation/manual-queue:按原因分组列出待人工处理的账号。可重试
  类型但已无下次排期的(次数用尽)单独成组 —— 否则会混进「验证码失败」里,客服
  看不出这个已经不会再自动试了。

前端:
- /automation 页新增「待人工处理」卡片,按分型分组展示(极验要过滑块、凭证要核对、
  次数用尽要人工确认),各带账号与最近失败原因。

测试:新增 26 条(分型顺序陷阱「上限」优先于「验证码」、退避阶梯、配额退避次日、
候选筛选、MySQL 方言编译、结算、待人工队列分组)。
提交前全量 297 通过、ruff 全绿、前端 tsc 通过。

注意:提交时另一路「测试改用 MySQL」的改造正在同一工作区进行,本次刻意只暂存
本功能相关文件,未触碰 tests/conftest.py、tests/support/ 及对方已转换的测试文件。
2026-10-02 20:51:19 +08:00

588 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""P1a 回归:CK 自动巡检。
两块约束必须一起守住:
1. **调度器行为**:只跑到点的、打开开关的作业;作业抛异常只记 failed,不能
把调度线程带崩;跑完排下一轮时间;已有配置不能被重启重置。
2. **本作业绝不改 CK**:P1a 的定位是「只检测、只标记」,所以巡检后
``cookie`` / ``status`` / ``message`` / ``attempt_*`` 必须一字未动 ——
线上全靠这条保证「自动化上线了但人工流程照旧」,回归时一旦有人顺手写了
``status``,这里要立刻红。
"""
from datetime import UTC, datetime, timedelta
import pytest
from fastapi import HTTPException
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from web.backend.database import Base
from web.backend.deps import require_permission
from web.backend.models import Account, AutomationJob, AutomationRun, LoginTask, User
from web.backend.services.automation import (
JobSpec,
register_default_jobs,
register_job,
reset_registry,
run_due_jobs,
run_job_now,
)
from web.backend.services.automation.jobs import ck_health
class _Base:
def setup_method(self):
self.engine = create_engine("sqlite://")
Base.metadata.create_all(self.engine)
self.session = sessionmaker(bind=self.engine)()
self.admin = User(
username="admin", password_hash="h", role="super_admin", is_active=True
)
self.support = User(
username="kf", password_hash="h", role="support", is_active=True
)
self.session.add_all([self.admin, self.support])
self.session.commit()
# 注册表是模块级的,不重置会被上一个用例注册的探针作业污染。
reset_registry()
register_default_jobs()
def teardown_method(self):
self.session.close()
def make_account(self, name="acc", *, auto_managed=True, assigned_to=None):
account = Account(
username=name,
password="p",
email=f"{name}@example.com",
email_password="ep",
assigned_to=assigned_to,
auto_managed=auto_managed,
)
self.session.add(account)
self.session.commit()
return account
def make_task(
self,
account,
*,
cookie="acf_uid=1",
status="success",
attempt_status="",
ck_checked_at=None,
ck_check_status="",
):
task = LoginTask(
batch_id="b1",
account_id=account.id,
created_by=self.admin.id,
status=status,
cookie=cookie,
attempt_status=attempt_status,
ck_checked_at=ck_checked_at,
ck_check_status=ck_check_status,
)
self.session.add(task)
self.session.commit()
return task
class TestCkHealthSelection(_Base):
def _selected_ids(self, job):
return [
t.id for t in ck_health._select_tasks(self.session, job, datetime.now(UTC))
]
def test_picks_only_rows_that_still_hold_cookie(self):
job = self._job()
with_cookie = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b"), cookie="")
assert self._selected_ids(job) == [with_cookie.id]
def test_skips_accounts_with_inflight_attempt(self):
job = self._job()
idle = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b"), attempt_status="running")
self.make_task(self.make_account("c"), attempt_status="pending")
assert self._selected_ids(job) == [idle.id]
def test_skips_accounts_opted_out_of_automation(self):
job = self._job()
managed = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b", auto_managed=False))
assert self._selected_ids(job) == [managed.id]
def test_respects_interval_and_prefers_stalest(self):
job = self._job()
fresh = self.make_task(
self.make_account("fresh"), ck_checked_at=datetime.now(UTC)
)
stale = self.make_task(
self.make_account("stale"),
ck_checked_at=datetime.now(UTC) - timedelta(hours=30),
)
never = self.make_task(self.make_account("never"), ck_checked_at=None)
ids = self._selected_ids(job)
assert fresh.id not in ids
# 从没检测过的排最前,其次是久未检测的
assert ids[0] == never.id
assert stale.id in ids
def test_generated_sql_is_mysql_compatible(self):
"""回归:选行 SQL 不能出现 MySQL 不认识的 NULLS FIRST。
生产是 MySQL、本地测试是 SQLite(SQLite 支持 NULLS FIRST,所以行为测试
发现不了),这里按 MySQL 方言编译一次断言。
"""
from sqlalchemy.dialects import mysql
query = ck_health.build_select_query(
self.session, self._job(), datetime.now(UTC)
)
sql = str(query.statement.compile(dialect=mysql.dialect())).upper()
assert "NULLS FIRST" not in sql and "NULLS LAST" not in sql
# 「没检测过的优先」要用可移植写法表达
assert "IS NULL DESC" in sql
def test_batch_size_caps_the_round(self):
job = self._job()
job.batch_size = 2
for index in range(5):
self.make_task(self.make_account(f"a{index}"))
assert len(self._selected_ids(job)) == 2
def _job(self):
return AutomationJob(
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
)
class TestCkHealthNeverTouchesCookie(_Base):
def test_patrol_only_writes_check_fields(self):
job = self._job()
account = self.make_account("acc")
task = self.make_task(account, attempt_status="failed", ck_check_status="")
before = (task.cookie, task.status, task.message, task.attempt_status)
ck_health.check_douyu_cookie = _fake_check(valid=True)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
self.session.refresh(task)
assert counters["scanned"] == 1 and counters["valid"] == 1
assert task.ck_check_status == "valid"
assert task.ck_checked_at is not None
assert task.ck_check_result["nickname"] == "测试"
# 核心断言:CK 本体与尝试状态一字未动
assert (task.cookie, task.status, task.message, task.attempt_status) == before
def test_invalid_result_marks_invalid_but_keeps_cookie(self):
job = self._job()
task = self.make_task(self.make_account("acc"))
# 必须是鉴权类失败(斗鱼说未登录)才会被标成 invalid
ck_health.check_douyu_cookie = _fake_check(
valid=False,
message="鱼丸接口: 账户登陆异常,请重新登陆;等级接口: 用户未登录",
)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
self.session.refresh(task)
assert counters["invalid"] == 1
assert task.ck_check_status == "invalid"
assert task.cookie == "acf_uid=1"
def test_detection_exception_does_not_abort_round(self):
"""单个账号检测抛异常不影响整轮,但也不能把它标成失效。"""
job = self._job()
first = self.make_task(self.make_account("a"), ck_check_status="valid")
second = self.make_task(self.make_account("b"), ck_check_status="valid")
ck_health.check_douyu_cookie = _fake_check(raises=True)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert counters["scanned"] == 2
# 查不到不等于失效:两行都保留原结论,只记录这次异常
assert counters["invalid"] == 0
assert counters["skipped"] == 2
for task in (first, second):
self.session.refresh(task)
assert task.ck_check_status == "valid"
assert task.ck_check_result["verdict"] == "error"
def _job(self):
return AutomationJob(
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
)
class TestScheduler(_Base):
def _register_probe(self, calls):
def run(db, job):
calls.append(job.key)
return {"scanned": 3, "valid": 2, "invalid": 1, "message": "ok"}
register_job(
JobSpec(
key="probe",
title="探针",
interval_seconds=3600,
batch_size=1,
concurrency=1,
run=run,
)
)
def test_only_enabled_jobs_run(self):
from web.backend.services.automation import ensure_jobs
calls: list[str] = []
self._register_probe(calls)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
{"enabled": True}
)
self.session.commit()
run_due_jobs(self.session)
assert calls == ["probe"]
calls.clear()
run_due_jobs(self.session)
# 上一轮已排了 next_run_at,未到点不再跑
assert calls == []
def test_job_failure_is_recorded_not_raised(self):
from web.backend.services.automation import ensure_jobs
def boom(db, job):
raise RuntimeError("接口挂了")
register_job(
JobSpec(
key="boom",
title="会炸的作业",
interval_seconds=3600,
batch_size=1,
concurrency=1,
run=boom,
)
)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "boom").update(
{"enabled": True}
)
self.session.commit()
outcomes = run_due_jobs(self.session)
assert outcomes[0]["status"] == "failed"
run = self.session.query(AutomationRun).one()
assert run.status == "failed" and "接口挂了" in run.message
def test_next_run_follows_tick_and_counters_persist(self):
from web.backend.services.automation import ensure_jobs
calls: list[str] = []
self._register_probe(calls)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
{"enabled": True}
)
self.session.commit()
run_due_jobs(self.session)
job = (
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").one()
)
assert job.last_status == "success"
# 下一轮在一个 tick 之后(作业每个 tick 都跑),与 interval_seconds 无关
from web.backend.services.automation.scheduler import TICK_SECONDS
assert _aware(job.next_run_at) - _aware(job.last_run_at) == timedelta(
seconds=TICK_SECONDS
)
run = self.session.query(AutomationRun).one()
assert (run.scanned, run.valid, run.invalid) == (3, 2, 1)
def test_ensure_jobs_keeps_operator_settings(self):
from web.backend.services.automation import ensure_jobs
ensure_jobs(self.session)
job = (
self.session.query(AutomationJob)
.filter(AutomationJob.key == ck_health.JOB_KEY)
.one()
)
job.enabled = True
job.batch_size = 7
self.session.commit()
ensure_jobs(self.session) # 模拟重启
self.session.refresh(job)
assert job.enabled is True and job.batch_size == 7
def test_run_job_now_ignores_switch(self):
from web.backend.services.automation import ensure_jobs
ensure_jobs(self.session)
outcome = run_job_now(self.session, ck_health.JOB_KEY)
assert outcome["job_key"] == ck_health.JOB_KEY
assert self.session.query(AutomationRun).count() == 1
def test_run_job_now_unknown_key(self):
with pytest.raises(KeyError):
run_job_now(self.session, "nope")
class TestAutomationPermissions(_Base):
def test_view_allows_operation_denies_support(self):
from web.backend.permissions import ROLE_PERMISSIONS
assert "automation:view" in ROLE_PERMISSIONS["operation"]
assert "automation:view" not in ROLE_PERMISSIONS["support"]
assert "automation:manage" in ROLE_PERMISSIONS["super_admin"]
assert "automation:manage" not in ROLE_PERMISSIONS["operation"]
def test_manage_dependency_rejects_operator(self):
operator = User(
username="op", password_hash="h", role="operation", is_active=True
)
checker = require_permission("automation:manage")
with pytest.raises(HTTPException) as exc:
checker(current_user=operator)
assert exc.value.status_code == 403
def test_view_dependency_accepts_operator(self):
operator = User(
username="op", password_hash="h", role="operation", is_active=True
)
checker = require_permission("automation:view")
assert checker(current_user=operator).id == operator.id
def _aware(value):
"""库里读出的是 naive UTC,比较前补时区。"""
return value if value.tzinfo else value.replace(tzinfo=UTC)
_REAL_CHECK = ck_health.check_douyu_cookie
def _fake_check(*, valid=True, raises=False, message=None):
def fake(cookie: str) -> dict:
if raises:
raise RuntimeError("网络异常")
return {
"valid": valid,
"message": message
if message is not None
else ("ok" if valid else "Cookie 已失效"),
"fish_ball": 1,
"nickname": "测试",
"level": 30,
"checked_at": datetime.now(UTC),
}
return fake
class TestAutomationApi(_Base):
"""接口层:序列化字段、开关落库、手动触发不受开关限制。"""
def _api(self):
from web.backend.routers import automation as api
return api
def test_list_jobs_exposes_scheduler_and_job_state(self):
api = self._api()
payload = api.list_jobs(db=self.session, current=self.admin)
assert payload["scheduler"]["tick_seconds"] == 300
assert payload["scheduler"]["running"] is False
keys = [item["key"] for item in payload["jobs"]]
assert ck_health.JOB_KEY in keys
job = next(item for item in payload["jobs"] if item["key"] == ck_health.JOB_KEY)
assert job["enabled"] is False
assert job["interval_seconds"] == 6 * 3600
assert job["batch_size"] == 200
assert job["latest_run"] is None
def test_update_job_persists_and_schedules_next_run(self):
api = self._api()
api.update_job(
ck_health.JOB_KEY,
api.JobUpdate(
enabled=True, interval_seconds=3600, batch_size=5, concurrency=2
),
db=self.session,
current=self.admin,
)
job = (
self.session.query(AutomationJob)
.filter(AutomationJob.key == ck_health.JOB_KEY)
.one()
)
assert job.enabled is True
assert job.interval_seconds == 3600
assert job.batch_size == 5
assert job.next_run_at is not None # 打开开关就排了一轮
# 审计留痕
from web.backend.models import AuditLog
entry = (
self.session.query(AuditLog)
.filter(AuditLog.action == "automation:update")
.one()
)
assert entry.target == ck_health.JOB_KEY
def test_run_endpoint_records_run_and_audit(self):
api = self._api()
self.make_task(self.make_account("a"))
ck_health.check_douyu_cookie = _fake_check(valid=True)
try:
outcome = api.run_job(
ck_health.JOB_KEY, db=self.session, current=self.admin
)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert outcome["status"] == "success"
assert outcome["scanned"] == 1
assert self.session.query(AutomationRun).count() == 1
from web.backend.models import AuditLog
assert (
self.session.query(AuditLog)
.filter(AuditLog.action == "automation:run")
.count()
== 1
)
def test_run_endpoint_unknown_job_returns_404(self):
api = self._api()
with pytest.raises(HTTPException) as exc:
api.run_job("nope", db=self.session, current=self.admin)
assert exc.value.status_code == 404
def test_runs_endpoint_filters_by_job_key(self):
api = self._api()
self.session.add(
AutomationRun(job_key=ck_health.JOB_KEY, status="success", scanned=1)
)
self.session.add(AutomationRun(job_key="other", status="failed"))
self.session.commit()
payload = api.list_runs(
job_key=ck_health.JOB_KEY, limit=50, db=self.session, current=self.admin
)
assert len(payload["items"]) == 1
assert payload["items"][0]["job_key"] == ck_health.JOB_KEY
class TestFailureClassification(_Base):
"""「查不到」不等于「失效」:非鉴权类失败不能覆盖原来的结论。
斗鱼接口抖动/被风控时,如果不加区分地写 invalid,一小时就能把上千个好账号
标成失效,客服会误判成 CK 大面积挂掉。这里盯三件事:分类正确、error 不改
结论、失败占比过高时自动暂停作业。
"""
def test_auth_failure_message_counts_as_invalid(self):
assert (
ck_health.classify_result(
{
"valid": False,
"message": "鱼丸接口: 账户登陆异常,请重新登陆;等级接口: 用户未登录",
}
)
== "invalid"
)
def test_network_failure_counts_as_error_not_invalid(self):
assert (
ck_health.classify_result(
{
"valid": False,
"message": "鱼丸接口: 请求失败: timeout;等级接口: 请求失败: timeout",
}
)
== "error"
)
assert (
ck_health.classify_result({"valid": False, "message": "响应异常"})
== "error"
)
def test_valid_still_valid(self):
assert ck_health.classify_result({"valid": True, "message": "有效"}) == "valid"
def test_error_keeps_previous_verdict_but_marks_checked(self):
job = self._job()
task = self.make_task(self.make_account("acc"), ck_check_status="valid")
ck_health.check_douyu_cookie = _fake_check(
valid=False, message="请求失败: timeout"
)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
self.session.refresh(task)
assert counters["valid"] == 0 and counters["invalid"] == 0
assert counters["skipped"] == 1 # error 计入 skipped
assert task.ck_check_status == "valid" # 原结论保留
assert task.ck_check_result["verdict"] == "error" # 但记录了这次查不到
assert task.ck_checked_at is not None
def test_mass_network_failure_pauses_the_job(self):
job = self._job()
job.enabled = True
job.batch_size = 40
for index in range(40):
self.make_task(self.make_account(f"a{index}"))
ck_health.check_douyu_cookie = _fake_check(
valid=False, message="请求失败: Connection refused"
)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert counters["skipped"] == 40
assert counters["detail"]["paused"] is True
assert job.enabled is False # 自动暂停,不再继续污染
assert "已自动暂停" in counters["message"]
def test_normal_invalid_does_not_pause(self):
"""真的都是死号时不能暂停 —— 首轮 100% invalid 是合法且预期的情况。"""
job = self._job()
job.enabled = True
job.batch_size = 40
for index in range(40):
self.make_task(self.make_account(f"a{index}"))
ck_health.check_douyu_cookie = _fake_check(
valid=False,
message="鱼丸接口: 账户登陆异常,请重新登陆;等级接口: 用户未登录",
)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert counters["invalid"] == 40
assert counters["skipped"] == 0
assert job.enabled is True
assert counters["detail"]["paused"] is False
def _job(self):
return AutomationJob(
key=ck_health.JOB_KEY,
interval_seconds=6 * 3600,
batch_size=200,
concurrency=3,
)