feat(automation): CK 自动巡检(P1a),只检测不改 CK

后台原本靠人工逐批点「CK 检测」发现失效账号,1.2 万个账号根本盯不过来。
P1a 把「哪些账号失效了」自动化:自动挑久未检测的账号、调现成检测接口、
把结果写回 ck_check_*。**本作业不替换任何 CK**,重登仍由人工点,符合规划里
「先做只读/只提示,确认无误再做自动动作」的渐进上线原则。

实现:
- 迁移 20261002_0037:automation_jobs / automation_runs 两张表,
  login_tasks 加 relogin_attempts / relogin_next_at / relogin_last_error
  (为 P1b 备好),accounts 加 auto_managed(特殊账号不参与自动化);
- services/automation/scheduler.py:进程内调度线程,60s tick,状态全部落库
  (重启不丢调度进度、不重复跑轮);AUTOMATION_ENABLED 默认关闭,作业开关
  也默认关闭;ensure_jobs 只补缺失行、不覆盖运维改过的配置;
  作业异常只记 failed,不拖垮调度线程;单容器 --workers 1,无需分布式锁;
- services/automation/jobs/ck_health.py:选行口径与人工检测一致
  (cookie_holding_filter:持有 CK 且无在途尝试),叠加久未检测、
  auto_managed、batch_size,按 ck_checked_at 升序(最久没查的优先);
  只写 ck_check_status / ck_check_result / ck_checked_at;
- routers/automation.py:GET /jobs、PATCH /jobs/{key}、POST /jobs/{key}/run、
  GET /runs;新增权限点 automation:view(运营可看)与 automation:manage
  (仅超管),改配置与手动触发都写审计;
- 顺带补 POST /api/login/batch 的审计(此前批次是谁发起、多少账号无据可查);
- 前端 /automation 页:作业卡片(开关/周期/单轮条数/并发)、运行历史、
  「立即跑一次」;总开关未打开、调度线程未运行时给明确提示。

验证:
- 22 条新测试覆盖调度器行为(只跑到点的、异常不抛出、下轮时间、配置不被重置)、
  选行条件(在途尝试/auto_managed/间隔/batch_size)、接口与权限,其中一条
  专门断言巡检后 cookie/status/message/attempt_status 一字未动;
- 全量 257 条通过;前端 tsc 与构建通过;
- 端到端用 3 个真实账号跑真实斗鱼接口:巡检结果与直接调 check_douyu_cookie
  逐条一致(3/3 valid),CK 与尝试状态全程未变。

顺带修掉一处会炸生产的坑:DB 时间列读出来是 naive UTC,直接与
datetime.now(UTC) 比较会 TypeError,现统一走 _as_utc() 补时区。
This commit is contained in:
yml2213
2026-10-02 19:47:41 +08:00
parent 6537d77323
commit f745a61df1
19 changed files with 1721 additions and 19 deletions
+443
View File
@@ -0,0 +1,443 @@
"""P1a 回归:CK 自动巡检。
两块约束必须一起守住:
1. **调度器行为**:只跑到点的、打开开关的作业;作业抛异常只记 failed,不能
把调度线程带崩;跑完排下一轮时间;已有配置不能被重启重置。
2. **本作业绝不改 CK**:P1a 的定位是「只检测、只标记」,所以巡检后
``cookie`` / ``status`` / ``message`` / ``attempt_*`` 必须一字未动 ——
线上全靠这条保证「自动化上线了但人工流程照旧」,回归时一旦有人顺手写了
``status``,这里要立刻红。
"""
from datetime import UTC, datetime, timedelta
import pytest
from fastapi import HTTPException
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from web.backend.database import Base
from web.backend.deps import require_permission
from web.backend.models import Account, AutomationJob, AutomationRun, LoginTask, User
from web.backend.services.automation import (
JobSpec,
register_default_jobs,
register_job,
reset_registry,
run_due_jobs,
run_job_now,
)
from web.backend.services.automation.jobs import ck_health
class _Base:
def setup_method(self):
self.engine = create_engine("sqlite://")
Base.metadata.create_all(self.engine)
self.session = sessionmaker(bind=self.engine)()
self.admin = User(
username="admin", password_hash="h", role="super_admin", is_active=True
)
self.support = User(
username="kf", password_hash="h", role="support", is_active=True
)
self.session.add_all([self.admin, self.support])
self.session.commit()
# 注册表是模块级的,不重置会被上一个用例注册的探针作业污染。
reset_registry()
register_default_jobs()
def teardown_method(self):
self.session.close()
def make_account(self, name="acc", *, auto_managed=True, assigned_to=None):
account = Account(
username=name,
password="p",
email=f"{name}@example.com",
email_password="ep",
assigned_to=assigned_to,
auto_managed=auto_managed,
)
self.session.add(account)
self.session.commit()
return account
def make_task(
self,
account,
*,
cookie="acf_uid=1",
status="success",
attempt_status="",
ck_checked_at=None,
ck_check_status="",
):
task = LoginTask(
batch_id="b1",
account_id=account.id,
created_by=self.admin.id,
status=status,
cookie=cookie,
attempt_status=attempt_status,
ck_checked_at=ck_checked_at,
ck_check_status=ck_check_status,
)
self.session.add(task)
self.session.commit()
return task
class TestCkHealthSelection(_Base):
def _selected_ids(self, job):
return [
t.id for t in ck_health._select_tasks(self.session, job, datetime.now(UTC))
]
def test_picks_only_rows_that_still_hold_cookie(self):
job = self._job()
with_cookie = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b"), cookie="")
assert self._selected_ids(job) == [with_cookie.id]
def test_skips_accounts_with_inflight_attempt(self):
job = self._job()
idle = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b"), attempt_status="running")
self.make_task(self.make_account("c"), attempt_status="pending")
assert self._selected_ids(job) == [idle.id]
def test_skips_accounts_opted_out_of_automation(self):
job = self._job()
managed = self.make_task(self.make_account("a"))
self.make_task(self.make_account("b", auto_managed=False))
assert self._selected_ids(job) == [managed.id]
def test_respects_interval_and_prefers_stalest(self):
job = self._job()
fresh = self.make_task(
self.make_account("fresh"), ck_checked_at=datetime.now(UTC)
)
stale = self.make_task(
self.make_account("stale"),
ck_checked_at=datetime.now(UTC) - timedelta(hours=30),
)
never = self.make_task(self.make_account("never"), ck_checked_at=None)
ids = self._selected_ids(job)
assert fresh.id not in ids
# 从没检测过的排最前,其次是久未检测的
assert ids[0] == never.id
assert stale.id in ids
def test_batch_size_caps_the_round(self):
job = self._job()
job.batch_size = 2
for index in range(5):
self.make_task(self.make_account(f"a{index}"))
assert len(self._selected_ids(job)) == 2
def _job(self):
return AutomationJob(
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
)
class TestCkHealthNeverTouchesCookie(_Base):
def test_patrol_only_writes_check_fields(self):
job = self._job()
account = self.make_account("acc")
task = self.make_task(account, attempt_status="failed", ck_check_status="")
before = (task.cookie, task.status, task.message, task.attempt_status)
ck_health.check_douyu_cookie = _fake_check(valid=True)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
self.session.refresh(task)
assert counters["scanned"] == 1 and counters["valid"] == 1
assert task.ck_check_status == "valid"
assert task.ck_checked_at is not None
assert task.ck_check_result["nickname"] == "测试"
# 核心断言:CK 本体与尝试状态一字未动
assert (task.cookie, task.status, task.message, task.attempt_status) == before
def test_invalid_result_marks_invalid_but_keeps_cookie(self):
job = self._job()
task = self.make_task(self.make_account("acc"))
ck_health.check_douyu_cookie = _fake_check(valid=False)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
self.session.refresh(task)
assert counters["invalid"] == 1
assert task.ck_check_status == "invalid"
assert task.cookie == "acf_uid=1"
def test_detection_exception_does_not_abort_round(self):
job = self._job()
self.make_task(self.make_account("a"))
self.make_task(self.make_account("b"))
ck_health.check_douyu_cookie = _fake_check(raises=True)
try:
counters = ck_health.run_ck_health(self.session, job)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert counters["scanned"] == 2
assert counters["invalid"] == 2
def _job(self):
return AutomationJob(
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
)
class TestScheduler(_Base):
def _register_probe(self, calls):
def run(db, job):
calls.append(job.key)
return {"scanned": 3, "valid": 2, "invalid": 1, "message": "ok"}
register_job(
JobSpec(
key="probe",
title="探针",
interval_seconds=3600,
batch_size=1,
concurrency=1,
run=run,
)
)
def test_only_enabled_jobs_run(self):
from web.backend.services.automation import ensure_jobs
calls: list[str] = []
self._register_probe(calls)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
{"enabled": True}
)
self.session.commit()
run_due_jobs(self.session)
assert calls == ["probe"]
calls.clear()
run_due_jobs(self.session)
# 上一轮已排了 next_run_at,未到点不再跑
assert calls == []
def test_job_failure_is_recorded_not_raised(self):
from web.backend.services.automation import ensure_jobs
def boom(db, job):
raise RuntimeError("接口挂了")
register_job(
JobSpec(
key="boom",
title="会炸的作业",
interval_seconds=3600,
batch_size=1,
concurrency=1,
run=boom,
)
)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "boom").update(
{"enabled": True}
)
self.session.commit()
outcomes = run_due_jobs(self.session)
assert outcomes[0]["status"] == "failed"
run = self.session.query(AutomationRun).one()
assert run.status == "failed" and "接口挂了" in run.message
def test_next_run_follows_interval_and_counters_persist(self):
from web.backend.services.automation import ensure_jobs
calls: list[str] = []
self._register_probe(calls)
ensure_jobs(self.session)
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
{"enabled": True}
)
self.session.commit()
run_due_jobs(self.session)
job = (
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").one()
)
assert job.last_status == "success"
assert job.next_run_at - job.last_run_at == timedelta(seconds=3600)
run = self.session.query(AutomationRun).one()
assert (run.scanned, run.valid, run.invalid) == (3, 2, 1)
def test_ensure_jobs_keeps_operator_settings(self):
from web.backend.services.automation import ensure_jobs
ensure_jobs(self.session)
job = (
self.session.query(AutomationJob)
.filter(AutomationJob.key == ck_health.JOB_KEY)
.one()
)
job.enabled = True
job.batch_size = 7
self.session.commit()
ensure_jobs(self.session) # 模拟重启
self.session.refresh(job)
assert job.enabled is True and job.batch_size == 7
def test_run_job_now_ignores_switch(self):
from web.backend.services.automation import ensure_jobs
ensure_jobs(self.session)
outcome = run_job_now(self.session, ck_health.JOB_KEY)
assert outcome["job_key"] == ck_health.JOB_KEY
assert self.session.query(AutomationRun).count() == 1
def test_run_job_now_unknown_key(self):
with pytest.raises(KeyError):
run_job_now(self.session, "nope")
class TestAutomationPermissions(_Base):
def test_view_allows_operation_denies_support(self):
from web.backend.permissions import ROLE_PERMISSIONS
assert "automation:view" in ROLE_PERMISSIONS["operation"]
assert "automation:view" not in ROLE_PERMISSIONS["support"]
assert "automation:manage" in ROLE_PERMISSIONS["super_admin"]
assert "automation:manage" not in ROLE_PERMISSIONS["operation"]
def test_manage_dependency_rejects_operator(self):
operator = User(
username="op", password_hash="h", role="operation", is_active=True
)
checker = require_permission("automation:manage")
with pytest.raises(HTTPException) as exc:
checker(current_user=operator)
assert exc.value.status_code == 403
def test_view_dependency_accepts_operator(self):
operator = User(
username="op", password_hash="h", role="operation", is_active=True
)
checker = require_permission("automation:view")
assert checker(current_user=operator).id == operator.id
_REAL_CHECK = ck_health.check_douyu_cookie
def _fake_check(*, valid=True, raises=False):
def fake(cookie: str) -> dict:
if raises:
raise RuntimeError("网络异常")
return {
"valid": valid,
"message": "ok" if valid else "Cookie 已失效",
"fish_ball": 1,
"nickname": "测试",
"level": 30,
"checked_at": datetime.now(UTC),
}
return fake
class TestAutomationApi(_Base):
"""接口层:序列化字段、开关落库、手动触发不受开关限制。"""
def _api(self):
from web.backend.routers import automation as api
return api
def test_list_jobs_exposes_scheduler_and_job_state(self):
api = self._api()
payload = api.list_jobs(db=self.session, current=self.admin)
assert payload["scheduler"]["tick_seconds"] == 60
assert payload["scheduler"]["running"] is False
keys = [item["key"] for item in payload["jobs"]]
assert keys == [ck_health.JOB_KEY]
job = payload["jobs"][0]
assert job["enabled"] is False
assert job["interval_seconds"] == 6 * 3600
assert job["batch_size"] == 200
assert job["latest_run"] is None
def test_update_job_persists_and_schedules_next_run(self):
api = self._api()
api.update_job(
ck_health.JOB_KEY,
api.JobUpdate(
enabled=True, interval_seconds=3600, batch_size=5, concurrency=2
),
db=self.session,
current=self.admin,
)
job = self.session.query(AutomationJob).one()
assert job.enabled is True
assert job.interval_seconds == 3600
assert job.batch_size == 5
assert job.next_run_at is not None # 打开开关就排了一轮
# 审计留痕
from web.backend.models import AuditLog
entry = (
self.session.query(AuditLog)
.filter(AuditLog.action == "automation:update")
.one()
)
assert entry.target == ck_health.JOB_KEY
def test_run_endpoint_records_run_and_audit(self):
api = self._api()
self.make_task(self.make_account("a"))
ck_health.check_douyu_cookie = _fake_check(valid=True)
try:
outcome = api.run_job(
ck_health.JOB_KEY, db=self.session, current=self.admin
)
finally:
ck_health.check_douyu_cookie = _REAL_CHECK
assert outcome["status"] == "success"
assert outcome["scanned"] == 1
assert self.session.query(AutomationRun).count() == 1
from web.backend.models import AuditLog
assert (
self.session.query(AuditLog)
.filter(AuditLog.action == "automation:run")
.count()
== 1
)
def test_run_endpoint_unknown_job_returns_404(self):
api = self._api()
with pytest.raises(HTTPException) as exc:
api.run_job("nope", db=self.session, current=self.admin)
assert exc.value.status_code == 404
def test_runs_endpoint_filters_by_job_key(self):
api = self._api()
self.session.add(
AutomationRun(job_key=ck_health.JOB_KEY, status="success", scanned=1)
)
self.session.add(AutomationRun(job_key="other", status="failed"))
self.session.commit()
payload = api.list_runs(
job_key=ck_health.JOB_KEY, limit=50, db=self.session, current=self.admin
)
assert len(payload["items"]) == 1
assert payload["items"][0]["job_key"] == ck_health.JOB_KEY
+1 -1
View File
@@ -22,7 +22,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261001_0036"
HEAD_REVISION = "20261002_0037"
# 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = (