feat(automation): CK 自动巡检(P1a),只检测不改 CK
后台原本靠人工逐批点「CK 检测」发现失效账号,1.2 万个账号根本盯不过来。
P1a 把「哪些账号失效了」自动化:自动挑久未检测的账号、调现成检测接口、
把结果写回 ck_check_*。**本作业不替换任何 CK**,重登仍由人工点,符合规划里
「先做只读/只提示,确认无误再做自动动作」的渐进上线原则。
实现:
- 迁移 20261002_0037:automation_jobs / automation_runs 两张表,
login_tasks 加 relogin_attempts / relogin_next_at / relogin_last_error
(为 P1b 备好),accounts 加 auto_managed(特殊账号不参与自动化);
- services/automation/scheduler.py:进程内调度线程,60s tick,状态全部落库
(重启不丢调度进度、不重复跑轮);AUTOMATION_ENABLED 默认关闭,作业开关
也默认关闭;ensure_jobs 只补缺失行、不覆盖运维改过的配置;
作业异常只记 failed,不拖垮调度线程;单容器 --workers 1,无需分布式锁;
- services/automation/jobs/ck_health.py:选行口径与人工检测一致
(cookie_holding_filter:持有 CK 且无在途尝试),叠加久未检测、
auto_managed、batch_size,按 ck_checked_at 升序(最久没查的优先);
只写 ck_check_status / ck_check_result / ck_checked_at;
- routers/automation.py:GET /jobs、PATCH /jobs/{key}、POST /jobs/{key}/run、
GET /runs;新增权限点 automation:view(运营可看)与 automation:manage
(仅超管),改配置与手动触发都写审计;
- 顺带补 POST /api/login/batch 的审计(此前批次是谁发起、多少账号无据可查);
- 前端 /automation 页:作业卡片(开关/周期/单轮条数/并发)、运行历史、
「立即跑一次」;总开关未打开、调度线程未运行时给明确提示。
验证:
- 22 条新测试覆盖调度器行为(只跑到点的、异常不抛出、下轮时间、配置不被重置)、
选行条件(在途尝试/auto_managed/间隔/batch_size)、接口与权限,其中一条
专门断言巡检后 cookie/status/message/attempt_status 一字未动;
- 全量 257 条通过;前端 tsc 与构建通过;
- 端到端用 3 个真实账号跑真实斗鱼接口:巡检结果与直接调 check_douyu_cookie
逐条一致(3/3 valid),CK 与尝试状态全程未变。
顺带修掉一处会炸生产的坑:DB 时间列读出来是 naive UTC,直接与
datetime.now(UTC) 比较会 TypeError,现统一走 _as_utc() 补时区。
This commit is contained in:
@@ -0,0 +1,443 @@
|
||||
"""P1a 回归:CK 自动巡检。
|
||||
|
||||
两块约束必须一起守住:
|
||||
|
||||
1. **调度器行为**:只跑到点的、打开开关的作业;作业抛异常只记 failed,不能
|
||||
把调度线程带崩;跑完排下一轮时间;已有配置不能被重启重置。
|
||||
2. **本作业绝不改 CK**:P1a 的定位是「只检测、只标记」,所以巡检后
|
||||
``cookie`` / ``status`` / ``message`` / ``attempt_*`` 必须一字未动 ——
|
||||
线上全靠这条保证「自动化上线了但人工流程照旧」,回归时一旦有人顺手写了
|
||||
``status``,这里要立刻红。
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from web.backend.database import Base
|
||||
from web.backend.deps import require_permission
|
||||
from web.backend.models import Account, AutomationJob, AutomationRun, LoginTask, User
|
||||
from web.backend.services.automation import (
|
||||
JobSpec,
|
||||
register_default_jobs,
|
||||
register_job,
|
||||
reset_registry,
|
||||
run_due_jobs,
|
||||
run_job_now,
|
||||
)
|
||||
from web.backend.services.automation.jobs import ck_health
|
||||
|
||||
|
||||
class _Base:
|
||||
def setup_method(self):
|
||||
self.engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(self.engine)
|
||||
self.session = sessionmaker(bind=self.engine)()
|
||||
self.admin = User(
|
||||
username="admin", password_hash="h", role="super_admin", is_active=True
|
||||
)
|
||||
self.support = User(
|
||||
username="kf", password_hash="h", role="support", is_active=True
|
||||
)
|
||||
self.session.add_all([self.admin, self.support])
|
||||
self.session.commit()
|
||||
# 注册表是模块级的,不重置会被上一个用例注册的探针作业污染。
|
||||
reset_registry()
|
||||
register_default_jobs()
|
||||
|
||||
def teardown_method(self):
|
||||
self.session.close()
|
||||
|
||||
def make_account(self, name="acc", *, auto_managed=True, assigned_to=None):
|
||||
account = Account(
|
||||
username=name,
|
||||
password="p",
|
||||
email=f"{name}@example.com",
|
||||
email_password="ep",
|
||||
assigned_to=assigned_to,
|
||||
auto_managed=auto_managed,
|
||||
)
|
||||
self.session.add(account)
|
||||
self.session.commit()
|
||||
return account
|
||||
|
||||
def make_task(
|
||||
self,
|
||||
account,
|
||||
*,
|
||||
cookie="acf_uid=1",
|
||||
status="success",
|
||||
attempt_status="",
|
||||
ck_checked_at=None,
|
||||
ck_check_status="",
|
||||
):
|
||||
task = LoginTask(
|
||||
batch_id="b1",
|
||||
account_id=account.id,
|
||||
created_by=self.admin.id,
|
||||
status=status,
|
||||
cookie=cookie,
|
||||
attempt_status=attempt_status,
|
||||
ck_checked_at=ck_checked_at,
|
||||
ck_check_status=ck_check_status,
|
||||
)
|
||||
self.session.add(task)
|
||||
self.session.commit()
|
||||
return task
|
||||
|
||||
|
||||
class TestCkHealthSelection(_Base):
|
||||
def _selected_ids(self, job):
|
||||
return [
|
||||
t.id for t in ck_health._select_tasks(self.session, job, datetime.now(UTC))
|
||||
]
|
||||
|
||||
def test_picks_only_rows_that_still_hold_cookie(self):
|
||||
job = self._job()
|
||||
with_cookie = self.make_task(self.make_account("a"))
|
||||
self.make_task(self.make_account("b"), cookie="")
|
||||
assert self._selected_ids(job) == [with_cookie.id]
|
||||
|
||||
def test_skips_accounts_with_inflight_attempt(self):
|
||||
job = self._job()
|
||||
idle = self.make_task(self.make_account("a"))
|
||||
self.make_task(self.make_account("b"), attempt_status="running")
|
||||
self.make_task(self.make_account("c"), attempt_status="pending")
|
||||
assert self._selected_ids(job) == [idle.id]
|
||||
|
||||
def test_skips_accounts_opted_out_of_automation(self):
|
||||
job = self._job()
|
||||
managed = self.make_task(self.make_account("a"))
|
||||
self.make_task(self.make_account("b", auto_managed=False))
|
||||
assert self._selected_ids(job) == [managed.id]
|
||||
|
||||
def test_respects_interval_and_prefers_stalest(self):
|
||||
job = self._job()
|
||||
fresh = self.make_task(
|
||||
self.make_account("fresh"), ck_checked_at=datetime.now(UTC)
|
||||
)
|
||||
stale = self.make_task(
|
||||
self.make_account("stale"),
|
||||
ck_checked_at=datetime.now(UTC) - timedelta(hours=30),
|
||||
)
|
||||
never = self.make_task(self.make_account("never"), ck_checked_at=None)
|
||||
ids = self._selected_ids(job)
|
||||
assert fresh.id not in ids
|
||||
# 从没检测过的排最前,其次是久未检测的
|
||||
assert ids[0] == never.id
|
||||
assert stale.id in ids
|
||||
|
||||
def test_batch_size_caps_the_round(self):
|
||||
job = self._job()
|
||||
job.batch_size = 2
|
||||
for index in range(5):
|
||||
self.make_task(self.make_account(f"a{index}"))
|
||||
assert len(self._selected_ids(job)) == 2
|
||||
|
||||
def _job(self):
|
||||
return AutomationJob(
|
||||
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
|
||||
)
|
||||
|
||||
|
||||
class TestCkHealthNeverTouchesCookie(_Base):
|
||||
def test_patrol_only_writes_check_fields(self):
|
||||
job = self._job()
|
||||
account = self.make_account("acc")
|
||||
task = self.make_task(account, attempt_status="failed", ck_check_status="")
|
||||
before = (task.cookie, task.status, task.message, task.attempt_status)
|
||||
|
||||
ck_health.check_douyu_cookie = _fake_check(valid=True)
|
||||
try:
|
||||
counters = ck_health.run_ck_health(self.session, job)
|
||||
finally:
|
||||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||||
|
||||
self.session.refresh(task)
|
||||
assert counters["scanned"] == 1 and counters["valid"] == 1
|
||||
assert task.ck_check_status == "valid"
|
||||
assert task.ck_checked_at is not None
|
||||
assert task.ck_check_result["nickname"] == "测试"
|
||||
# 核心断言:CK 本体与尝试状态一字未动
|
||||
assert (task.cookie, task.status, task.message, task.attempt_status) == before
|
||||
|
||||
def test_invalid_result_marks_invalid_but_keeps_cookie(self):
|
||||
job = self._job()
|
||||
task = self.make_task(self.make_account("acc"))
|
||||
ck_health.check_douyu_cookie = _fake_check(valid=False)
|
||||
try:
|
||||
counters = ck_health.run_ck_health(self.session, job)
|
||||
finally:
|
||||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||||
self.session.refresh(task)
|
||||
assert counters["invalid"] == 1
|
||||
assert task.ck_check_status == "invalid"
|
||||
assert task.cookie == "acf_uid=1"
|
||||
|
||||
def test_detection_exception_does_not_abort_round(self):
|
||||
job = self._job()
|
||||
self.make_task(self.make_account("a"))
|
||||
self.make_task(self.make_account("b"))
|
||||
ck_health.check_douyu_cookie = _fake_check(raises=True)
|
||||
try:
|
||||
counters = ck_health.run_ck_health(self.session, job)
|
||||
finally:
|
||||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||||
assert counters["scanned"] == 2
|
||||
assert counters["invalid"] == 2
|
||||
|
||||
def _job(self):
|
||||
return AutomationJob(
|
||||
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
|
||||
)
|
||||
|
||||
|
||||
class TestScheduler(_Base):
|
||||
def _register_probe(self, calls):
|
||||
def run(db, job):
|
||||
calls.append(job.key)
|
||||
return {"scanned": 3, "valid": 2, "invalid": 1, "message": "ok"}
|
||||
|
||||
register_job(
|
||||
JobSpec(
|
||||
key="probe",
|
||||
title="探针",
|
||||
interval_seconds=3600,
|
||||
batch_size=1,
|
||||
concurrency=1,
|
||||
run=run,
|
||||
)
|
||||
)
|
||||
|
||||
def test_only_enabled_jobs_run(self):
|
||||
from web.backend.services.automation import ensure_jobs
|
||||
|
||||
calls: list[str] = []
|
||||
self._register_probe(calls)
|
||||
ensure_jobs(self.session)
|
||||
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
|
||||
{"enabled": True}
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
run_due_jobs(self.session)
|
||||
assert calls == ["probe"]
|
||||
|
||||
calls.clear()
|
||||
run_due_jobs(self.session)
|
||||
# 上一轮已排了 next_run_at,未到点不再跑
|
||||
assert calls == []
|
||||
|
||||
def test_job_failure_is_recorded_not_raised(self):
|
||||
from web.backend.services.automation import ensure_jobs
|
||||
|
||||
def boom(db, job):
|
||||
raise RuntimeError("接口挂了")
|
||||
|
||||
register_job(
|
||||
JobSpec(
|
||||
key="boom",
|
||||
title="会炸的作业",
|
||||
interval_seconds=3600,
|
||||
batch_size=1,
|
||||
concurrency=1,
|
||||
run=boom,
|
||||
)
|
||||
)
|
||||
ensure_jobs(self.session)
|
||||
self.session.query(AutomationJob).filter(AutomationJob.key == "boom").update(
|
||||
{"enabled": True}
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
outcomes = run_due_jobs(self.session)
|
||||
assert outcomes[0]["status"] == "failed"
|
||||
run = self.session.query(AutomationRun).one()
|
||||
assert run.status == "failed" and "接口挂了" in run.message
|
||||
|
||||
def test_next_run_follows_interval_and_counters_persist(self):
|
||||
from web.backend.services.automation import ensure_jobs
|
||||
|
||||
calls: list[str] = []
|
||||
self._register_probe(calls)
|
||||
ensure_jobs(self.session)
|
||||
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
|
||||
{"enabled": True}
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
run_due_jobs(self.session)
|
||||
job = (
|
||||
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").one()
|
||||
)
|
||||
assert job.last_status == "success"
|
||||
assert job.next_run_at - job.last_run_at == timedelta(seconds=3600)
|
||||
run = self.session.query(AutomationRun).one()
|
||||
assert (run.scanned, run.valid, run.invalid) == (3, 2, 1)
|
||||
|
||||
def test_ensure_jobs_keeps_operator_settings(self):
|
||||
from web.backend.services.automation import ensure_jobs
|
||||
|
||||
ensure_jobs(self.session)
|
||||
job = (
|
||||
self.session.query(AutomationJob)
|
||||
.filter(AutomationJob.key == ck_health.JOB_KEY)
|
||||
.one()
|
||||
)
|
||||
job.enabled = True
|
||||
job.batch_size = 7
|
||||
self.session.commit()
|
||||
|
||||
ensure_jobs(self.session) # 模拟重启
|
||||
self.session.refresh(job)
|
||||
assert job.enabled is True and job.batch_size == 7
|
||||
|
||||
def test_run_job_now_ignores_switch(self):
|
||||
from web.backend.services.automation import ensure_jobs
|
||||
|
||||
ensure_jobs(self.session)
|
||||
outcome = run_job_now(self.session, ck_health.JOB_KEY)
|
||||
assert outcome["job_key"] == ck_health.JOB_KEY
|
||||
assert self.session.query(AutomationRun).count() == 1
|
||||
|
||||
def test_run_job_now_unknown_key(self):
|
||||
with pytest.raises(KeyError):
|
||||
run_job_now(self.session, "nope")
|
||||
|
||||
|
||||
class TestAutomationPermissions(_Base):
|
||||
def test_view_allows_operation_denies_support(self):
|
||||
from web.backend.permissions import ROLE_PERMISSIONS
|
||||
|
||||
assert "automation:view" in ROLE_PERMISSIONS["operation"]
|
||||
assert "automation:view" not in ROLE_PERMISSIONS["support"]
|
||||
assert "automation:manage" in ROLE_PERMISSIONS["super_admin"]
|
||||
assert "automation:manage" not in ROLE_PERMISSIONS["operation"]
|
||||
|
||||
def test_manage_dependency_rejects_operator(self):
|
||||
operator = User(
|
||||
username="op", password_hash="h", role="operation", is_active=True
|
||||
)
|
||||
checker = require_permission("automation:manage")
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
checker(current_user=operator)
|
||||
assert exc.value.status_code == 403
|
||||
|
||||
def test_view_dependency_accepts_operator(self):
|
||||
operator = User(
|
||||
username="op", password_hash="h", role="operation", is_active=True
|
||||
)
|
||||
checker = require_permission("automation:view")
|
||||
assert checker(current_user=operator).id == operator.id
|
||||
|
||||
|
||||
_REAL_CHECK = ck_health.check_douyu_cookie
|
||||
|
||||
|
||||
def _fake_check(*, valid=True, raises=False):
|
||||
def fake(cookie: str) -> dict:
|
||||
if raises:
|
||||
raise RuntimeError("网络异常")
|
||||
return {
|
||||
"valid": valid,
|
||||
"message": "ok" if valid else "Cookie 已失效",
|
||||
"fish_ball": 1,
|
||||
"nickname": "测试",
|
||||
"level": 30,
|
||||
"checked_at": datetime.now(UTC),
|
||||
}
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
class TestAutomationApi(_Base):
|
||||
"""接口层:序列化字段、开关落库、手动触发不受开关限制。"""
|
||||
|
||||
def _api(self):
|
||||
from web.backend.routers import automation as api
|
||||
|
||||
return api
|
||||
|
||||
def test_list_jobs_exposes_scheduler_and_job_state(self):
|
||||
api = self._api()
|
||||
payload = api.list_jobs(db=self.session, current=self.admin)
|
||||
assert payload["scheduler"]["tick_seconds"] == 60
|
||||
assert payload["scheduler"]["running"] is False
|
||||
keys = [item["key"] for item in payload["jobs"]]
|
||||
assert keys == [ck_health.JOB_KEY]
|
||||
job = payload["jobs"][0]
|
||||
assert job["enabled"] is False
|
||||
assert job["interval_seconds"] == 6 * 3600
|
||||
assert job["batch_size"] == 200
|
||||
assert job["latest_run"] is None
|
||||
|
||||
def test_update_job_persists_and_schedules_next_run(self):
|
||||
api = self._api()
|
||||
api.update_job(
|
||||
ck_health.JOB_KEY,
|
||||
api.JobUpdate(
|
||||
enabled=True, interval_seconds=3600, batch_size=5, concurrency=2
|
||||
),
|
||||
db=self.session,
|
||||
current=self.admin,
|
||||
)
|
||||
job = self.session.query(AutomationJob).one()
|
||||
assert job.enabled is True
|
||||
assert job.interval_seconds == 3600
|
||||
assert job.batch_size == 5
|
||||
assert job.next_run_at is not None # 打开开关就排了一轮
|
||||
|
||||
# 审计留痕
|
||||
from web.backend.models import AuditLog
|
||||
|
||||
entry = (
|
||||
self.session.query(AuditLog)
|
||||
.filter(AuditLog.action == "automation:update")
|
||||
.one()
|
||||
)
|
||||
assert entry.target == ck_health.JOB_KEY
|
||||
|
||||
def test_run_endpoint_records_run_and_audit(self):
|
||||
api = self._api()
|
||||
self.make_task(self.make_account("a"))
|
||||
ck_health.check_douyu_cookie = _fake_check(valid=True)
|
||||
try:
|
||||
outcome = api.run_job(
|
||||
ck_health.JOB_KEY, db=self.session, current=self.admin
|
||||
)
|
||||
finally:
|
||||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||||
assert outcome["status"] == "success"
|
||||
assert outcome["scanned"] == 1
|
||||
assert self.session.query(AutomationRun).count() == 1
|
||||
|
||||
from web.backend.models import AuditLog
|
||||
|
||||
assert (
|
||||
self.session.query(AuditLog)
|
||||
.filter(AuditLog.action == "automation:run")
|
||||
.count()
|
||||
== 1
|
||||
)
|
||||
|
||||
def test_run_endpoint_unknown_job_returns_404(self):
|
||||
api = self._api()
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
api.run_job("nope", db=self.session, current=self.admin)
|
||||
assert exc.value.status_code == 404
|
||||
|
||||
def test_runs_endpoint_filters_by_job_key(self):
|
||||
api = self._api()
|
||||
self.session.add(
|
||||
AutomationRun(job_key=ck_health.JOB_KEY, status="success", scanned=1)
|
||||
)
|
||||
self.session.add(AutomationRun(job_key="other", status="failed"))
|
||||
self.session.commit()
|
||||
payload = api.list_runs(
|
||||
job_key=ck_health.JOB_KEY, limit=50, db=self.session, current=self.admin
|
||||
)
|
||||
assert len(payload["items"]) == 1
|
||||
assert payload["items"][0]["job_key"] == ck_health.JOB_KEY
|
||||
@@ -22,7 +22,7 @@ TESTS_DIR = Path(__file__).resolve().parent
|
||||
PROJECT_ROOT = TESTS_DIR.parent
|
||||
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
|
||||
|
||||
HEAD_REVISION = "20261001_0036"
|
||||
HEAD_REVISION = "20261002_0037"
|
||||
|
||||
# 迁移新增、但不声明在模型里的复合查询索引。
|
||||
EXTRA_INDEXES = (
|
||||
|
||||
Reference in New Issue
Block a user