feat(automation): CK 自动巡检(P1a),只检测不改 CK

后台原本靠人工逐批点「CK 检测」发现失效账号,1.2 万个账号根本盯不过来。
P1a 把「哪些账号失效了」自动化:自动挑久未检测的账号、调现成检测接口、
把结果写回 ck_check_*。**本作业不替换任何 CK**,重登仍由人工点,符合规划里
「先做只读/只提示,确认无误再做自动动作」的渐进上线原则。

实现:
- 迁移 20261002_0037:automation_jobs / automation_runs 两张表,
  login_tasks 加 relogin_attempts / relogin_next_at / relogin_last_error
  (为 P1b 备好),accounts 加 auto_managed(特殊账号不参与自动化);
- services/automation/scheduler.py:进程内调度线程,60s tick,状态全部落库
  (重启不丢调度进度、不重复跑轮);AUTOMATION_ENABLED 默认关闭,作业开关
  也默认关闭;ensure_jobs 只补缺失行、不覆盖运维改过的配置;
  作业异常只记 failed,不拖垮调度线程;单容器 --workers 1,无需分布式锁;
- services/automation/jobs/ck_health.py:选行口径与人工检测一致
  (cookie_holding_filter:持有 CK 且无在途尝试),叠加久未检测、
  auto_managed、batch_size,按 ck_checked_at 升序(最久没查的优先);
  只写 ck_check_status / ck_check_result / ck_checked_at;
- routers/automation.py:GET /jobs、PATCH /jobs/{key}、POST /jobs/{key}/run、
  GET /runs;新增权限点 automation:view(运营可看)与 automation:manage
  (仅超管),改配置与手动触发都写审计;
- 顺带补 POST /api/login/batch 的审计(此前批次是谁发起、多少账号无据可查);
- 前端 /automation 页:作业卡片(开关/周期/单轮条数/并发)、运行历史、
  「立即跑一次」;总开关未打开、调度线程未运行时给明确提示。

验证:
- 22 条新测试覆盖调度器行为(只跑到点的、异常不抛出、下轮时间、配置不被重置)、
  选行条件(在途尝试/auto_managed/间隔/batch_size)、接口与权限,其中一条
  专门断言巡检后 cookie/status/message/attempt_status 一字未动;
- 全量 257 条通过;前端 tsc 与构建通过;
- 端到端用 3 个真实账号跑真实斗鱼接口:巡检结果与直接调 check_douyu_cookie
  逐条一致(3/3 valid),CK 与尝试状态全程未变。

顺带修掉一处会炸生产的坑:DB 时间列读出来是 naive UTC,直接与
datetime.now(UTC) 比较会 TypeError,现统一走 _as_utc() 补时区。
This commit is contained in:
yml2213
2026-10-02 19:47:41 +08:00
parent 6537d77323
commit f745a61df1
19 changed files with 1721 additions and 19 deletions
@@ -0,0 +1,55 @@
"""自动化调度:作业注册、调度线程与各作业实现。
登记作业请放在 ``register_default_jobs()``,由 ``main.py`` 在启动时调用一次,
避免 import 顺序影响注册结果。
"""
from __future__ import annotations
from .jobs.ck_health import JOB_KEY as CK_HEALTH_KEY
from .jobs.ck_health import run_ck_health
from .scheduler import (
JobSpec,
automation_enabled,
ensure_jobs,
register_job,
registered_jobs,
reset_registry,
run_due_jobs,
run_job_now,
scheduler_status,
start_scheduler,
stop_scheduler,
)
__all__ = [
"CK_HEALTH_KEY",
"JobSpec",
"automation_enabled",
"ensure_jobs",
"register_default_jobs",
"register_job",
"registered_jobs",
"reset_registry",
"run_ck_health",
"run_due_jobs",
"run_job_now",
"scheduler_status",
"start_scheduler",
"stop_scheduler",
]
def register_default_jobs() -> None:
"""登记内置作业(幂等)。"""
register_job(
JobSpec(
key=CK_HEALTH_KEY,
title="CK 自动巡检",
# 6 小时一轮;全库 12,881 条按此反推 batch_size=200(见规划附录 C)。
interval_seconds=6 * 3600,
batch_size=200,
concurrency=5,
run=run_ck_health,
)
)
@@ -0,0 +1 @@
"""自动化作业实现。"""
@@ -0,0 +1,125 @@
"""CK 巡检作业(P1a)。
只做一件事:把「持有 CK 但久未检测」的账号挑出来,调现成的
``check_douyu_cookie()`` 检测,把结果写回 ``ck_check_*``。
**本作业不修改 CK 本身**:不写 ``cookie`` / ``status`` / ``message``,也不碰
``attempt_*``。P1a 上线后人工照旧可以点检测和重登,自动重登是 P1b 的事。
"""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import or_
from sqlalchemy.orm import Session
from ....models import Account, AutomationJob, LoginTask
from ...cookie_check_service import check_douyu_cookie
from ...douyu_service import cookie_operable_filter
JOB_KEY = "ck_health"
FLUSH_EVERY = 25
def _select_tasks(db: Session, job: AutomationJob, now: datetime) -> list[LoginTask]:
"""挑本轮要检测的记录。
口径与人工点检测一致(``cookie_operable_filter``:持有 CK 且没有在途尝试),
再加三条:距上次检测超过作业间隔、账号允许自动管理、最久没查的优先。
"""
cutoff = now - timedelta(seconds=max(int(job.interval_seconds or 0), 60))
return (
db.query(LoginTask)
.join(Account, LoginTask.account_id == Account.id)
.filter(cookie_operable_filter())
.filter(Account.auto_managed.is_(True))
.filter(
or_(
LoginTask.ck_checked_at.is_(None),
LoginTask.ck_checked_at < cutoff,
)
)
.order_by(LoginTask.ck_checked_at.asc().nullsfirst(), LoginTask.id.asc())
.limit(max(int(job.batch_size or 0), 1))
.all()
)
def _detect(task: LoginTask) -> dict[str, Any]:
try:
return check_douyu_cookie(task.cookie or "")
except Exception as exc: # noqa: BLE001 单个账号失败不影响整轮
return {
"valid": False,
"message": f"检测异常: {exc}",
"fish_ball": None,
"nickname": None,
"level": None,
"checked_at": datetime.now(UTC),
}
def run_ck_health(db: Session, job: AutomationJob) -> dict[str, Any]:
now = datetime.now(UTC)
tasks = _select_tasks(db, job, now)
if not tasks:
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
results: dict[int, dict[str, Any]] = {}
concurrency = max(min(int(job.concurrency or 1), 16), 1)
with ThreadPoolExecutor(max_workers=concurrency) as pool:
futures = {pool.submit(_detect, task): task for task in tasks}
for future in as_completed(futures):
task = futures[future]
results[task.id] = future.result()
# 只写检测结果三件套;cookie / status / attempt_* 一律不动。
valid = invalid = 0
for index, task in enumerate(tasks, start=1):
result = results.get(task.id)
if result is None:
continue
is_valid = bool(result.get("valid"))
if is_valid:
valid += 1
else:
invalid += 1
task.ck_check_status = "valid" if is_valid else "invalid"
task.ck_check_result = {
"fish_ball": result.get("fish_ball"),
"nickname": result.get("nickname"),
"level": result.get("level"),
"message": result.get("message", ""),
}
task.ck_checked_at = now
if index % FLUSH_EVERY == 0:
db.commit()
db.commit()
# 剩余待检数量只用于提示,不参与调度。
remaining = (
db.query(LoginTask)
.join(Account, LoginTask.account_id == Account.id)
.filter(cookie_operable_filter())
.filter(Account.auto_managed.is_(True))
.filter(
or_(
LoginTask.ck_checked_at.is_(None),
LoginTask.ck_checked_at
< now - timedelta(seconds=max(int(job.interval_seconds or 0), 60)),
)
)
.count()
)
message = f"本轮检测 {len(tasks)} 条,剩余待检 {remaining} 条"
return {
"scanned": len(tasks),
"valid": valid,
"invalid": invalid,
"skipped": 0,
"message": message,
"detail": {"remaining": remaining, "concurrency": concurrency},
}
@@ -0,0 +1,267 @@
"""进程内自动化调度器。
背景:后台的日常动作原本全靠人工点(判断 CK 失效、重登、收集资料、记台账)。
自动化按「渐进上线」推进,因此:
- 调度器总开关 ``AUTOMATION_ENABLED`` **默认关闭**;
- 每个作业自己还有开关,**默认关闭**,在 ``/automation`` 页面打开;
- 作业状态全部落库(``automation_jobs`` / ``automation_runs``),进程重启不丢
调度进度,也不会因为重启而重复跑一轮。
部署是单容器 ``--workers 1``,所以一个进程内线程就够,不需要分布式锁。
"""
from __future__ import annotations
import os
import threading
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from loguru import logger
from sqlalchemy.orm import Session
from ...database import SessionLocal
from ...models import AutomationJob, AutomationRun
# 调度轮询间隔:作业间隔都以它为单位对齐(默认 6h 的作业不会差这一分钟)。
TICK_SECONDS = 60
TRUTHY = {"1", "true", "yes", "on"}
@dataclass(frozen=True)
class JobSpec:
"""一个可调度的作业。``run`` 接收会话与作业行,返回本轮计数。"""
key: str
title: str
interval_seconds: int
batch_size: int
concurrency: int
run: Callable[[Session, AutomationJob], dict[str, Any]]
_REGISTRY: dict[str, JobSpec] = {}
_thread: threading.Thread | None = None
_stop_event = threading.Event()
_lock = threading.Lock()
def register_job(spec: JobSpec) -> None:
_REGISTRY[spec.key] = spec
def registered_jobs() -> dict[str, JobSpec]:
return dict(_REGISTRY)
def reset_registry() -> None:
"""清空作业注册表(测试隔离用;正常启动只调用 register_default_jobs)。"""
_REGISTRY.clear()
def automation_enabled() -> bool:
return os.getenv("AUTOMATION_ENABLED", "").strip().lower() in TRUTHY
def ensure_jobs(db: Session) -> None:
"""把注册表里的作业补进 ``automation_jobs``(幂等)。
只补缺失的行,**不覆盖已有配置** —— 运维在页面上改过的开关、间隔、批量
不能因为重启被重置回默认值。
"""
existing = {row.key for row in db.query(AutomationJob.key).all()}
for spec in _REGISTRY.values():
if spec.key in existing:
continue
db.add(
AutomationJob(
key=spec.key,
title=spec.title,
enabled=False,
interval_seconds=spec.interval_seconds,
batch_size=spec.batch_size,
concurrency=spec.concurrency,
config=None,
next_run_at=None,
last_status="",
last_message="",
updated_at=datetime.now(UTC),
)
)
db.commit()
def _as_utc(value: datetime | None) -> datetime | None:
"""DB 时间列存的是 naive UTC,读出来要补时区标记才能跟 now 比较。"""
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
def _due(job: AutomationJob, now: datetime) -> bool:
"""到点判定:没排过时间视为到点(刚打开开关就会跑一轮)。"""
next_run_at = _as_utc(job.next_run_at)
return next_run_at is None or next_run_at <= now
def run_due_jobs(db: Session, *, now: datetime | None = None) -> list[dict[str, Any]]:
"""跑一轮「到点的作业」,返回每轮结果。
抽成普通函数是为了可测:测试直接调用即可,不必启动线程或等 60 秒。
作业串行执行 —— 目前只有 CK 巡检,单轮约 20~40 秒,远小于 tick,串行
反而省掉了并发调度的复杂度。
"""
now = now or datetime.now(UTC)
outcomes: list[dict[str, Any]] = []
jobs = (
db.query(AutomationJob)
.filter(AutomationJob.enabled.is_(True))
.order_by(AutomationJob.key.asc())
.all()
)
for job in jobs:
spec = _REGISTRY.get(job.key)
if spec is None or not _due(job, now):
continue
outcomes.append(_run_one(db, job, spec))
# 到期但没注册实现的作业(例如降级回滚后残留的行)只留日志,不静默消失。
for job in jobs:
if job.key not in _REGISTRY:
logger.warning("自动化作业 {} 没有注册实现,已跳过", job.key)
return outcomes
def _run_one(db: Session, job: AutomationJob, spec: JobSpec) -> dict[str, Any]:
started = datetime.now(UTC)
run = AutomationRun(job_key=job.key, started_at=started, status="running")
db.add(run)
db.commit()
counters: dict[str, Any] = {}
status = "success"
message = ""
try:
counters = spec.run(db, job) or {}
message = str(counters.pop("message", "") or "")
except Exception as exc: # noqa: BLE001 作业边界要兜住任何异常,不能拖垮调度线程
db.rollback()
status = "failed"
message = f"{type(exc).__name__}: {exc}"[:500]
logger.exception("自动化作业 {} 执行失败", job.key)
finished = datetime.now(UTC)
run.finished_at = finished
run.status = status
run.message = message
run.scanned = int(counters.get("scanned", 0) or 0)
run.valid = int(counters.get("valid", 0) or 0)
run.invalid = int(counters.get("invalid", 0) or 0)
run.relogin_started = int(counters.get("relogin_started", 0) or 0)
run.relogin_success = int(counters.get("relogin_success", 0) or 0)
run.skipped = int(counters.get("skipped", 0) or 0)
run.detail = counters.get("detail") or None
job.last_run_at = finished
job.next_run_at = finished + timedelta(
seconds=max(int(job.interval_seconds or spec.interval_seconds), 60)
)
job.last_status = status
job.last_message = message
job.updated_at = finished
db.commit()
logger.info(
"自动化作业 {} 完成:status={} scanned={} valid={} invalid={} skipped={}",
job.key,
status,
run.scanned,
run.valid,
run.invalid,
run.skipped,
)
return {
"job_key": job.key,
"status": status,
"message": message,
"scanned": run.scanned,
"valid": run.valid,
"invalid": run.invalid,
"skipped": run.skipped,
}
def run_job_now(db: Session, key: str) -> dict[str, Any]:
"""立刻跑一个作业,**不看 enabled 开关**。
页面上的「立即跑一次」用这个:运维需要能在正式打开开关之前先试一轮。
"""
job = db.query(AutomationJob).filter(AutomationJob.key == key).first()
spec = _REGISTRY.get(key)
if job is None or spec is None:
raise KeyError(key)
return _run_one(db, job, spec)
def _loop() -> None:
db = SessionLocal()
try:
ensure_jobs(db)
except Exception: # noqa: BLE001 初始化失败不应让线程静默退出
logger.exception("自动化作业初始化失败")
finally:
db.close()
logger.info("自动化调度器已启动,tick={}s", TICK_SECONDS)
while not _stop_event.wait(TICK_SECONDS):
db = SessionLocal()
try:
run_due_jobs(db)
except Exception: # noqa: BLE001 单轮异常不能让调度线程死掉
db.rollback()
logger.exception("自动化调度轮询异常")
finally:
db.close()
logger.info("自动化调度器已停止")
def start_scheduler() -> bool:
"""启动调度线程。返回是否真的启动了(未开启总开关时返回 False)。"""
global _thread
if not automation_enabled():
logger.info("AUTOMATION_ENABLED 未开启,自动化调度器不启动")
return False
with _lock:
if _thread is not None and _thread.is_alive():
return False
_stop_event.clear()
_thread = threading.Thread(
target=_loop, name="automation-scheduler", daemon=True
)
_thread.start()
return True
def stop_scheduler() -> None:
global _thread
with _lock:
if _thread is None:
return
_stop_event.set()
_thread.join(timeout=5)
_thread = None
def scheduler_status() -> dict[str, Any]:
return {
"enabled": automation_enabled(),
"running": _thread is not None and _thread.is_alive(),
"tick_seconds": TICK_SECONDS,
"registered": sorted(_REGISTRY),
}