Files
live-hub-py/web/backend/services/automation/scheduler.py
T
yml2213 f745a61df1 feat(automation): CK 自动巡检(P1a),只检测不改 CK
后台原本靠人工逐批点「CK 检测」发现失效账号,1.2 万个账号根本盯不过来。
P1a 把「哪些账号失效了」自动化:自动挑久未检测的账号、调现成检测接口、
把结果写回 ck_check_*。**本作业不替换任何 CK**,重登仍由人工点,符合规划里
「先做只读/只提示,确认无误再做自动动作」的渐进上线原则。

实现:
- 迁移 20261002_0037:automation_jobs / automation_runs 两张表,
  login_tasks 加 relogin_attempts / relogin_next_at / relogin_last_error
  (为 P1b 备好),accounts 加 auto_managed(特殊账号不参与自动化);
- services/automation/scheduler.py:进程内调度线程,60s tick,状态全部落库
  (重启不丢调度进度、不重复跑轮);AUTOMATION_ENABLED 默认关闭,作业开关
  也默认关闭;ensure_jobs 只补缺失行、不覆盖运维改过的配置;
  作业异常只记 failed,不拖垮调度线程;单容器 --workers 1,无需分布式锁;
- services/automation/jobs/ck_health.py:选行口径与人工检测一致
  (cookie_holding_filter:持有 CK 且无在途尝试),叠加久未检测、
  auto_managed、batch_size,按 ck_checked_at 升序(最久没查的优先);
  只写 ck_check_status / ck_check_result / ck_checked_at;
- routers/automation.py:GET /jobs、PATCH /jobs/{key}、POST /jobs/{key}/run、
  GET /runs;新增权限点 automation:view(运营可看)与 automation:manage
  (仅超管),改配置与手动触发都写审计;
- 顺带补 POST /api/login/batch 的审计(此前批次是谁发起、多少账号无据可查);
- 前端 /automation 页:作业卡片(开关/周期/单轮条数/并发)、运行历史、
  「立即跑一次」;总开关未打开、调度线程未运行时给明确提示。

验证:
- 22 条新测试覆盖调度器行为(只跑到点的、异常不抛出、下轮时间、配置不被重置)、
  选行条件(在途尝试/auto_managed/间隔/batch_size)、接口与权限,其中一条
  专门断言巡检后 cookie/status/message/attempt_status 一字未动;
- 全量 257 条通过;前端 tsc 与构建通过;
- 端到端用 3 个真实账号跑真实斗鱼接口:巡检结果与直接调 check_douyu_cookie
  逐条一致(3/3 valid),CK 与尝试状态全程未变。

顺带修掉一处会炸生产的坑:DB 时间列读出来是 naive UTC,直接与
datetime.now(UTC) 比较会 TypeError,现统一走 _as_utc() 补时区。
2026-10-02 19:47:41 +08:00

268 lines
8.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""进程内自动化调度器。
背景:后台的日常动作原本全靠人工点(判断 CK 失效、重登、收集资料、记台账)。
自动化按「渐进上线」推进,因此:
- 调度器总开关 ``AUTOMATION_ENABLED`` **默认关闭**;
- 每个作业自己还有开关,**默认关闭**,在 ``/automation`` 页面打开;
- 作业状态全部落库(``automation_jobs`` / ``automation_runs``),进程重启不丢
调度进度,也不会因为重启而重复跑一轮。
部署是单容器 ``--workers 1``,所以一个进程内线程就够,不需要分布式锁。
"""
from __future__ import annotations
import os
import threading
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from loguru import logger
from sqlalchemy.orm import Session
from ...database import SessionLocal
from ...models import AutomationJob, AutomationRun
# 调度轮询间隔:作业间隔都以它为单位对齐(默认 6h 的作业不会差这一分钟)。
TICK_SECONDS = 60
TRUTHY = {"1", "true", "yes", "on"}
@dataclass(frozen=True)
class JobSpec:
"""一个可调度的作业。``run`` 接收会话与作业行,返回本轮计数。"""
key: str
title: str
interval_seconds: int
batch_size: int
concurrency: int
run: Callable[[Session, AutomationJob], dict[str, Any]]
_REGISTRY: dict[str, JobSpec] = {}
_thread: threading.Thread | None = None
_stop_event = threading.Event()
_lock = threading.Lock()
def register_job(spec: JobSpec) -> None:
_REGISTRY[spec.key] = spec
def registered_jobs() -> dict[str, JobSpec]:
return dict(_REGISTRY)
def reset_registry() -> None:
"""清空作业注册表(测试隔离用;正常启动只调用 register_default_jobs)。"""
_REGISTRY.clear()
def automation_enabled() -> bool:
return os.getenv("AUTOMATION_ENABLED", "").strip().lower() in TRUTHY
def ensure_jobs(db: Session) -> None:
"""把注册表里的作业补进 ``automation_jobs``(幂等)。
只补缺失的行,**不覆盖已有配置** —— 运维在页面上改过的开关、间隔、批量
不能因为重启被重置回默认值。
"""
existing = {row.key for row in db.query(AutomationJob.key).all()}
for spec in _REGISTRY.values():
if spec.key in existing:
continue
db.add(
AutomationJob(
key=spec.key,
title=spec.title,
enabled=False,
interval_seconds=spec.interval_seconds,
batch_size=spec.batch_size,
concurrency=spec.concurrency,
config=None,
next_run_at=None,
last_status="",
last_message="",
updated_at=datetime.now(UTC),
)
)
db.commit()
def _as_utc(value: datetime | None) -> datetime | None:
"""DB 时间列存的是 naive UTC,读出来要补时区标记才能跟 now 比较。"""
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
def _due(job: AutomationJob, now: datetime) -> bool:
"""到点判定:没排过时间视为到点(刚打开开关就会跑一轮)。"""
next_run_at = _as_utc(job.next_run_at)
return next_run_at is None or next_run_at <= now
def run_due_jobs(db: Session, *, now: datetime | None = None) -> list[dict[str, Any]]:
"""跑一轮「到点的作业」,返回每轮结果。
抽成普通函数是为了可测:测试直接调用即可,不必启动线程或等 60 秒。
作业串行执行 —— 目前只有 CK 巡检,单轮约 20~40 秒,远小于 tick,串行
反而省掉了并发调度的复杂度。
"""
now = now or datetime.now(UTC)
outcomes: list[dict[str, Any]] = []
jobs = (
db.query(AutomationJob)
.filter(AutomationJob.enabled.is_(True))
.order_by(AutomationJob.key.asc())
.all()
)
for job in jobs:
spec = _REGISTRY.get(job.key)
if spec is None or not _due(job, now):
continue
outcomes.append(_run_one(db, job, spec))
# 到期但没注册实现的作业(例如降级回滚后残留的行)只留日志,不静默消失。
for job in jobs:
if job.key not in _REGISTRY:
logger.warning("自动化作业 {} 没有注册实现,已跳过", job.key)
return outcomes
def _run_one(db: Session, job: AutomationJob, spec: JobSpec) -> dict[str, Any]:
started = datetime.now(UTC)
run = AutomationRun(job_key=job.key, started_at=started, status="running")
db.add(run)
db.commit()
counters: dict[str, Any] = {}
status = "success"
message = ""
try:
counters = spec.run(db, job) or {}
message = str(counters.pop("message", "") or "")
except Exception as exc: # noqa: BLE001 作业边界要兜住任何异常,不能拖垮调度线程
db.rollback()
status = "failed"
message = f"{type(exc).__name__}: {exc}"[:500]
logger.exception("自动化作业 {} 执行失败", job.key)
finished = datetime.now(UTC)
run.finished_at = finished
run.status = status
run.message = message
run.scanned = int(counters.get("scanned", 0) or 0)
run.valid = int(counters.get("valid", 0) or 0)
run.invalid = int(counters.get("invalid", 0) or 0)
run.relogin_started = int(counters.get("relogin_started", 0) or 0)
run.relogin_success = int(counters.get("relogin_success", 0) or 0)
run.skipped = int(counters.get("skipped", 0) or 0)
run.detail = counters.get("detail") or None
job.last_run_at = finished
job.next_run_at = finished + timedelta(
seconds=max(int(job.interval_seconds or spec.interval_seconds), 60)
)
job.last_status = status
job.last_message = message
job.updated_at = finished
db.commit()
logger.info(
"自动化作业 {} 完成:status={} scanned={} valid={} invalid={} skipped={}",
job.key,
status,
run.scanned,
run.valid,
run.invalid,
run.skipped,
)
return {
"job_key": job.key,
"status": status,
"message": message,
"scanned": run.scanned,
"valid": run.valid,
"invalid": run.invalid,
"skipped": run.skipped,
}
def run_job_now(db: Session, key: str) -> dict[str, Any]:
"""立刻跑一个作业,**不看 enabled 开关**。
页面上的「立即跑一次」用这个:运维需要能在正式打开开关之前先试一轮。
"""
job = db.query(AutomationJob).filter(AutomationJob.key == key).first()
spec = _REGISTRY.get(key)
if job is None or spec is None:
raise KeyError(key)
return _run_one(db, job, spec)
def _loop() -> None:
db = SessionLocal()
try:
ensure_jobs(db)
except Exception: # noqa: BLE001 初始化失败不应让线程静默退出
logger.exception("自动化作业初始化失败")
finally:
db.close()
logger.info("自动化调度器已启动,tick={}s", TICK_SECONDS)
while not _stop_event.wait(TICK_SECONDS):
db = SessionLocal()
try:
run_due_jobs(db)
except Exception: # noqa: BLE001 单轮异常不能让调度线程死掉
db.rollback()
logger.exception("自动化调度轮询异常")
finally:
db.close()
logger.info("自动化调度器已停止")
def start_scheduler() -> bool:
"""启动调度线程。返回是否真的启动了(未开启总开关时返回 False)。"""
global _thread
if not automation_enabled():
logger.info("AUTOMATION_ENABLED 未开启,自动化调度器不启动")
return False
with _lock:
if _thread is not None and _thread.is_alive():
return False
_stop_event.clear()
_thread = threading.Thread(
target=_loop, name="automation-scheduler", daemon=True
)
_thread.start()
return True
def stop_scheduler() -> None:
global _thread
with _lock:
if _thread is None:
return
_stop_event.set()
_thread.join(timeout=5)
_thread = None
def scheduler_status() -> dict[str, Any]:
return {
"enabled": automation_enabled(),
"running": _thread is not None and _thread.is_alive(),
"tick_seconds": TICK_SECONDS,
"registered": sorted(_REGISTRY),
}