后台原本靠人工逐批点「CK 检测」发现失效账号,1.2 万个账号根本盯不过来。
P1a 把「哪些账号失效了」自动化:自动挑久未检测的账号、调现成检测接口、
把结果写回 ck_check_*。**本作业不替换任何 CK**,重登仍由人工点,符合规划里
「先做只读/只提示,确认无误再做自动动作」的渐进上线原则。
实现:
- 迁移 20261002_0037:automation_jobs / automation_runs 两张表,
login_tasks 加 relogin_attempts / relogin_next_at / relogin_last_error
(为 P1b 备好),accounts 加 auto_managed(特殊账号不参与自动化);
- services/automation/scheduler.py:进程内调度线程,60s tick,状态全部落库
(重启不丢调度进度、不重复跑轮);AUTOMATION_ENABLED 默认关闭,作业开关
也默认关闭;ensure_jobs 只补缺失行、不覆盖运维改过的配置;
作业异常只记 failed,不拖垮调度线程;单容器 --workers 1,无需分布式锁;
- services/automation/jobs/ck_health.py:选行口径与人工检测一致
(cookie_holding_filter:持有 CK 且无在途尝试),叠加久未检测、
auto_managed、batch_size,按 ck_checked_at 升序(最久没查的优先);
只写 ck_check_status / ck_check_result / ck_checked_at;
- routers/automation.py:GET /jobs、PATCH /jobs/{key}、POST /jobs/{key}/run、
GET /runs;新增权限点 automation:view(运营可看)与 automation:manage
(仅超管),改配置与手动触发都写审计;
- 顺带补 POST /api/login/batch 的审计(此前批次是谁发起、多少账号无据可查);
- 前端 /automation 页:作业卡片(开关/周期/单轮条数/并发)、运行历史、
「立即跑一次」;总开关未打开、调度线程未运行时给明确提示。
验证:
- 22 条新测试覆盖调度器行为(只跑到点的、异常不抛出、下轮时间、配置不被重置)、
选行条件(在途尝试/auto_managed/间隔/batch_size)、接口与权限,其中一条
专门断言巡检后 cookie/status/message/attempt_status 一字未动;
- 全量 257 条通过;前端 tsc 与构建通过;
- 端到端用 3 个真实账号跑真实斗鱼接口:巡检结果与直接调 check_douyu_cookie
逐条一致(3/3 valid),CK 与尝试状态全程未变。
顺带修掉一处会炸生产的坑:DB 时间列读出来是 naive UTC,直接与
datetime.now(UTC) 比较会 TypeError,现统一走 _as_utc() 补时区。
268 lines
8.4 KiB
Python
268 lines
8.4 KiB
Python
"""进程内自动化调度器。
|
||
|
||
背景:后台的日常动作原本全靠人工点(判断 CK 失效、重登、收集资料、记台账)。
|
||
自动化按「渐进上线」推进,因此:
|
||
|
||
- 调度器总开关 ``AUTOMATION_ENABLED`` **默认关闭**;
|
||
- 每个作业自己还有开关,**默认关闭**,在 ``/automation`` 页面打开;
|
||
- 作业状态全部落库(``automation_jobs`` / ``automation_runs``),进程重启不丢
|
||
调度进度,也不会因为重启而重复跑一轮。
|
||
|
||
部署是单容器 ``--workers 1``,所以一个进程内线程就够,不需要分布式锁。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import threading
|
||
from collections.abc import Callable
|
||
from dataclasses import dataclass
|
||
from datetime import UTC, datetime, timedelta
|
||
from typing import Any
|
||
|
||
from loguru import logger
|
||
from sqlalchemy.orm import Session
|
||
|
||
from ...database import SessionLocal
|
||
from ...models import AutomationJob, AutomationRun
|
||
|
||
# 调度轮询间隔:作业间隔都以它为单位对齐(默认 6h 的作业不会差这一分钟)。
|
||
TICK_SECONDS = 60
|
||
|
||
TRUTHY = {"1", "true", "yes", "on"}
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class JobSpec:
|
||
"""一个可调度的作业。``run`` 接收会话与作业行,返回本轮计数。"""
|
||
|
||
key: str
|
||
title: str
|
||
interval_seconds: int
|
||
batch_size: int
|
||
concurrency: int
|
||
run: Callable[[Session, AutomationJob], dict[str, Any]]
|
||
|
||
|
||
_REGISTRY: dict[str, JobSpec] = {}
|
||
_thread: threading.Thread | None = None
|
||
_stop_event = threading.Event()
|
||
_lock = threading.Lock()
|
||
|
||
|
||
def register_job(spec: JobSpec) -> None:
|
||
_REGISTRY[spec.key] = spec
|
||
|
||
|
||
def registered_jobs() -> dict[str, JobSpec]:
|
||
return dict(_REGISTRY)
|
||
|
||
|
||
def reset_registry() -> None:
|
||
"""清空作业注册表(测试隔离用;正常启动只调用 register_default_jobs)。"""
|
||
_REGISTRY.clear()
|
||
|
||
|
||
def automation_enabled() -> bool:
|
||
return os.getenv("AUTOMATION_ENABLED", "").strip().lower() in TRUTHY
|
||
|
||
|
||
def ensure_jobs(db: Session) -> None:
|
||
"""把注册表里的作业补进 ``automation_jobs``(幂等)。
|
||
|
||
只补缺失的行,**不覆盖已有配置** —— 运维在页面上改过的开关、间隔、批量
|
||
不能因为重启被重置回默认值。
|
||
"""
|
||
existing = {row.key for row in db.query(AutomationJob.key).all()}
|
||
for spec in _REGISTRY.values():
|
||
if spec.key in existing:
|
||
continue
|
||
db.add(
|
||
AutomationJob(
|
||
key=spec.key,
|
||
title=spec.title,
|
||
enabled=False,
|
||
interval_seconds=spec.interval_seconds,
|
||
batch_size=spec.batch_size,
|
||
concurrency=spec.concurrency,
|
||
config=None,
|
||
next_run_at=None,
|
||
last_status="",
|
||
last_message="",
|
||
updated_at=datetime.now(UTC),
|
||
)
|
||
)
|
||
db.commit()
|
||
|
||
|
||
def _as_utc(value: datetime | None) -> datetime | None:
|
||
"""DB 时间列存的是 naive UTC,读出来要补时区标记才能跟 now 比较。"""
|
||
if value is None:
|
||
return None
|
||
if value.tzinfo is None:
|
||
return value.replace(tzinfo=UTC)
|
||
return value.astimezone(UTC)
|
||
|
||
|
||
def _due(job: AutomationJob, now: datetime) -> bool:
|
||
"""到点判定:没排过时间视为到点(刚打开开关就会跑一轮)。"""
|
||
next_run_at = _as_utc(job.next_run_at)
|
||
return next_run_at is None or next_run_at <= now
|
||
|
||
|
||
def run_due_jobs(db: Session, *, now: datetime | None = None) -> list[dict[str, Any]]:
|
||
"""跑一轮「到点的作业」,返回每轮结果。
|
||
|
||
抽成普通函数是为了可测:测试直接调用即可,不必启动线程或等 60 秒。
|
||
作业串行执行 —— 目前只有 CK 巡检,单轮约 20~40 秒,远小于 tick,串行
|
||
反而省掉了并发调度的复杂度。
|
||
"""
|
||
now = now or datetime.now(UTC)
|
||
outcomes: list[dict[str, Any]] = []
|
||
jobs = (
|
||
db.query(AutomationJob)
|
||
.filter(AutomationJob.enabled.is_(True))
|
||
.order_by(AutomationJob.key.asc())
|
||
.all()
|
||
)
|
||
for job in jobs:
|
||
spec = _REGISTRY.get(job.key)
|
||
if spec is None or not _due(job, now):
|
||
continue
|
||
outcomes.append(_run_one(db, job, spec))
|
||
|
||
# 到期但没注册实现的作业(例如降级回滚后残留的行)只留日志,不静默消失。
|
||
for job in jobs:
|
||
if job.key not in _REGISTRY:
|
||
logger.warning("自动化作业 {} 没有注册实现,已跳过", job.key)
|
||
return outcomes
|
||
|
||
|
||
def _run_one(db: Session, job: AutomationJob, spec: JobSpec) -> dict[str, Any]:
|
||
started = datetime.now(UTC)
|
||
run = AutomationRun(job_key=job.key, started_at=started, status="running")
|
||
db.add(run)
|
||
db.commit()
|
||
|
||
counters: dict[str, Any] = {}
|
||
status = "success"
|
||
message = ""
|
||
try:
|
||
counters = spec.run(db, job) or {}
|
||
message = str(counters.pop("message", "") or "")
|
||
except Exception as exc: # noqa: BLE001 作业边界要兜住任何异常,不能拖垮调度线程
|
||
db.rollback()
|
||
status = "failed"
|
||
message = f"{type(exc).__name__}: {exc}"[:500]
|
||
logger.exception("自动化作业 {} 执行失败", job.key)
|
||
|
||
finished = datetime.now(UTC)
|
||
run.finished_at = finished
|
||
run.status = status
|
||
run.message = message
|
||
run.scanned = int(counters.get("scanned", 0) or 0)
|
||
run.valid = int(counters.get("valid", 0) or 0)
|
||
run.invalid = int(counters.get("invalid", 0) or 0)
|
||
run.relogin_started = int(counters.get("relogin_started", 0) or 0)
|
||
run.relogin_success = int(counters.get("relogin_success", 0) or 0)
|
||
run.skipped = int(counters.get("skipped", 0) or 0)
|
||
run.detail = counters.get("detail") or None
|
||
|
||
job.last_run_at = finished
|
||
job.next_run_at = finished + timedelta(
|
||
seconds=max(int(job.interval_seconds or spec.interval_seconds), 60)
|
||
)
|
||
job.last_status = status
|
||
job.last_message = message
|
||
job.updated_at = finished
|
||
db.commit()
|
||
|
||
logger.info(
|
||
"自动化作业 {} 完成:status={} scanned={} valid={} invalid={} skipped={}",
|
||
job.key,
|
||
status,
|
||
run.scanned,
|
||
run.valid,
|
||
run.invalid,
|
||
run.skipped,
|
||
)
|
||
return {
|
||
"job_key": job.key,
|
||
"status": status,
|
||
"message": message,
|
||
"scanned": run.scanned,
|
||
"valid": run.valid,
|
||
"invalid": run.invalid,
|
||
"skipped": run.skipped,
|
||
}
|
||
|
||
|
||
def run_job_now(db: Session, key: str) -> dict[str, Any]:
|
||
"""立刻跑一个作业,**不看 enabled 开关**。
|
||
|
||
页面上的「立即跑一次」用这个:运维需要能在正式打开开关之前先试一轮。
|
||
"""
|
||
job = db.query(AutomationJob).filter(AutomationJob.key == key).first()
|
||
spec = _REGISTRY.get(key)
|
||
if job is None or spec is None:
|
||
raise KeyError(key)
|
||
return _run_one(db, job, spec)
|
||
|
||
|
||
def _loop() -> None:
|
||
db = SessionLocal()
|
||
try:
|
||
ensure_jobs(db)
|
||
except Exception: # noqa: BLE001 初始化失败不应让线程静默退出
|
||
logger.exception("自动化作业初始化失败")
|
||
finally:
|
||
db.close()
|
||
|
||
logger.info("自动化调度器已启动,tick={}s", TICK_SECONDS)
|
||
while not _stop_event.wait(TICK_SECONDS):
|
||
db = SessionLocal()
|
||
try:
|
||
run_due_jobs(db)
|
||
except Exception: # noqa: BLE001 单轮异常不能让调度线程死掉
|
||
db.rollback()
|
||
logger.exception("自动化调度轮询异常")
|
||
finally:
|
||
db.close()
|
||
logger.info("自动化调度器已停止")
|
||
|
||
|
||
def start_scheduler() -> bool:
|
||
"""启动调度线程。返回是否真的启动了(未开启总开关时返回 False)。"""
|
||
global _thread
|
||
if not automation_enabled():
|
||
logger.info("AUTOMATION_ENABLED 未开启,自动化调度器不启动")
|
||
return False
|
||
with _lock:
|
||
if _thread is not None and _thread.is_alive():
|
||
return False
|
||
_stop_event.clear()
|
||
_thread = threading.Thread(
|
||
target=_loop, name="automation-scheduler", daemon=True
|
||
)
|
||
_thread.start()
|
||
return True
|
||
|
||
|
||
def stop_scheduler() -> None:
|
||
global _thread
|
||
with _lock:
|
||
if _thread is None:
|
||
return
|
||
_stop_event.set()
|
||
_thread.join(timeout=5)
|
||
_thread = None
|
||
|
||
|
||
def scheduler_status() -> dict[str, Any]:
|
||
return {
|
||
"enabled": automation_enabled(),
|
||
"running": _thread is not None and _thread.is_alive(),
|
||
"tick_seconds": TICK_SECONDS,
|
||
"registered": sorted(_REGISTRY),
|
||
}
|