feat(automation): CK 自动巡检(P1a),只检测不改 CK
后台原本靠人工逐批点「CK 检测」发现失效账号,1.2 万个账号根本盯不过来。
P1a 把「哪些账号失效了」自动化:自动挑久未检测的账号、调现成检测接口、
把结果写回 ck_check_*。**本作业不替换任何 CK**,重登仍由人工点,符合规划里
「先做只读/只提示,确认无误再做自动动作」的渐进上线原则。
实现:
- 迁移 20261002_0037:automation_jobs / automation_runs 两张表,
login_tasks 加 relogin_attempts / relogin_next_at / relogin_last_error
(为 P1b 备好),accounts 加 auto_managed(特殊账号不参与自动化);
- services/automation/scheduler.py:进程内调度线程,60s tick,状态全部落库
(重启不丢调度进度、不重复跑轮);AUTOMATION_ENABLED 默认关闭,作业开关
也默认关闭;ensure_jobs 只补缺失行、不覆盖运维改过的配置;
作业异常只记 failed,不拖垮调度线程;单容器 --workers 1,无需分布式锁;
- services/automation/jobs/ck_health.py:选行口径与人工检测一致
(cookie_holding_filter:持有 CK 且无在途尝试),叠加久未检测、
auto_managed、batch_size,按 ck_checked_at 升序(最久没查的优先);
只写 ck_check_status / ck_check_result / ck_checked_at;
- routers/automation.py:GET /jobs、PATCH /jobs/{key}、POST /jobs/{key}/run、
GET /runs;新增权限点 automation:view(运营可看)与 automation:manage
(仅超管),改配置与手动触发都写审计;
- 顺带补 POST /api/login/batch 的审计(此前批次是谁发起、多少账号无据可查);
- 前端 /automation 页:作业卡片(开关/周期/单轮条数/并发)、运行历史、
「立即跑一次」;总开关未打开、调度线程未运行时给明确提示。
验证:
- 22 条新测试覆盖调度器行为(只跑到点的、异常不抛出、下轮时间、配置不被重置)、
选行条件(在途尝试/auto_managed/间隔/batch_size)、接口与权限,其中一条
专门断言巡检后 cookie/status/message/attempt_status 一字未动;
- 全量 257 条通过;前端 tsc 与构建通过;
- 端到端用 3 个真实账号跑真实斗鱼接口:巡检结果与直接调 check_douyu_cookie
逐条一致(3/3 valid),CK 与尝试状态全程未变。
顺带修掉一处会炸生产的坑:DB 时间列读出来是 naive UTC,直接与
datetime.now(UTC) 比较会 TypeError,现统一走 _as_utc() 补时区。
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
"""自动化调度:作业注册、调度线程与各作业实现。
|
||||
|
||||
登记作业请放在 ``register_default_jobs()``,由 ``main.py`` 在启动时调用一次,
|
||||
避免 import 顺序影响注册结果。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .jobs.ck_health import JOB_KEY as CK_HEALTH_KEY
|
||||
from .jobs.ck_health import run_ck_health
|
||||
from .scheduler import (
|
||||
JobSpec,
|
||||
automation_enabled,
|
||||
ensure_jobs,
|
||||
register_job,
|
||||
registered_jobs,
|
||||
reset_registry,
|
||||
run_due_jobs,
|
||||
run_job_now,
|
||||
scheduler_status,
|
||||
start_scheduler,
|
||||
stop_scheduler,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CK_HEALTH_KEY",
|
||||
"JobSpec",
|
||||
"automation_enabled",
|
||||
"ensure_jobs",
|
||||
"register_default_jobs",
|
||||
"register_job",
|
||||
"registered_jobs",
|
||||
"reset_registry",
|
||||
"run_ck_health",
|
||||
"run_due_jobs",
|
||||
"run_job_now",
|
||||
"scheduler_status",
|
||||
"start_scheduler",
|
||||
"stop_scheduler",
|
||||
]
|
||||
|
||||
|
||||
def register_default_jobs() -> None:
|
||||
"""登记内置作业(幂等)。"""
|
||||
register_job(
|
||||
JobSpec(
|
||||
key=CK_HEALTH_KEY,
|
||||
title="CK 自动巡检",
|
||||
# 6 小时一轮;全库 12,881 条按此反推 batch_size=200(见规划附录 C)。
|
||||
interval_seconds=6 * 3600,
|
||||
batch_size=200,
|
||||
concurrency=5,
|
||||
run=run_ck_health,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""自动化作业实现。"""
|
||||
@@ -0,0 +1,125 @@
|
||||
"""CK 巡检作业(P1a)。
|
||||
|
||||
只做一件事:把「持有 CK 但久未检测」的账号挑出来,调现成的
|
||||
``check_douyu_cookie()`` 检测,把结果写回 ``ck_check_*``。
|
||||
|
||||
**本作业不修改 CK 本身**:不写 ``cookie`` / ``status`` / ``message``,也不碰
|
||||
``attempt_*``。P1a 上线后人工照旧可以点检测和重登,自动重登是 P1b 的事。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....models import Account, AutomationJob, LoginTask
|
||||
from ...cookie_check_service import check_douyu_cookie
|
||||
from ...douyu_service import cookie_operable_filter
|
||||
|
||||
JOB_KEY = "ck_health"
|
||||
FLUSH_EVERY = 25
|
||||
|
||||
|
||||
def _select_tasks(db: Session, job: AutomationJob, now: datetime) -> list[LoginTask]:
|
||||
"""挑本轮要检测的记录。
|
||||
|
||||
口径与人工点检测一致(``cookie_operable_filter``:持有 CK 且没有在途尝试),
|
||||
再加三条:距上次检测超过作业间隔、账号允许自动管理、最久没查的优先。
|
||||
"""
|
||||
cutoff = now - timedelta(seconds=max(int(job.interval_seconds or 0), 60))
|
||||
return (
|
||||
db.query(LoginTask)
|
||||
.join(Account, LoginTask.account_id == Account.id)
|
||||
.filter(cookie_operable_filter())
|
||||
.filter(Account.auto_managed.is_(True))
|
||||
.filter(
|
||||
or_(
|
||||
LoginTask.ck_checked_at.is_(None),
|
||||
LoginTask.ck_checked_at < cutoff,
|
||||
)
|
||||
)
|
||||
.order_by(LoginTask.ck_checked_at.asc().nullsfirst(), LoginTask.id.asc())
|
||||
.limit(max(int(job.batch_size or 0), 1))
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def _detect(task: LoginTask) -> dict[str, Any]:
|
||||
try:
|
||||
return check_douyu_cookie(task.cookie or "")
|
||||
except Exception as exc: # noqa: BLE001 单个账号失败不影响整轮
|
||||
return {
|
||||
"valid": False,
|
||||
"message": f"检测异常: {exc}",
|
||||
"fish_ball": None,
|
||||
"nickname": None,
|
||||
"level": None,
|
||||
"checked_at": datetime.now(UTC),
|
||||
}
|
||||
|
||||
|
||||
def run_ck_health(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
tasks = _select_tasks(db, job, now)
|
||||
if not tasks:
|
||||
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
|
||||
|
||||
results: dict[int, dict[str, Any]] = {}
|
||||
concurrency = max(min(int(job.concurrency or 1), 16), 1)
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = {pool.submit(_detect, task): task for task in tasks}
|
||||
for future in as_completed(futures):
|
||||
task = futures[future]
|
||||
results[task.id] = future.result()
|
||||
|
||||
# 只写检测结果三件套;cookie / status / attempt_* 一律不动。
|
||||
valid = invalid = 0
|
||||
for index, task in enumerate(tasks, start=1):
|
||||
result = results.get(task.id)
|
||||
if result is None:
|
||||
continue
|
||||
is_valid = bool(result.get("valid"))
|
||||
if is_valid:
|
||||
valid += 1
|
||||
else:
|
||||
invalid += 1
|
||||
task.ck_check_status = "valid" if is_valid else "invalid"
|
||||
task.ck_check_result = {
|
||||
"fish_ball": result.get("fish_ball"),
|
||||
"nickname": result.get("nickname"),
|
||||
"level": result.get("level"),
|
||||
"message": result.get("message", ""),
|
||||
}
|
||||
task.ck_checked_at = now
|
||||
if index % FLUSH_EVERY == 0:
|
||||
db.commit()
|
||||
db.commit()
|
||||
|
||||
# 剩余待检数量只用于提示,不参与调度。
|
||||
remaining = (
|
||||
db.query(LoginTask)
|
||||
.join(Account, LoginTask.account_id == Account.id)
|
||||
.filter(cookie_operable_filter())
|
||||
.filter(Account.auto_managed.is_(True))
|
||||
.filter(
|
||||
or_(
|
||||
LoginTask.ck_checked_at.is_(None),
|
||||
LoginTask.ck_checked_at
|
||||
< now - timedelta(seconds=max(int(job.interval_seconds or 0), 60)),
|
||||
)
|
||||
)
|
||||
.count()
|
||||
)
|
||||
message = f"本轮检测 {len(tasks)} 条,剩余待检 {remaining} 条"
|
||||
return {
|
||||
"scanned": len(tasks),
|
||||
"valid": valid,
|
||||
"invalid": invalid,
|
||||
"skipped": 0,
|
||||
"message": message,
|
||||
"detail": {"remaining": remaining, "concurrency": concurrency},
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
"""进程内自动化调度器。
|
||||
|
||||
背景:后台的日常动作原本全靠人工点(判断 CK 失效、重登、收集资料、记台账)。
|
||||
自动化按「渐进上线」推进,因此:
|
||||
|
||||
- 调度器总开关 ``AUTOMATION_ENABLED`` **默认关闭**;
|
||||
- 每个作业自己还有开关,**默认关闭**,在 ``/automation`` 页面打开;
|
||||
- 作业状态全部落库(``automation_jobs`` / ``automation_runs``),进程重启不丢
|
||||
调度进度,也不会因为重启而重复跑一轮。
|
||||
|
||||
部署是单容器 ``--workers 1``,所以一个进程内线程就够,不需要分布式锁。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...database import SessionLocal
|
||||
from ...models import AutomationJob, AutomationRun
|
||||
|
||||
# 调度轮询间隔:作业间隔都以它为单位对齐(默认 6h 的作业不会差这一分钟)。
|
||||
TICK_SECONDS = 60
|
||||
|
||||
TRUTHY = {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JobSpec:
|
||||
"""一个可调度的作业。``run`` 接收会话与作业行,返回本轮计数。"""
|
||||
|
||||
key: str
|
||||
title: str
|
||||
interval_seconds: int
|
||||
batch_size: int
|
||||
concurrency: int
|
||||
run: Callable[[Session, AutomationJob], dict[str, Any]]
|
||||
|
||||
|
||||
_REGISTRY: dict[str, JobSpec] = {}
|
||||
_thread: threading.Thread | None = None
|
||||
_stop_event = threading.Event()
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def register_job(spec: JobSpec) -> None:
|
||||
_REGISTRY[spec.key] = spec
|
||||
|
||||
|
||||
def registered_jobs() -> dict[str, JobSpec]:
|
||||
return dict(_REGISTRY)
|
||||
|
||||
|
||||
def reset_registry() -> None:
|
||||
"""清空作业注册表(测试隔离用;正常启动只调用 register_default_jobs)。"""
|
||||
_REGISTRY.clear()
|
||||
|
||||
|
||||
def automation_enabled() -> bool:
|
||||
return os.getenv("AUTOMATION_ENABLED", "").strip().lower() in TRUTHY
|
||||
|
||||
|
||||
def ensure_jobs(db: Session) -> None:
|
||||
"""把注册表里的作业补进 ``automation_jobs``(幂等)。
|
||||
|
||||
只补缺失的行,**不覆盖已有配置** —— 运维在页面上改过的开关、间隔、批量
|
||||
不能因为重启被重置回默认值。
|
||||
"""
|
||||
existing = {row.key for row in db.query(AutomationJob.key).all()}
|
||||
for spec in _REGISTRY.values():
|
||||
if spec.key in existing:
|
||||
continue
|
||||
db.add(
|
||||
AutomationJob(
|
||||
key=spec.key,
|
||||
title=spec.title,
|
||||
enabled=False,
|
||||
interval_seconds=spec.interval_seconds,
|
||||
batch_size=spec.batch_size,
|
||||
concurrency=spec.concurrency,
|
||||
config=None,
|
||||
next_run_at=None,
|
||||
last_status="",
|
||||
last_message="",
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _as_utc(value: datetime | None) -> datetime | None:
|
||||
"""DB 时间列存的是 naive UTC,读出来要补时区标记才能跟 now 比较。"""
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=UTC)
|
||||
return value.astimezone(UTC)
|
||||
|
||||
|
||||
def _due(job: AutomationJob, now: datetime) -> bool:
|
||||
"""到点判定:没排过时间视为到点(刚打开开关就会跑一轮)。"""
|
||||
next_run_at = _as_utc(job.next_run_at)
|
||||
return next_run_at is None or next_run_at <= now
|
||||
|
||||
|
||||
def run_due_jobs(db: Session, *, now: datetime | None = None) -> list[dict[str, Any]]:
|
||||
"""跑一轮「到点的作业」,返回每轮结果。
|
||||
|
||||
抽成普通函数是为了可测:测试直接调用即可,不必启动线程或等 60 秒。
|
||||
作业串行执行 —— 目前只有 CK 巡检,单轮约 20~40 秒,远小于 tick,串行
|
||||
反而省掉了并发调度的复杂度。
|
||||
"""
|
||||
now = now or datetime.now(UTC)
|
||||
outcomes: list[dict[str, Any]] = []
|
||||
jobs = (
|
||||
db.query(AutomationJob)
|
||||
.filter(AutomationJob.enabled.is_(True))
|
||||
.order_by(AutomationJob.key.asc())
|
||||
.all()
|
||||
)
|
||||
for job in jobs:
|
||||
spec = _REGISTRY.get(job.key)
|
||||
if spec is None or not _due(job, now):
|
||||
continue
|
||||
outcomes.append(_run_one(db, job, spec))
|
||||
|
||||
# 到期但没注册实现的作业(例如降级回滚后残留的行)只留日志,不静默消失。
|
||||
for job in jobs:
|
||||
if job.key not in _REGISTRY:
|
||||
logger.warning("自动化作业 {} 没有注册实现,已跳过", job.key)
|
||||
return outcomes
|
||||
|
||||
|
||||
def _run_one(db: Session, job: AutomationJob, spec: JobSpec) -> dict[str, Any]:
|
||||
started = datetime.now(UTC)
|
||||
run = AutomationRun(job_key=job.key, started_at=started, status="running")
|
||||
db.add(run)
|
||||
db.commit()
|
||||
|
||||
counters: dict[str, Any] = {}
|
||||
status = "success"
|
||||
message = ""
|
||||
try:
|
||||
counters = spec.run(db, job) or {}
|
||||
message = str(counters.pop("message", "") or "")
|
||||
except Exception as exc: # noqa: BLE001 作业边界要兜住任何异常,不能拖垮调度线程
|
||||
db.rollback()
|
||||
status = "failed"
|
||||
message = f"{type(exc).__name__}: {exc}"[:500]
|
||||
logger.exception("自动化作业 {} 执行失败", job.key)
|
||||
|
||||
finished = datetime.now(UTC)
|
||||
run.finished_at = finished
|
||||
run.status = status
|
||||
run.message = message
|
||||
run.scanned = int(counters.get("scanned", 0) or 0)
|
||||
run.valid = int(counters.get("valid", 0) or 0)
|
||||
run.invalid = int(counters.get("invalid", 0) or 0)
|
||||
run.relogin_started = int(counters.get("relogin_started", 0) or 0)
|
||||
run.relogin_success = int(counters.get("relogin_success", 0) or 0)
|
||||
run.skipped = int(counters.get("skipped", 0) or 0)
|
||||
run.detail = counters.get("detail") or None
|
||||
|
||||
job.last_run_at = finished
|
||||
job.next_run_at = finished + timedelta(
|
||||
seconds=max(int(job.interval_seconds or spec.interval_seconds), 60)
|
||||
)
|
||||
job.last_status = status
|
||||
job.last_message = message
|
||||
job.updated_at = finished
|
||||
db.commit()
|
||||
|
||||
logger.info(
|
||||
"自动化作业 {} 完成:status={} scanned={} valid={} invalid={} skipped={}",
|
||||
job.key,
|
||||
status,
|
||||
run.scanned,
|
||||
run.valid,
|
||||
run.invalid,
|
||||
run.skipped,
|
||||
)
|
||||
return {
|
||||
"job_key": job.key,
|
||||
"status": status,
|
||||
"message": message,
|
||||
"scanned": run.scanned,
|
||||
"valid": run.valid,
|
||||
"invalid": run.invalid,
|
||||
"skipped": run.skipped,
|
||||
}
|
||||
|
||||
|
||||
def run_job_now(db: Session, key: str) -> dict[str, Any]:
|
||||
"""立刻跑一个作业,**不看 enabled 开关**。
|
||||
|
||||
页面上的「立即跑一次」用这个:运维需要能在正式打开开关之前先试一轮。
|
||||
"""
|
||||
job = db.query(AutomationJob).filter(AutomationJob.key == key).first()
|
||||
spec = _REGISTRY.get(key)
|
||||
if job is None or spec is None:
|
||||
raise KeyError(key)
|
||||
return _run_one(db, job, spec)
|
||||
|
||||
|
||||
def _loop() -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ensure_jobs(db)
|
||||
except Exception: # noqa: BLE001 初始化失败不应让线程静默退出
|
||||
logger.exception("自动化作业初始化失败")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
logger.info("自动化调度器已启动,tick={}s", TICK_SECONDS)
|
||||
while not _stop_event.wait(TICK_SECONDS):
|
||||
db = SessionLocal()
|
||||
try:
|
||||
run_due_jobs(db)
|
||||
except Exception: # noqa: BLE001 单轮异常不能让调度线程死掉
|
||||
db.rollback()
|
||||
logger.exception("自动化调度轮询异常")
|
||||
finally:
|
||||
db.close()
|
||||
logger.info("自动化调度器已停止")
|
||||
|
||||
|
||||
def start_scheduler() -> bool:
|
||||
"""启动调度线程。返回是否真的启动了(未开启总开关时返回 False)。"""
|
||||
global _thread
|
||||
if not automation_enabled():
|
||||
logger.info("AUTOMATION_ENABLED 未开启,自动化调度器不启动")
|
||||
return False
|
||||
with _lock:
|
||||
if _thread is not None and _thread.is_alive():
|
||||
return False
|
||||
_stop_event.clear()
|
||||
_thread = threading.Thread(
|
||||
target=_loop, name="automation-scheduler", daemon=True
|
||||
)
|
||||
_thread.start()
|
||||
return True
|
||||
|
||||
|
||||
def stop_scheduler() -> None:
|
||||
global _thread
|
||||
with _lock:
|
||||
if _thread is None:
|
||||
return
|
||||
_stop_event.set()
|
||||
_thread.join(timeout=5)
|
||||
_thread = None
|
||||
|
||||
|
||||
def scheduler_status() -> dict[str, Any]:
|
||||
return {
|
||||
"enabled": automation_enabled(),
|
||||
"running": _thread is not None and _thread.is_alive(),
|
||||
"tick_seconds": TICK_SECONDS,
|
||||
"registered": sorted(_REGISTRY),
|
||||
}
|
||||
Reference in New Issue
Block a user