yml2213andCursor
39388c56b5
客服默认不再有 cookie:operate；自动巡检菜单改按 automation:view 显示。 Co-authored-by: Cursor <cursoragent@cursor.com>
1021 lines
37 KiB
Python
1021 lines
37 KiB
Python
"""P1a 回归：CK 自动巡检。
|
||
|
||
两块约束必须一起守住：
|
||
|
||
1. **调度器行为**：只跑到点的、打开开关的作业；作业抛异常只记 failed，不能
|
||
把调度线程带崩；跑完排下一轮时间；已有配置不能被重启重置。
|
||
2. **本作业绝不改 CK**：P1a 的定位是「只检测、只标记」，所以巡检后
|
||
**账号的** ``cookie`` / ``status`` / ``relogin_*`` 以及登录尝试历史的
|
||
``status`` / ``message`` / ``attempt_*`` 必须一字未动 ——
|
||
线上全靠这条保证「自动化上线了但人工流程照旧」，回归时一旦有人顺手写了
|
||
``cookie`` 或 ``status``，这里要立刻红。
|
||
|
||
> CK 与健康/重登状态自 2026-10-04 起是 **``accounts`` 的列**
|
||
> （见 ``docs/设计-账号全生命周期.md`` §3.3）；``login_tasks`` 退化为纯登录
|
||
> 尝试历史，它的 ``cookie`` / ``ck_check_*`` / ``relogin_*`` 旧列留作回退、
|
||
> 新代码不再读写，测试也不许再往上面写值。
|
||
"""
|
||
|
||
import threading
|
||
import time
|
||
from datetime import UTC, datetime, timedelta
|
||
|
||
import pytest
|
||
from fastapi import HTTPException
|
||
|
||
from tests.support.db import get_test_engine, make_test_session
|
||
from web.backend.deps import require_permission
|
||
from web.backend.models import Account, AutomationJob, AutomationRun, LoginTask, User
|
||
from web.backend.services.automation import (
|
||
JobSpec,
|
||
register_default_jobs,
|
||
register_job,
|
||
reset_registry,
|
||
run_due_jobs,
|
||
run_job_now,
|
||
scheduler,
|
||
)
|
||
from web.backend.services.automation.jobs import ck_health
|
||
|
||
|
||
class _Base:
|
||
def setup_method(self):
|
||
self.engine = get_test_engine()
|
||
self.session = make_test_session(self.engine)
|
||
self.admin = User(
|
||
username="admin", password_hash="h", role="super_admin", is_active=True
|
||
)
|
||
self.support = User(
|
||
username="kf", password_hash="h", role="support", is_active=True
|
||
)
|
||
self.session.add_all([self.admin, self.support])
|
||
self.session.commit()
|
||
# 注册表是模块级的，不重置会被上一个用例注册的探针作业污染。
|
||
reset_registry()
|
||
register_default_jobs()
|
||
|
||
def teardown_method(self):
|
||
self.session.close()
|
||
|
||
def make_account(
|
||
self,
|
||
name="acc",
|
||
*,
|
||
cookie="acf_uid=1",
|
||
status="active",
|
||
auto_managed=True,
|
||
assigned_to=None,
|
||
ck_check_status="",
|
||
ck_check_result=None,
|
||
ck_checked_at=None,
|
||
relogin_attempts=0,
|
||
relogin_next_at=None,
|
||
relogin_last_error="",
|
||
relogin_failure_kind="",
|
||
):
|
||
"""建一个账号。**CK 与健康/重登状态都是账号自己的列。**
|
||
|
||
默认 ``cookie="acf_uid=1"``（与迁移前 ``make_task`` 的默认一致）：
|
||
巡检的候选口径是「账号有 CK」，所以不显式给 cookie
|
||
的账号本来就不该进队列 —— 要造这种账号就传 ``cookie=""``。
|
||
"""
|
||
account = Account(
|
||
username=name,
|
||
password="p",
|
||
email=f"{name}@example.com",
|
||
email_password="ep",
|
||
assigned_to=assigned_to,
|
||
auto_managed=auto_managed,
|
||
cookie=cookie,
|
||
status=status,
|
||
ck_check_status=ck_check_status,
|
||
ck_check_result=ck_check_result,
|
||
ck_checked_at=ck_checked_at,
|
||
relogin_attempts=relogin_attempts,
|
||
relogin_next_at=relogin_next_at,
|
||
relogin_last_error=relogin_last_error,
|
||
relogin_failure_kind=relogin_failure_kind,
|
||
)
|
||
self.session.add(account)
|
||
self.session.commit()
|
||
return account
|
||
|
||
def make_attempt(
|
||
self,
|
||
account,
|
||
*,
|
||
status="success",
|
||
message="",
|
||
attempt_status="",
|
||
attempt_message="",
|
||
attempt_batch_id="",
|
||
attempt_at=None,
|
||
):
|
||
"""建一条登录尝试历史（``LoginTask`` 已退化为纯尝试流水）。
|
||
|
||
只写它保留下来的字段：``cookie`` / ``ck_check_*`` / ``relogin_*``
|
||
下一次发版就要 drop，测试再往上写值只会掩盖解耦没做干净。
|
||
"""
|
||
attempt = LoginTask(
|
||
batch_id="b1",
|
||
account_id=account.id,
|
||
created_by=self.admin.id,
|
||
status=status,
|
||
message=message,
|
||
attempt_status=attempt_status,
|
||
attempt_message=attempt_message,
|
||
attempt_batch_id=attempt_batch_id,
|
||
attempt_at=attempt_at,
|
||
)
|
||
self.session.add(attempt)
|
||
self.session.commit()
|
||
return attempt
|
||
|
||
|
||
class TestCkHealthSelection(_Base):
|
||
def _selected_ids(self, job):
|
||
"""返回被选中**账号**的 id（解耦后按账号选行，不再按登录记录选）。"""
|
||
return [
|
||
a.id
|
||
for a in ck_health._select_accounts(self.session, job, datetime.now(UTC))
|
||
]
|
||
|
||
def test_picks_only_accounts_that_still_hold_cookie(self):
|
||
job = self._job()
|
||
with_cookie = self.make_account("a")
|
||
self.make_account("b", cookie="")
|
||
assert self._selected_ids(job) == [with_cookie.id]
|
||
|
||
def test_skips_accounts_with_inflight_attempt(self):
|
||
job = self._job()
|
||
idle = self.make_account("a")
|
||
running = self.make_account("b")
|
||
self.make_attempt(running, attempt_status="running")
|
||
pending = self.make_account("c")
|
||
self.make_attempt(pending, attempt_status="pending")
|
||
assert self._selected_ids(job) == [idle.id]
|
||
|
||
def test_skips_accounts_opted_out_of_automation(self):
|
||
job = self._job()
|
||
managed = self.make_account("a")
|
||
self.make_account("b", auto_managed=False)
|
||
assert self._selected_ids(job) == [managed.id]
|
||
|
||
def test_account_without_login_history_is_selectable(self):
|
||
"""回归：解耦前选行是 ``LoginTask JOIN Account``，从没登录过的账号永远进不了队列。
|
||
|
||
CK 现在挂在账号上，选行只按账号判定 —— 没有尝试历史的账号也必须被巡检。
|
||
"""
|
||
job = self._job()
|
||
account = self.make_account("only-account")
|
||
assert self.session.query(LoginTask).count() == 0
|
||
assert self._selected_ids(job) == [account.id]
|
||
|
||
def test_respects_interval_and_prefers_stalest(self):
|
||
job = self._job()
|
||
fresh = self.make_account("fresh", ck_checked_at=datetime.now(UTC))
|
||
stale = self.make_account(
|
||
"stale", ck_checked_at=datetime.now(UTC) - timedelta(hours=30)
|
||
)
|
||
never = self.make_account("never", ck_checked_at=None)
|
||
ids = self._selected_ids(job)
|
||
assert fresh.id not in ids
|
||
# 从没检测过的排最前，其次是久未检测的
|
||
assert ids[0] == never.id
|
||
assert stale.id in ids
|
||
|
||
def test_high_value_is_rechecked_sooner_and_sorted_first(self):
|
||
"""第 1、2 档 6 小时一查，并且到点后排在普通账号前面。"""
|
||
job = self._job()
|
||
job.interval_seconds = 24 * 3600
|
||
now = datetime.now(UTC)
|
||
plain_recent = self.make_account(
|
||
"plain-recent", ck_checked_at=now - timedelta(hours=7)
|
||
)
|
||
plain_old = self.make_account(
|
||
"plain-old", ck_checked_at=now - timedelta(hours=30)
|
||
)
|
||
valuable = self.make_account("valuable", ck_checked_at=now - timedelta(hours=7))
|
||
valuable.overview_queried_at = now
|
||
valuable.high_value_owned = []
|
||
valuable.points = 800
|
||
self.session.commit()
|
||
|
||
ids = self._selected_ids(job)
|
||
assert valuable.id in ids
|
||
assert plain_recent.id not in ids
|
||
assert plain_old.id in ids
|
||
assert ids.index(valuable.id) < ids.index(plain_old.id)
|
||
|
||
def test_generated_sql_is_mysql_compatible(self):
|
||
"""回归：选行 SQL 不能出现 MySQL 不认识的 NULLS FIRST。
|
||
|
||
生产是 MySQL、本地测试是 SQLite（SQLite 支持 NULLS FIRST，所以行为测试
|
||
发现不了），这里按 MySQL 方言编译一次断言。
|
||
"""
|
||
from sqlalchemy.dialects import mysql
|
||
|
||
query = ck_health.build_select_query(
|
||
self.session, self._job(), datetime.now(UTC)
|
||
)
|
||
sql = str(query.statement.compile(dialect=mysql.dialect())).upper()
|
||
assert "NULLS FIRST" not in sql and "NULLS LAST" not in sql
|
||
# 「没检测过的优先」要用可移植写法表达
|
||
assert "IS NULL DESC" in sql
|
||
|
||
def test_batch_size_caps_the_round(self):
|
||
job = self._job()
|
||
job.batch_size = 2
|
||
for index in range(5):
|
||
self.make_account(f"a{index}")
|
||
assert len(self._selected_ids(job)) == 2
|
||
|
||
def _job(self):
|
||
return AutomationJob(
|
||
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
|
||
)
|
||
|
||
|
||
class TestCkHealthNeverTouchesCookie(_Base):
|
||
def test_patrol_only_writes_check_fields(self):
|
||
job = self._job()
|
||
# 账号自带 CK、生命周期状态与重登进度；巡检只许动 ck_check_* 三件套。
|
||
account = self.make_account(
|
||
"acc",
|
||
cookie="acf_uid=1",
|
||
status="active",
|
||
relogin_attempts=2,
|
||
relogin_next_at=datetime.now(UTC) + timedelta(hours=1),
|
||
relogin_last_error="上次登录失败",
|
||
relogin_failure_kind="network",
|
||
)
|
||
attempt = self.make_attempt(
|
||
account,
|
||
status="success",
|
||
message="历史登录结果",
|
||
attempt_status="failed",
|
||
attempt_message="历史失败",
|
||
attempt_batch_id="b0",
|
||
)
|
||
before = (
|
||
account.cookie,
|
||
account.status,
|
||
account.relogin_attempts,
|
||
account.relogin_next_at,
|
||
account.relogin_last_error,
|
||
account.relogin_failure_kind,
|
||
)
|
||
before_attempt = (
|
||
attempt.status,
|
||
attempt.message,
|
||
attempt.attempt_status,
|
||
attempt.attempt_message,
|
||
attempt.attempt_batch_id,
|
||
attempt.attempt_at,
|
||
)
|
||
|
||
ck_health.check_douyu_cookie = _fake_check(valid=True)
|
||
try:
|
||
counters = ck_health.run_ck_health(self.session, job)
|
||
finally:
|
||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||
|
||
self.session.refresh(account)
|
||
self.session.refresh(attempt)
|
||
assert counters["scanned"] == 1 and counters["valid"] == 1
|
||
assert account.ck_check_status == "valid"
|
||
assert account.ck_checked_at is not None
|
||
assert account.ck_check_result["nickname"] == "测试"
|
||
# 核心断言：账号的 CK 本体、生命周期状态、重登进度一字未动
|
||
assert (
|
||
account.cookie,
|
||
account.status,
|
||
account.relogin_attempts,
|
||
account.relogin_next_at,
|
||
account.relogin_last_error,
|
||
account.relogin_failure_kind,
|
||
) == before
|
||
# 登录尝试历史同样不是巡检该碰的东西
|
||
assert (
|
||
attempt.status,
|
||
attempt.message,
|
||
attempt.attempt_status,
|
||
attempt.attempt_message,
|
||
attempt.attempt_batch_id,
|
||
attempt.attempt_at,
|
||
) == before_attempt
|
||
|
||
def test_invalid_result_marks_invalid_but_keeps_cookie(self):
|
||
job = self._job()
|
||
account = self.make_account("acc", cookie="acf_uid=1")
|
||
# 必须是鉴权类失败（斗鱼说未登录）才会被标成 invalid
|
||
ck_health.check_douyu_cookie = _fake_check(
|
||
valid=False,
|
||
message="鱼丸接口: 账户登陆异常，请重新登陆；等级接口: 用户未登录",
|
||
)
|
||
try:
|
||
counters = ck_health.run_ck_health(self.session, job)
|
||
finally:
|
||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||
self.session.refresh(account)
|
||
assert counters["invalid"] == 1
|
||
assert account.ck_check_status == "invalid"
|
||
assert account.cookie == "acf_uid=1"
|
||
|
||
def test_detection_exception_does_not_abort_round(self):
|
||
"""单个账号检测抛异常不影响整轮，但也不能把它标成失效。"""
|
||
job = self._job()
|
||
first = self.make_account("a", ck_check_status="valid")
|
||
second = self.make_account("b", ck_check_status="valid")
|
||
ck_health.check_douyu_cookie = _fake_check(raises=True)
|
||
try:
|
||
counters = ck_health.run_ck_health(self.session, job)
|
||
finally:
|
||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||
assert counters["scanned"] == 2
|
||
# 查不到不等于失效：两个账号都保留原结论，只记录这次异常
|
||
assert counters["invalid"] == 0
|
||
assert counters["skipped"] == 2
|
||
for account in (first, second):
|
||
self.session.refresh(account)
|
||
assert account.ck_check_status == "valid"
|
||
assert account.ck_check_result["verdict"] == "error"
|
||
|
||
def _job(self):
|
||
return AutomationJob(
|
||
key=ck_health.JOB_KEY, interval_seconds=6 * 3600, batch_size=200
|
||
)
|
||
|
||
|
||
class TestScheduler(_Base):
|
||
def _register_probe(self, calls):
|
||
def run(db, job):
|
||
calls.append(job.key)
|
||
return {"scanned": 3, "valid": 2, "invalid": 1, "message": "ok"}
|
||
|
||
register_job(
|
||
JobSpec(
|
||
key="probe",
|
||
title="探针",
|
||
interval_seconds=3600,
|
||
batch_size=1,
|
||
concurrency=1,
|
||
run=run,
|
||
)
|
||
)
|
||
|
||
def test_only_enabled_jobs_run(self):
|
||
from web.backend.services.automation import ensure_jobs
|
||
|
||
calls: list[str] = []
|
||
self._register_probe(calls)
|
||
ensure_jobs(self.session)
|
||
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
|
||
{"enabled": True}
|
||
)
|
||
self.session.commit()
|
||
|
||
run_due_jobs(self.session)
|
||
assert calls == ["probe"]
|
||
|
||
calls.clear()
|
||
run_due_jobs(self.session)
|
||
# 上一轮已排了 next_run_at，未到点不再跑
|
||
assert calls == []
|
||
|
||
def test_job_failure_is_recorded_not_raised(self):
|
||
from web.backend.services.automation import ensure_jobs
|
||
|
||
def boom(db, job):
|
||
raise RuntimeError("接口挂了")
|
||
|
||
register_job(
|
||
JobSpec(
|
||
key="boom",
|
||
title="会炸的作业",
|
||
interval_seconds=3600,
|
||
batch_size=1,
|
||
concurrency=1,
|
||
run=boom,
|
||
)
|
||
)
|
||
ensure_jobs(self.session)
|
||
self.session.query(AutomationJob).filter(AutomationJob.key == "boom").update(
|
||
{"enabled": True}
|
||
)
|
||
self.session.commit()
|
||
|
||
outcomes = run_due_jobs(self.session)
|
||
assert outcomes[0]["status"] == "failed"
|
||
run = self.session.query(AutomationRun).one()
|
||
assert run.status == "failed" and "接口挂了" in run.message
|
||
|
||
def test_next_run_follows_tick_and_counters_persist(self):
|
||
from web.backend.services.automation import ensure_jobs
|
||
|
||
calls: list[str] = []
|
||
self._register_probe(calls)
|
||
ensure_jobs(self.session)
|
||
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").update(
|
||
{"enabled": True}
|
||
)
|
||
self.session.commit()
|
||
|
||
run_due_jobs(self.session)
|
||
job = (
|
||
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").one()
|
||
)
|
||
assert job.last_status == "success"
|
||
# 下一轮在一个作业周期之后，与 interval_seconds（每行多久重检）无关
|
||
from web.backend.services.automation.scheduler import JOB_INTERVAL_SECONDS
|
||
|
||
assert _aware(job.next_run_at) - _aware(job.last_run_at) == timedelta(
|
||
seconds=JOB_INTERVAL_SECONDS
|
||
)
|
||
run = self.session.query(AutomationRun).one()
|
||
assert (run.scanned, run.valid, run.invalid) == (3, 2, 1)
|
||
|
||
def test_ensure_jobs_keeps_operator_settings(self):
|
||
from web.backend.services.automation import ensure_jobs
|
||
|
||
ensure_jobs(self.session)
|
||
job = (
|
||
self.session.query(AutomationJob)
|
||
.filter(AutomationJob.key == ck_health.JOB_KEY)
|
||
.one()
|
||
)
|
||
job.enabled = True
|
||
job.batch_size = 7
|
||
self.session.commit()
|
||
|
||
ensure_jobs(self.session) # 模拟重启
|
||
self.session.refresh(job)
|
||
assert job.enabled is True and job.batch_size == 7
|
||
|
||
def test_run_job_now_ignores_switch(self):
|
||
from web.backend.services.automation import ensure_jobs
|
||
|
||
ensure_jobs(self.session)
|
||
outcome = run_job_now(self.session, ck_health.JOB_KEY)
|
||
assert outcome["job_key"] == ck_health.JOB_KEY
|
||
assert self.session.query(AutomationRun).count() == 1
|
||
|
||
def test_run_job_now_unknown_key(self):
|
||
with pytest.raises(KeyError):
|
||
run_job_now(self.session, "nope")
|
||
|
||
|
||
class TestAutomationPermissions(_Base):
|
||
def test_view_allows_operation_denies_support(self):
|
||
from web.backend.permissions import ROLE_PERMISSIONS
|
||
|
||
assert "automation:view" in ROLE_PERMISSIONS["operation"]
|
||
assert "automation:view" not in ROLE_PERMISSIONS["support"]
|
||
assert "cookie:operate" not in ROLE_PERMISSIONS["support"]
|
||
assert "automation:manage" in ROLE_PERMISSIONS["super_admin"]
|
||
assert "automation:manage" not in ROLE_PERMISSIONS["operation"]
|
||
|
||
def test_manage_dependency_rejects_operator(self):
|
||
operator = User(
|
||
username="op", password_hash="h", role="operation", is_active=True
|
||
)
|
||
checker = require_permission("automation:manage")
|
||
with pytest.raises(HTTPException) as exc:
|
||
checker(current_user=operator)
|
||
assert exc.value.status_code == 403
|
||
|
||
def test_view_dependency_accepts_operator(self):
|
||
operator = User(
|
||
username="op", password_hash="h", role="operation", is_active=True
|
||
)
|
||
checker = require_permission("automation:view")
|
||
assert checker(current_user=operator).id == operator.id
|
||
|
||
|
||
def _aware(value):
|
||
"""库里读出的是 naive UTC，比较前补时区。"""
|
||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||
|
||
|
||
_REAL_CHECK = ck_health.check_douyu_cookie
|
||
|
||
|
||
def _fake_check(*, valid=True, raises=False, message=None):
|
||
def fake(cookie: str) -> dict:
|
||
if raises:
|
||
raise RuntimeError("网络异常")
|
||
return {
|
||
"valid": valid,
|
||
"message": message
|
||
if message is not None
|
||
else ("ok" if valid else "Cookie 已失效"),
|
||
"fish_ball": 1,
|
||
"nickname": "测试",
|
||
"level": 30,
|
||
"checked_at": datetime.now(UTC),
|
||
}
|
||
|
||
return fake
|
||
|
||
|
||
class TestAutomationApi(_Base):
|
||
"""接口层：序列化字段、开关落库、手动触发不受开关限制。"""
|
||
|
||
def _api(self):
|
||
from web.backend.routers import automation as api
|
||
|
||
return api
|
||
|
||
def test_list_jobs_exposes_scheduler_and_job_state(self):
|
||
api = self._api()
|
||
payload = api.list_jobs(db=self.session, current=self.admin)
|
||
# 心跳（轮询间隔）与作业周期是两个概念，接口都要暴露出来：
|
||
# 心跳密、只做「有没有到点」的检查；作业周期决定同一个作业隔多久跑下一轮。
|
||
assert (
|
||
payload["scheduler"]["tick_seconds"]
|
||
< payload["scheduler"]["job_interval_seconds"]
|
||
)
|
||
assert payload["scheduler"]["job_interval_seconds"] == 300
|
||
assert payload["scheduler"]["running"] is False
|
||
keys = [item["key"] for item in payload["jobs"]]
|
||
assert ck_health.JOB_KEY in keys
|
||
job = next(item for item in payload["jobs"] if item["key"] == ck_health.JOB_KEY)
|
||
assert job["enabled"] is False
|
||
assert job["interval_seconds"] == 6 * 3600
|
||
assert job["batch_size"] == 200
|
||
assert job["latest_run"] is None
|
||
|
||
def test_update_job_persists_and_schedules_next_run(self):
|
||
api = self._api()
|
||
api.update_job(
|
||
ck_health.JOB_KEY,
|
||
api.JobUpdate(
|
||
enabled=True, interval_seconds=3600, batch_size=5, concurrency=2
|
||
),
|
||
db=self.session,
|
||
current=self.admin,
|
||
)
|
||
job = (
|
||
self.session.query(AutomationJob)
|
||
.filter(AutomationJob.key == ck_health.JOB_KEY)
|
||
.one()
|
||
)
|
||
assert job.enabled is True
|
||
assert job.interval_seconds == 3600
|
||
assert job.batch_size == 5
|
||
assert job.next_run_at is not None # 打开开关就排了一轮
|
||
|
||
# 审计留痕
|
||
from web.backend.models import AuditLog
|
||
|
||
entry = (
|
||
self.session.query(AuditLog)
|
||
.filter(AuditLog.action == "automation:update")
|
||
.one()
|
||
)
|
||
assert entry.target == ck_health.JOB_KEY
|
||
|
||
def test_run_endpoint_records_run_and_audit(self):
|
||
api = self._api()
|
||
self.make_account("a")
|
||
ck_health.check_douyu_cookie = _fake_check(valid=True)
|
||
try:
|
||
outcome = api.run_job(
|
||
ck_health.JOB_KEY, db=self.session, current=self.admin
|
||
)
|
||
finally:
|
||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||
assert outcome["status"] == "success"
|
||
assert outcome["scanned"] == 1
|
||
assert self.session.query(AutomationRun).count() == 1
|
||
|
||
from web.backend.models import AuditLog
|
||
|
||
assert (
|
||
self.session.query(AuditLog)
|
||
.filter(AuditLog.action == "automation:run")
|
||
.count()
|
||
== 1
|
||
)
|
||
|
||
def test_run_endpoint_unknown_job_returns_404(self):
|
||
api = self._api()
|
||
with pytest.raises(HTTPException) as exc:
|
||
api.run_job("nope", db=self.session, current=self.admin)
|
||
assert exc.value.status_code == 404
|
||
|
||
def test_runs_endpoint_filters_by_job_key(self):
|
||
api = self._api()
|
||
self.session.add(
|
||
AutomationRun(job_key=ck_health.JOB_KEY, status="success", scanned=1)
|
||
)
|
||
self.session.add(AutomationRun(job_key="other", status="failed"))
|
||
self.session.commit()
|
||
payload = api.list_runs(
|
||
job_key=ck_health.JOB_KEY, limit=50, db=self.session, current=self.admin
|
||
)
|
||
assert len(payload["items"]) == 1
|
||
assert payload["items"][0]["job_key"] == ck_health.JOB_KEY
|
||
|
||
|
||
class TestFailureClassification(_Base):
|
||
"""「查不到」不等于「失效」：非鉴权类失败不能覆盖原来的结论。
|
||
|
||
斗鱼接口抖动/被风控时，如果不加区分地写 invalid，一小时就能把上千个好账号
|
||
标成失效，客服会误判成 CK 大面积挂掉。这里盯三件事：分类正确、error 不改
|
||
结论、失败占比过高时自动暂停作业。
|
||
"""
|
||
|
||
def test_auth_failure_message_counts_as_invalid(self):
|
||
assert (
|
||
ck_health.classify_result(
|
||
{
|
||
"valid": False,
|
||
"message": "鱼丸接口: 账户登陆异常，请重新登陆；等级接口: 用户未登录",
|
||
}
|
||
)
|
||
== "invalid"
|
||
)
|
||
|
||
def test_network_failure_counts_as_error_not_invalid(self):
|
||
assert (
|
||
ck_health.classify_result(
|
||
{
|
||
"valid": False,
|
||
"message": "鱼丸接口: 请求失败: timeout；等级接口: 请求失败: timeout",
|
||
}
|
||
)
|
||
== "error"
|
||
)
|
||
assert (
|
||
ck_health.classify_result({"valid": False, "message": "响应异常"})
|
||
== "error"
|
||
)
|
||
|
||
def test_valid_still_valid(self):
|
||
assert ck_health.classify_result({"valid": True, "message": "有效"}) == "valid"
|
||
|
||
def test_error_keeps_previous_verdict_but_marks_checked(self):
|
||
job = self._job()
|
||
account = self.make_account("acc", ck_check_status="valid")
|
||
ck_health.check_douyu_cookie = _fake_check(
|
||
valid=False, message="请求失败: timeout"
|
||
)
|
||
try:
|
||
counters = ck_health.run_ck_health(self.session, job)
|
||
finally:
|
||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||
self.session.refresh(account)
|
||
assert counters["valid"] == 0 and counters["invalid"] == 0
|
||
assert counters["skipped"] == 1 # error 计入 skipped
|
||
assert account.ck_check_status == "valid" # 原结论保留
|
||
assert account.ck_check_result["verdict"] == "error" # 但记录了这次查不到
|
||
assert account.ck_checked_at is not None
|
||
|
||
def test_mass_network_failure_pauses_the_job(self):
|
||
job = self._job()
|
||
job.enabled = True
|
||
job.batch_size = 40
|
||
for index in range(40):
|
||
self.make_account(f"a{index}")
|
||
ck_health.check_douyu_cookie = _fake_check(
|
||
valid=False, message="请求失败: Connection refused"
|
||
)
|
||
try:
|
||
counters = ck_health.run_ck_health(self.session, job)
|
||
finally:
|
||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||
assert counters["skipped"] == 40
|
||
assert counters["detail"]["paused"] is True
|
||
assert job.enabled is False # 自动暂停，不再继续污染
|
||
assert "已自动暂停" in counters["message"]
|
||
|
||
def test_normal_invalid_does_not_pause(self):
|
||
"""真的都是死号时不能暂停 —— 首轮 100% invalid 是合法且预期的情况。"""
|
||
job = self._job()
|
||
job.enabled = True
|
||
job.batch_size = 40
|
||
for index in range(40):
|
||
self.make_account(f"a{index}")
|
||
ck_health.check_douyu_cookie = _fake_check(
|
||
valid=False,
|
||
message="鱼丸接口: 账户登陆异常，请重新登陆；等级接口: 用户未登录",
|
||
)
|
||
try:
|
||
counters = ck_health.run_ck_health(self.session, job)
|
||
finally:
|
||
ck_health.check_douyu_cookie = _REAL_CHECK
|
||
assert counters["invalid"] == 40
|
||
assert counters["skipped"] == 0
|
||
assert job.enabled is True
|
||
assert counters["detail"]["paused"] is False
|
||
|
||
def _job(self):
|
||
return AutomationJob(
|
||
key=ck_health.JOB_KEY,
|
||
interval_seconds=6 * 3600,
|
||
batch_size=200,
|
||
concurrency=3,
|
||
)
|
||
|
||
|
||
class TestOrphanRunCleanup(_Base):
|
||
"""回归：进程重启后，卡在 running 的运行记录必须被收尾。
|
||
|
||
作业在调度线程里同步跑，进程一重启那轮就没机会写回结果，记录会永远停在
|
||
running，页面上一直显示「执行中」。登录任务早有孤儿清理，运行记录原先漏了
|
||
—— 线上部署时打断了一轮，run 14 就是这么卡住的。
|
||
"""
|
||
|
||
def test_running_runs_are_closed_on_startup(self):
|
||
from web.backend.services.automation.scheduler import cleanup_orphan_runs
|
||
|
||
stale = AutomationRun(
|
||
job_key="ck_relogin", started_at=datetime.now(UTC), status="running"
|
||
)
|
||
done = AutomationRun(
|
||
job_key="ck_health",
|
||
started_at=datetime.now(UTC),
|
||
finished_at=datetime.now(UTC),
|
||
status="success",
|
||
)
|
||
self.session.add_all([stale, done])
|
||
self.session.commit()
|
||
|
||
cleaned = cleanup_orphan_runs(self.session)
|
||
self.session.refresh(stale)
|
||
self.session.refresh(done)
|
||
assert cleaned == 1
|
||
assert stale.status == "interrupted"
|
||
assert stale.finished_at is not None
|
||
assert stale.message == "服务重启中断"
|
||
assert done.status == "success" # 已完成的不受影响
|
||
|
||
|
||
class TestHeartbeatDecoupledFromInterval(_Base):
|
||
"""回归：心跳频率不能直接当作业周期用。
|
||
|
||
曾经 TICK_SECONDS 既是心跳又是作业周期（next_run_at = 结束 + 300s，心跳也是
|
||
300s）。两者同频会系统性跳轮：作业 19 秒跑完 → next 在 5 分钟后，而下一次
|
||
心跳正好在 5 分钟整，常常差几十秒没到点，于是整轮空转 —— 实际变成每 10 分钟
|
||
才跑一次。线上表现就是「检测只要 19 秒，却总是排队等很久」。
|
||
"""
|
||
|
||
def test_heartbeat_is_much_shorter_than_job_interval(self):
|
||
from web.backend.services.automation.scheduler import (
|
||
JOB_INTERVAL_SECONDS,
|
||
TICK_SECONDS,
|
||
)
|
||
|
||
assert TICK_SECONDS < JOB_INTERVAL_SECONDS, (
|
||
"心跳必须比作业周期密，否则会出现「差几十秒没到点→整轮空转」的跳轮"
|
||
)
|
||
# 至少密 4 倍：作业耗时波动时不会积压成整轮跳过
|
||
assert JOB_INTERVAL_SECONDS / TICK_SECONDS >= 4
|
||
|
||
def test_job_runs_again_once_its_own_interval_elapsed(self):
|
||
"""心跳多跳几次后，到点的作业必须被跑到。"""
|
||
|
||
job = self._register_probe()
|
||
run_due_jobs(self.session)
|
||
first = self.session.query(AutomationRun).count()
|
||
|
||
# 周期未到：无论心跳跳多少次都不跑
|
||
for _ in range(3):
|
||
run_due_jobs(self.session)
|
||
assert self.session.query(AutomationRun).count() == first
|
||
|
||
# 把 next_run_at 拨到过去（等价于「周期已到」），下一次心跳就该跑
|
||
job.next_run_at = datetime.now(UTC) - timedelta(seconds=1)
|
||
self.session.commit()
|
||
run_due_jobs(self.session)
|
||
assert self.session.query(AutomationRun).count() == first + 1
|
||
|
||
def _register_probe(self):
|
||
from web.backend.services.automation.scheduler import (
|
||
JobSpec,
|
||
register_job,
|
||
reset_registry,
|
||
)
|
||
|
||
reset_registry()
|
||
register_job(
|
||
JobSpec(
|
||
key="probe",
|
||
title="探针",
|
||
interval_seconds=6 * 3600,
|
||
batch_size=10,
|
||
concurrency=1,
|
||
run=lambda db, job: {"scanned": 1, "valid": 1, "invalid": 0},
|
||
)
|
||
)
|
||
from web.backend.services.automation import ensure_jobs
|
||
|
||
ensure_jobs(self.session)
|
||
job = (
|
||
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").one()
|
||
)
|
||
job.enabled = True
|
||
job.next_run_at = datetime.now(UTC) - timedelta(seconds=1)
|
||
self.session.commit()
|
||
return job
|
||
|
||
|
||
class TestSchedulerRunsJobsConcurrently:
|
||
"""作业之间**并发**跑 —— 慢作业不该让快作业干等。
|
||
|
||
以前是串行（源码注释：「作业串行执行 —— 目前只有 CK 巡检，单轮约 20~40 秒」）。
|
||
但「账号资料定时补全」一轮要 4 分钟以上，串行会把 CK 巡检饿住。
|
||
|
||
调度循环用 `wait=False`：**派发完就返回**，tick 不再被慢作业拖住。
|
||
"""
|
||
|
||
def setup_method(self):
|
||
self.engine = get_test_engine()
|
||
self.session = make_test_session(self.engine)
|
||
reset_registry()
|
||
|
||
def teardown_method(self):
|
||
reset_registry()
|
||
self.session.close()
|
||
|
||
def _job(self, key: str, run, *, enabled: bool = True):
|
||
register_job(
|
||
JobSpec(
|
||
key=key,
|
||
title=key,
|
||
interval_seconds=3600,
|
||
batch_size=1,
|
||
concurrency=1,
|
||
run=run,
|
||
)
|
||
)
|
||
self.session.add(
|
||
AutomationJob(
|
||
key=key,
|
||
title=key,
|
||
interval_seconds=3600,
|
||
batch_size=1,
|
||
concurrency=1,
|
||
enabled=enabled,
|
||
)
|
||
)
|
||
self.session.commit()
|
||
|
||
def test_wait_false_returns_without_blocking(self):
|
||
"""`wait=False` 必须**立刻返回**，不等作业跑完。"""
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
|
||
def slow(db, job):
|
||
started.set()
|
||
release.wait(timeout=5)
|
||
return {"scanned": 1}
|
||
|
||
self._job("slow", slow)
|
||
outcomes = run_due_jobs(self.session, wait=False)
|
||
try:
|
||
assert outcomes == [], "wait=False 不该返回 outcomes（它不等结果）"
|
||
assert started.wait(timeout=5), "作业应已在后台开始"
|
||
finally:
|
||
release.set()
|
||
# 等在途标记清掉，避免污染后续用例
|
||
for _ in range(50):
|
||
if not scheduler._IN_FLIGHT:
|
||
break
|
||
time.sleep(0.1)
|
||
|
||
def test_in_flight_job_is_not_dispatched_twice(self):
|
||
"""**在途守卫**：同一个作业不会重复派发。
|
||
|
||
这是并发化之后最容易出的事故：`next_run_at` 要等作业跑完才更新，
|
||
所以一个跑得比 tick 还久的作业，下一个 tick 看它仍然是「到期」——
|
||
没有守卫就会被**再派发一次**，等于把速率翻倍
|
||
（对斗鱼接口就是"并发放开"，正是风控最敏感的地方）。
|
||
"""
|
||
calls: list[int] = []
|
||
release = threading.Event()
|
||
entered = threading.Event()
|
||
|
||
def blocking(db, job):
|
||
calls.append(1)
|
||
entered.set()
|
||
release.wait(timeout=5)
|
||
return {}
|
||
|
||
self._job("blocking", blocking)
|
||
run_due_jobs(self.session, wait=False)
|
||
try:
|
||
assert entered.wait(timeout=5)
|
||
# 第一次还在跑，再触发一轮 —— 不该被重复派发
|
||
run_due_jobs(self.session, wait=False)
|
||
time.sleep(0.3)
|
||
assert len(calls) == 1, "在途作业被重复派发了"
|
||
finally:
|
||
release.set()
|
||
for _ in range(50):
|
||
if not scheduler._IN_FLIGHT:
|
||
break
|
||
time.sleep(0.1)
|
||
|
||
def test_release_clears_marker_even_on_failure(self):
|
||
"""失败的作业也必须清掉在途标记，否则它以后**再也不会跑**。"""
|
||
scheduler._IN_FLIGHT.add("ghost")
|
||
scheduler._release("ghost")
|
||
assert "ghost" not in scheduler._IN_FLIGHT
|
||
|
||
def test_failed_job_is_released_and_can_run_again(self):
|
||
"""作业抛异常后仍要能被下一轮再派发（别把它永久卡死在"在途"）。"""
|
||
calls: list[int] = []
|
||
|
||
def boom(db, job):
|
||
calls.append(1)
|
||
raise RuntimeError("接口挂了")
|
||
|
||
self._job("boom", boom)
|
||
run_due_jobs(self.session, wait=False)
|
||
for _ in range(50):
|
||
if not scheduler._IN_FLIGHT:
|
||
break
|
||
time.sleep(0.1)
|
||
assert not scheduler._IN_FLIGHT, "失败后没清在途标记，作业会被永久跳过"
|
||
assert calls, "作业应该真的跑过一次"
|
||
|
||
|
||
class TestRunIntervalIsScheduling:
|
||
"""「多久跑一次」与「同一行多久重检」是**两个独立概念**。
|
||
|
||
用户反馈：界面上那个「单账号重检间隔(小时)」很别扭，直接设「多久跑一次、
|
||
一次跑多少个」更符合直觉 —— 对，因为**调度频率以前根本改不了**：
|
||
|
||
job.next_run_at = finished + timedelta(seconds=JOB_INTERVAL_SECONDS) # 硬编码 5 分钟
|
||
|
||
于是不管界面上填什么，下一轮永远是「跑完 + 5 分钟」。
|
||
"""
|
||
|
||
def setup_method(self):
|
||
self.engine = get_test_engine()
|
||
self.session = make_test_session(self.engine)
|
||
reset_registry()
|
||
|
||
def teardown_method(self):
|
||
reset_registry()
|
||
self.session.close()
|
||
|
||
def _job(self, run_interval: int | None):
|
||
calls: list[int] = []
|
||
|
||
def probe(db, job):
|
||
calls.append(1)
|
||
return {"scanned": 1}
|
||
|
||
register_job(
|
||
JobSpec(
|
||
key="probe",
|
||
title="probe",
|
||
interval_seconds=6 * 3600, # 同一行重检间隔（不参与调度）
|
||
batch_size=10,
|
||
concurrency=1,
|
||
run=probe,
|
||
**({"run_interval_seconds": run_interval} if run_interval else {}),
|
||
)
|
||
)
|
||
self.session.add(
|
||
AutomationJob(
|
||
key="probe",
|
||
title="probe",
|
||
interval_seconds=6 * 3600,
|
||
run_interval_seconds=run_interval or 300,
|
||
batch_size=10,
|
||
concurrency=1,
|
||
enabled=True,
|
||
)
|
||
)
|
||
self.session.commit()
|
||
return calls
|
||
|
||
def _next_run_delay(self) -> float:
|
||
run_due_jobs(self.session)
|
||
job = (
|
||
self.session.query(AutomationJob).filter(AutomationJob.key == "probe").one()
|
||
)
|
||
assert job.last_run_at is not None and job.next_run_at is not None
|
||
return (job.next_run_at - job.last_run_at).total_seconds()
|
||
|
||
def test_next_run_follows_run_interval_not_the_row_interval(self):
|
||
"""下一轮按**运行间隔**排，不是按「同一行重检间隔」。"""
|
||
self._job(run_interval=1800)
|
||
delay = self._next_run_delay()
|
||
assert delay == 1800, f"应按运行间隔 1800 排，实际 {delay}"
|
||
assert delay != 6 * 3600, "不能再用「同一行重检间隔」当调度周期"
|
||
|
||
def test_default_run_interval_keeps_old_behaviour(self):
|
||
"""不填时默认 300 秒 —— 与改动前的硬编码行为一致（不会突然变快变慢）。"""
|
||
self._job(run_interval=None)
|
||
assert self._next_run_delay() == 300
|
||
|
||
def test_too_small_interval_is_clamped(self):
|
||
"""太小的间隔是忙轮询，没有意义 —— 有下限。"""
|
||
self._job(run_interval=1)
|
||
assert self._next_run_delay() >= scheduler.MIN_RUN_INTERVAL_SECONDS
|