diff --git a/tests/test_automation_ck_health.py b/tests/test_automation_ck_health.py index dc58006..f5dce36 100644 --- a/tests/test_automation_ck_health.py +++ b/tests/test_automation_ck_health.py @@ -582,3 +582,36 @@ class TestFailureClassification(_Base): batch_size=200, concurrency=3, ) + + +class TestOrphanRunCleanup(_Base): + """回归:进程重启后,卡在 running 的运行记录必须被收尾。 + + 作业在调度线程里同步跑,进程一重启那轮就没机会写回结果,记录会永远停在 + running,页面上一直显示「执行中」。登录任务早有孤儿清理,运行记录原先漏了 + —— 线上部署时打断了一轮,run 14 就是这么卡住的。 + """ + + def test_running_runs_are_closed_on_startup(self): + from web.backend.services.automation.scheduler import cleanup_orphan_runs + + stale = AutomationRun( + job_key="ck_relogin", started_at=datetime.now(UTC), status="running" + ) + done = AutomationRun( + job_key="ck_health", + started_at=datetime.now(UTC), + finished_at=datetime.now(UTC), + status="success", + ) + self.session.add_all([stale, done]) + self.session.commit() + + cleaned = cleanup_orphan_runs(self.session) + self.session.refresh(stale) + self.session.refresh(done) + assert cleaned == 1 + assert stale.status == "failed" + assert stale.finished_at is not None + assert stale.message == "服务重启中断" + assert done.status == "success" # 已完成的不受影响 diff --git a/web/backend/main.py b/web/backend/main.py index 7dc8ce6..2933a47 100644 --- a/web/backend/main.py +++ b/web/backend/main.py @@ -66,6 +66,11 @@ async def lifespan(app: FastAPI): cleaned_relogin = cleanup_orphan_relogin_tasks(db) if cleaned_relogin: logger.info(f"启动清理 CK 重登残留任务: {cleaned_relogin} 条") + from .services.automation.scheduler import cleanup_orphan_runs + + cleaned_runs = cleanup_orphan_runs(db) + if cleaned_runs: + logger.info(f"启动收尾中断的自动化运行记录: {cleaned_runs} 条") cleaned_login = cleanup_orphan_login_tasks(db) if cleaned_login: logger.info(f"启动清理登录残留任务: {cleaned_login} 条") diff --git a/web/backend/services/automation/scheduler.py b/web/backend/services/automation/scheduler.py index aea822b..ca9326d 100644 --- a/web/backend/services/automation/scheduler.py +++ b/web/backend/services/automation/scheduler.py @@ -118,6 +118,26 @@ def _due(job: AutomationJob, now: datetime) -> bool: return next_run_at is None or next_run_at <= now +def cleanup_orphan_runs(db: Session, message: str = "服务重启中断") -> int: + """把上次进程留下的 ``running`` 运行记录收尾。 + + 作业是在调度线程里同步跑的:进程一重启(部署、崩溃、手工重启),那轮作业就 + 再没机会写回结果,``automation_runs`` 会永远停在 ``running``,页面上一直显示 + 「执行中」。登录任务有自己的孤儿清理,运行记录原先漏了。 + + 收尾为 ``failed`` 而不是 ``success``:那轮确实没跑完,如实记录;被中断的账号 + 由各自的孤儿清理负责标记,并会按退避重试。 + """ + orphans = db.query(AutomationRun).filter(AutomationRun.status == "running").all() + for run in orphans: + run.status = "failed" + run.finished_at = datetime.now(UTC) + run.message = message + if orphans: + db.commit() + return len(orphans) + + def run_due_jobs(db: Session, *, now: datetime | None = None) -> list[dict[str, Any]]: """跑一轮「到点的作业」,返回每轮结果。