From 5fd7df27dcf21e635471a374e0934d9a661a5af6 Mon Sep 17 00:00:00 2001 From: yml Date: Sat, 21 Nov 2026 23:04:08 +0800 Subject: [PATCH] =?UTF-8?q?feat(automation):=20=E6=8C=89=E6=88=90=E5=8A=9F?= =?UTF-8?q?=E7=8E=87=E7=81=B0=E5=BA=A6=E6=89=A9=E7=BC=A9=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E9=87=8D=E7=99=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/斗鱼/斗鱼账号平台最终实施设计.md | 20 +- tests/test_account_supply_service.py | 60 ++++++ tests/test_automation_ck_relogin.py | 95 +++++++++ web/backend/services/account_observability.py | 58 +++++- .../services/automation/jobs/ck_relogin.py | 196 +++++++++++++++++- 5 files changed, 422 insertions(+), 7 deletions(-) diff --git a/docs/斗鱼/斗鱼账号平台最终实施设计.md b/docs/斗鱼/斗鱼账号平台最终实施设计.md index 00c6cb5..7ea737f 100644 --- a/docs/斗鱼/斗鱼账号平台最终实施设计.md +++ b/docs/斗鱼/斗鱼账号平台最终实施设计.md @@ -11,7 +11,7 @@ 更新时间:2026-11-21 -整体完成度约 **75%**。 +整体完成度约 **82%**。 | 阶段 | 完成度 | 已落地 | 主要缺口 | | --- | ---: | --- | --- | @@ -19,7 +19,7 @@ | 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 | | 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 | | 阶段 3:商品与供给 | 100% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警、7 天查询/登录/CK 趋势 | 无 | -| 阶段 4:安全恢复自动化 | 50% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效 | 灰度批次自动扩缩和长期趋势告警 | +| 阶段 4:安全恢复自动化 | 90% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效,按 7 天成功率自动扩缩批次,长期低成功率告警 | 生产灰度观察与动态预算参数校准 | | 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 | ## 为什么重构 @@ -530,6 +530,21 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。 - 逐步恢复资料同步 - 按成功率和风控调整预算 +自动重登的 `batch_size` 是人工配置的安全上限,实际有效批量独立保存在作业配置中: + +- 冷启动从 `min(batch_size, 10)` 开始; +- 最近 7 天完成样本不少于 10 条时参与调整; +- 成功率 `>= 90%` 时每次最多增长 25%,但不超过安全上限和积压数; +- 成功率 `< 70%` 时批量减半; +- 邮箱配额满不计入成功率分母,避免外部供给限制误触发缩容; +- 现有单轮硬失败率熔断继续优先级最高,触发后直接暂停作业。 + +平台状态还会对长期趋势告警: + +- 近 7 天查询样本不少于 20 且成功率低于 80%; +- 近 7 天登录样本不少于 10 且成功率低于 60%; +- 近 7 天重登失败至少 5 次且失败数高于成功数。 + ### 阶段 5:清理旧逻辑 删除或降级: @@ -620,3 +635,4 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。 | 2026-11-21 | 增加保守默认供给目标、账号平台供给看板和库存/CK/工作队列阈值告警 | | 2026-11-21 | 增加供给目标管理 UI,支持新增、编辑、启停和调整安全库存 | | 2026-11-21 | 完成阶段 3:增加 7 天查询、登录和 CK 生命周期趋势,并补充同步时间索引 | +| 2026-11-21 | 阶段 4:自动重登按 7 天成功率灰度扩缩批次,并增加长期低成功率告警 | diff --git a/tests/test_account_supply_service.py b/tests/test_account_supply_service.py index a5d7ef0..b3e8186 100644 --- a/tests/test_account_supply_service.py +++ b/tests/test_account_supply_service.py @@ -344,3 +344,63 @@ def test_account_platform_status_builds_seven_day_trends(db): assert login_today["failed"] == 1 assert login_today["success_rate"] == 50.0 assert lifecycle_today["ck_invalid"] == 1 + + +def test_account_platform_status_alerts_on_persistently_low_trends(db): + account = _account("platform-trend-alerts") + db.add(account) + db.flush() + now = datetime.now(UTC) + db.add_all( + [ + AccountSyncRun( + account_id=account.id, + domain="storehome", + started_at=now, + finished_at=now, + status="failed", + duration_ms=100, + ) + for _ in range(20) + ] + ) + db.add_all( + [ + LoginAttempt( + account_id=account.id, + category="relogin_ordinary", + trigger_source="automation", + started_at=now, + finished_at=now, + status="failed", + failure_kind="network", + quota_day=now.date(), + ) + for _ in range(10) + ] + ) + db.add_all( + [ + AccountEvent( + account_id=account.id, + event_type="relogin_failed", + category="auth", + occurred_at=now, + title="重登失败", + source="pytest", + actor_type="system", + source_ref=f"pytest-relogin-alert:{account.id}:{index}", + ) + for index in range(5) + ] + ) + db.commit() + + status = account_platform_status(db, now=now) + codes = {alert["code"] for alert in status["alerts"]} + + assert { + "query_success_rate_low", + "login_success_rate_low", + "relogin_failure_trend_high", + } <= codes diff --git a/tests/test_automation_ck_relogin.py b/tests/test_automation_ck_relogin.py index b3f9f54..3c171ff 100644 --- a/tests/test_automation_ck_relogin.py +++ b/tests/test_automation_ck_relogin.py @@ -23,6 +23,7 @@ from tests.support.db import get_test_engine, make_test_session from web.backend.models import ( Account, AutomationJob, + LoginAttempt, LoginQuotaSetting, LoginTask, User, @@ -230,6 +231,100 @@ class TestFailureCircuit(_Base): assert paused is False +class TestAdaptiveBatch(_Base): + def _attempts( + self, + account: Account, + *, + success: int, + failed: int, + failure_kind: str = "network", + ): + now = datetime.now(UTC) + rows = [ + LoginAttempt( + account_id=account.id, + category="relogin_ordinary", + trigger_source="automation", + started_at=now, + finished_at=now, + status="success", + quota_day=now.date(), + ) + for _ in range(success) + ] + rows.extend( + LoginAttempt( + account_id=account.id, + category="relogin_ordinary", + trigger_source="automation", + started_at=now, + finished_at=now, + status="failed", + failure_kind=failure_kind, + quota_day=now.date(), + ) + for _ in range(failed) + ) + self.session.add_all(rows) + self.session.commit() + + def test_cold_start_uses_initial_batch_not_configured_ceiling(self): + job = self.job() + job.batch_size = 50 + job.config = {} + self.session.commit() + + decision = ck_relogin.resolve_adaptive_batch_size( + self.session, + job, + now=datetime.now(UTC), + pending=100, + ) + + assert decision.batch_size == 10 + assert decision.action == "cold_start" + + def test_high_recent_success_grows_by_at_most_twenty_five_percent(self): + account = self.make_account("adaptive-success") + self._attempts(account, success=9, failed=1) + job = self.job() + job.batch_size = 50 + job.config = {"adaptive_batch_size": 10} + self.session.commit() + + decision = ck_relogin.resolve_adaptive_batch_size( + self.session, + job, + now=datetime.now(UTC), + pending=100, + ) + + assert decision.success_rate == 90.0 + assert decision.batch_size == 13 + assert decision.action == "grow" + assert job.config["adaptive_batch_size"] == 13 + + def test_low_recent_success_halves_batch(self): + account = self.make_account("adaptive-shrink") + self._attempts(account, success=2, failed=8) + job = self.job() + job.batch_size = 50 + job.config = {"adaptive_batch_size": 20} + self.session.commit() + + decision = ck_relogin.resolve_adaptive_batch_size( + self.session, + job, + now=datetime.now(UTC), + pending=100, + ) + + assert decision.success_rate == 20.0 + assert decision.batch_size == 10 + assert decision.action == "shrink" + + class TestBackoff(_Base): def test_ladder_is_escalating(self): now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC) diff --git a/web/backend/services/account_observability.py b/web/backend/services/account_observability.py index ccbadc4..e228f1b 100644 --- a/web/backend/services/account_observability.py +++ b/web/backend/services/account_observability.py @@ -403,6 +403,62 @@ def account_platform_status( } ) + trends = _platform_trends(db, moment) + query_sample = sum( + int(row["success"]) + int(row["failed"]) + for row in trends["query"] + ) + query_success = sum(int(row["success"]) for row in trends["query"]) + query_rate = _rate(query_success, query_sample) + if query_sample >= 20 and query_rate < 80: + alerts.append( + { + "code": "query_success_rate_low", + "severity": "critical" if query_rate < 50 else "warning", + "count": query_sample, + "message": ( + f"近 {TREND_DAYS} 天查询成功率 {query_rate:.1f}%" + f"({query_sample} 次已完成查询)" + ), + } + ) + + login_sample = sum( + int(row["success"]) + int(row["failed"]) + for row in trends["login"] + ) + login_success = sum(int(row["success"]) for row in trends["login"]) + login_rate = _rate(login_success, login_sample) + if login_sample >= 10 and login_rate < 60: + alerts.append( + { + "code": "login_success_rate_low", + "severity": "critical" if login_rate < 40 else "warning", + "count": login_sample, + "message": ( + f"近 {TREND_DAYS} 天登录成功率 {login_rate:.1f}%" + f"({login_sample} 次已完成尝试)" + ), + } + ) + + relogin_ok = sum(int(row["relogin_ok"]) for row in trends["lifecycle"]) + relogin_failed = sum( + int(row["relogin_failed"]) for row in trends["lifecycle"] + ) + if relogin_failed >= 5 and relogin_failed > relogin_ok: + alerts.append( + { + "code": "relogin_failure_trend_high", + "severity": "warning", + "count": relogin_failed, + "message": ( + f"近 {TREND_DAYS} 天重登失败 {relogin_failed} 次," + f"成功 {relogin_ok} 次" + ), + } + ) + return { "checked_at": moment, "accounts": { @@ -425,7 +481,7 @@ def account_platform_status( }, "supply_targets": supply_targets, "alerts": alerts, - "trends": _platform_trends(db, moment), + "trends": trends, } diff --git a/web/backend/services/automation/jobs/ck_relogin.py b/web/backend/services/automation/jobs/ck_relogin.py index b4c8d10..d58d463 100644 --- a/web/backend/services/automation/jobs/ck_relogin.py +++ b/web/backend/services/automation/jobs/ck_relogin.py @@ -13,7 +13,9 @@ from __future__ import annotations +import math import random +from dataclasses import dataclass from datetime import UTC, datetime, timedelta from typing import Any @@ -21,7 +23,14 @@ from sqlalchemy import func, or_ from sqlalchemy.orm import Session from ....database import WorkerSessionLocal -from ....models import Account, AutomationJob, LoginQuotaSetting, LoginTask, User +from ....models import ( + Account, + AutomationJob, + LoginAttempt, + LoginQuotaSetting, + LoginTask, + User, +) from ....models import ProxyConfig as ProxyConfigModel from ...account_events_service import EVENT_RELOGIN_FAILED, emit from ...account_lifecycle import ( @@ -62,6 +71,23 @@ AUTO_MAX_TOTAL_TIME = 240 QUOTA_JITTER_MINUTES = 30 DEFAULT_FAILURE_PAUSE_RATIO = 0.4 DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10 +DEFAULT_ADAPTIVE_INITIAL_BATCH = 10 +DEFAULT_ADAPTIVE_WINDOW_DAYS = 7 +DEFAULT_ADAPTIVE_MIN_SAMPLE = 10 +DEFAULT_ADAPTIVE_GROW_RATE = 90.0 +DEFAULT_ADAPTIVE_SHRINK_RATE = 70.0 +DEFAULT_ADAPTIVE_GROW_FACTOR = 1.25 + + +@dataclass(frozen=True) +class AdaptiveBatchDecision: + batch_size: int + configured_max: int + previous_batch_size: int + success_rate: float + sample_size: int + pending: int + action: str def _cutoff(now: datetime) -> datetime: @@ -75,6 +101,7 @@ def build_select_query( *, max_attempts: int = 2, system_user_id: int | None = None, + batch_limit: int | None = None, ): """构造本轮重登候选的查询(与取行分开,便于按方言编译做回归)。 @@ -111,12 +138,13 @@ def build_select_query( pass # 预留:将来若需要限制到某个归属人,在这里加过滤 # 名额先给第 1、2 档(两套还能换 / 还剩一套 / 积分≥300)。 # 同一档里仍按退避时间,避免把还没到点的高价值号提前拉起来。 + limit = int(batch_limit if batch_limit is not None else job.batch_size or 0) return query.order_by( value_tier_expr().asc(), Account.relogin_next_at.is_(None).desc(), Account.relogin_next_at.asc(), Account.id.asc(), - ).limit(max(int(job.batch_size or 0), 1)) + ).limit(max(limit, 1)) def _load_batch_state( @@ -271,6 +299,138 @@ def resolve_failure_circuit_config( return resolved +def _recent_relogin_success( + db: Session, + *, + now: datetime, + window_days: int, +) -> tuple[int, float]: + start_day = now.date() - timedelta(days=max(window_days, 1) - 1) + rows = ( + db.query( + LoginAttempt.status, + LoginAttempt.failure_kind, + func.count(LoginAttempt.id), + ) + .filter( + LoginAttempt.quota_day >= start_day, + LoginAttempt.category.in_( + ("relogin_high_value", "relogin_ordinary") + ), + ) + .group_by(LoginAttempt.status, LoginAttempt.failure_kind) + .all() + ) + success = 0 + failed = 0 + for status, failure_kind, count in rows: + count_value = int(count or 0) + if status == "success": + success += count_value + elif status in {"failed", "error", "stopped"} and failure_kind != "email_quota": + failed += count_value + sample = success + failed + rate = success * 100 / sample if sample else 0.0 + return sample, round(rate, 1) + + +def resolve_adaptive_batch_size( + db: Session, + job: AutomationJob, + *, + now: datetime, + pending: int, +) -> AdaptiveBatchDecision: + """Scale the effective relogin batch inside the operator-configured ceiling.""" + config = dict(job.config or {}) + configured_max = max(int(job.batch_size or 1), 1) + if not bool(config.get("adaptive_batch_enabled", True)): + decision = AdaptiveBatchDecision( + batch_size=configured_max, + configured_max=configured_max, + previous_batch_size=configured_max, + success_rate=0.0, + sample_size=0, + pending=max(0, int(pending)), + action="fixed", + ) + config["adaptive_batch_size"] = decision.batch_size + job.config = config + return decision + + window_days = max( + 1, + int(config.get("adaptive_window_days", DEFAULT_ADAPTIVE_WINDOW_DAYS) or 1), + ) + min_sample = max( + 1, + int(config.get("adaptive_min_sample", DEFAULT_ADAPTIVE_MIN_SAMPLE) or 1), + ) + grow_rate = float( + config.get("adaptive_grow_rate", DEFAULT_ADAPTIVE_GROW_RATE) + ) + shrink_rate = float( + config.get("adaptive_shrink_rate", DEFAULT_ADAPTIVE_SHRINK_RATE) + ) + grow_factor = max( + 1.01, + float(config.get("adaptive_grow_factor", DEFAULT_ADAPTIVE_GROW_FACTOR)), + ) + initial = min( + configured_max, + max( + 1, + int( + config.get( + "adaptive_initial_batch", + DEFAULT_ADAPTIVE_INITIAL_BATCH, + ) + or 1 + ), + ), + ) + previous = min( + configured_max, + max(1, int(config.get("adaptive_batch_size", initial) or initial)), + ) + sample, success_rate = _recent_relogin_success( + db, + now=now, + window_days=window_days, + ) + + target = previous + action = "hold" + if sample >= min_sample and success_rate < shrink_rate: + target = max(1, previous // 2) + action = "shrink" + elif sample >= min_sample and success_rate >= grow_rate: + target = min( + configured_max, + max(previous + 1, math.ceil(previous * grow_factor)), + ) + action = "grow" if target > previous else "hold" + elif sample < min_sample: + action = "cold_start" + + if pending > 0: + target = min(target, max(1, int(pending))) + decision = AdaptiveBatchDecision( + batch_size=max(1, target), + configured_max=configured_max, + previous_batch_size=previous, + success_rate=success_rate, + sample_size=sample, + pending=max(0, int(pending)), + action=action, + ) + config["adaptive_batch_size"] = decision.batch_size + config["adaptive_last_success_rate"] = decision.success_rate + config["adaptive_last_sample"] = decision.sample_size + job.config = config + return decision + + def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: now = datetime.now(UTC) config = resolve_failure_circuit_config(db, job.config) @@ -286,8 +446,22 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: "message": "库里没有可用用户,无法以系统身份发起重登", } - accounts = build_select_query(db, job, now, max_attempts=max_attempts).all() + pending = count_pending_relogin(db, max_attempts=max_attempts) + adaptive = resolve_adaptive_batch_size( + db, + job, + now=now, + pending=pending, + ) + accounts = build_select_query( + db, + job, + now, + max_attempts=max_attempts, + batch_limit=adaptive.batch_size, + ).all() if not accounts: + db.commit() return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0} selected_account_ids = tuple(int(account.id) for account in accounts) @@ -370,7 +544,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: job.updated_at = now parts = [ - f"本轮重登 {counters['scanned']} 条", + f"本轮重登 {counters['scanned']} 条(有效批量 {adaptive.batch_size})", f"成功 {counters['valid']}", f"失败 {counters['invalid']}", ] @@ -384,6 +558,11 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})") if paused: parts.append("失败率过高,已自动暂停作业") + if adaptive.action != "hold": + parts.append( + f"批量策略 {adaptive.previous_batch_size}->{adaptive.batch_size}" + f"({adaptive.action},近 7 天成功率 {adaptive.success_rate:.1f}%)" + ) return { "scanned": counters["scanned"], "valid": counters["valid"], @@ -400,6 +579,15 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: "hard_failures": hard_failures, "hard_failure_ratio": round(hard_failure_ratio, 4), "paused": paused, + "adaptive_batch": { + "batch_size": adaptive.batch_size, + "configured_max": adaptive.configured_max, + "previous_batch_size": adaptive.previous_batch_size, + "success_rate": adaptive.success_rate, + "sample_size": adaptive.sample_size, + "pending": adaptive.pending, + "action": adaptive.action, + }, }, }