feat(automation): 按成功率灰度扩缩自动重登

This commit is contained in:
yml
2026-11-21 23:04:08 +08:00
parent 3e618c1a88
commit 5fd7df27dc
5 changed files with 422 additions and 7 deletions
@@ -11,7 +11,7 @@
更新时间:2026-11-21
整体完成度约 **75%**。
整体完成度约 **82%**。
| 阶段 | 完成度 | 已落地 | 主要缺口 |
| --- | ---: | --- | --- |
@@ -19,7 +19,7 @@
| 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 |
| 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 |
| 阶段 3:商品与供给 | 100% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警、7 天查询/登录/CK 趋势 | 无 |
| 阶段 4:安全恢复自动化 | 50% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效 | 灰度批次自动扩缩和长期趋势告警 |
| 阶段 4:安全恢复自动化 | 90% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效,按 7 天成功率自动扩缩批次,长期低成功率告警 | 生产灰度观察与动态预算参数校准 |
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
## 为什么重构
@@ -530,6 +530,21 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。
- 逐步恢复资料同步
- 按成功率和风控调整预算
自动重登的 `batch_size` 是人工配置的安全上限,实际有效批量独立保存在作业配置中:
- 冷启动从 `min(batch_size, 10)` 开始;
- 最近 7 天完成样本不少于 10 条时参与调整;
- 成功率 `>= 90%` 时每次最多增长 25%,但不超过安全上限和积压数;
- 成功率 `< 70%` 时批量减半;
- 邮箱配额满不计入成功率分母,避免外部供给限制误触发缩容;
- 现有单轮硬失败率熔断继续优先级最高,触发后直接暂停作业。
平台状态还会对长期趋势告警:
- 近 7 天查询样本不少于 20 且成功率低于 80%;
- 近 7 天登录样本不少于 10 且成功率低于 60%;
- 近 7 天重登失败至少 5 次且失败数高于成功数。
### 阶段 5:清理旧逻辑
删除或降级:
@@ -620,3 +635,4 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。
| 2026-11-21 | 增加保守默认供给目标、账号平台供给看板和库存/CK/工作队列阈值告警 |
| 2026-11-21 | 增加供给目标管理 UI,支持新增、编辑、启停和调整安全库存 |
| 2026-11-21 | 完成阶段 3:增加 7 天查询、登录和 CK 生命周期趋势,并补充同步时间索引 |
| 2026-11-21 | 阶段 4:自动重登按 7 天成功率灰度扩缩批次,并增加长期低成功率告警 |
+60
View File
@@ -344,3 +344,63 @@ def test_account_platform_status_builds_seven_day_trends(db):
assert login_today["failed"] == 1
assert login_today["success_rate"] == 50.0
assert lifecycle_today["ck_invalid"] == 1
def test_account_platform_status_alerts_on_persistently_low_trends(db):
account = _account("platform-trend-alerts")
db.add(account)
db.flush()
now = datetime.now(UTC)
db.add_all(
[
AccountSyncRun(
account_id=account.id,
domain="storehome",
started_at=now,
finished_at=now,
status="failed",
duration_ms=100,
)
for _ in range(20)
]
)
db.add_all(
[
LoginAttempt(
account_id=account.id,
category="relogin_ordinary",
trigger_source="automation",
started_at=now,
finished_at=now,
status="failed",
failure_kind="network",
quota_day=now.date(),
)
for _ in range(10)
]
)
db.add_all(
[
AccountEvent(
account_id=account.id,
event_type="relogin_failed",
category="auth",
occurred_at=now,
title="重登失败",
source="pytest",
actor_type="system",
source_ref=f"pytest-relogin-alert:{account.id}:{index}",
)
for index in range(5)
]
)
db.commit()
status = account_platform_status(db, now=now)
codes = {alert["code"] for alert in status["alerts"]}
assert {
"query_success_rate_low",
"login_success_rate_low",
"relogin_failure_trend_high",
} <= codes
+95
View File
@@ -23,6 +23,7 @@ from tests.support.db import get_test_engine, make_test_session
from web.backend.models import (
Account,
AutomationJob,
LoginAttempt,
LoginQuotaSetting,
LoginTask,
User,
@@ -230,6 +231,100 @@ class TestFailureCircuit(_Base):
assert paused is False
class TestAdaptiveBatch(_Base):
def _attempts(
self,
account: Account,
*,
success: int,
failed: int,
failure_kind: str = "network",
):
now = datetime.now(UTC)
rows = [
LoginAttempt(
account_id=account.id,
category="relogin_ordinary",
trigger_source="automation",
started_at=now,
finished_at=now,
status="success",
quota_day=now.date(),
)
for _ in range(success)
]
rows.extend(
LoginAttempt(
account_id=account.id,
category="relogin_ordinary",
trigger_source="automation",
started_at=now,
finished_at=now,
status="failed",
failure_kind=failure_kind,
quota_day=now.date(),
)
for _ in range(failed)
)
self.session.add_all(rows)
self.session.commit()
def test_cold_start_uses_initial_batch_not_configured_ceiling(self):
job = self.job()
job.batch_size = 50
job.config = {}
self.session.commit()
decision = ck_relogin.resolve_adaptive_batch_size(
self.session,
job,
now=datetime.now(UTC),
pending=100,
)
assert decision.batch_size == 10
assert decision.action == "cold_start"
def test_high_recent_success_grows_by_at_most_twenty_five_percent(self):
account = self.make_account("adaptive-success")
self._attempts(account, success=9, failed=1)
job = self.job()
job.batch_size = 50
job.config = {"adaptive_batch_size": 10}
self.session.commit()
decision = ck_relogin.resolve_adaptive_batch_size(
self.session,
job,
now=datetime.now(UTC),
pending=100,
)
assert decision.success_rate == 90.0
assert decision.batch_size == 13
assert decision.action == "grow"
assert job.config["adaptive_batch_size"] == 13
def test_low_recent_success_halves_batch(self):
account = self.make_account("adaptive-shrink")
self._attempts(account, success=2, failed=8)
job = self.job()
job.batch_size = 50
job.config = {"adaptive_batch_size": 20}
self.session.commit()
decision = ck_relogin.resolve_adaptive_batch_size(
self.session,
job,
now=datetime.now(UTC),
pending=100,
)
assert decision.success_rate == 20.0
assert decision.batch_size == 10
assert decision.action == "shrink"
class TestBackoff(_Base):
def test_ladder_is_escalating(self):
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
+57 -1
View File
@@ -403,6 +403,62 @@ def account_platform_status(
}
)
trends = _platform_trends(db, moment)
query_sample = sum(
int(row["success"]) + int(row["failed"])
for row in trends["query"]
)
query_success = sum(int(row["success"]) for row in trends["query"])
query_rate = _rate(query_success, query_sample)
if query_sample >= 20 and query_rate < 80:
alerts.append(
{
"code": "query_success_rate_low",
"severity": "critical" if query_rate < 50 else "warning",
"count": query_sample,
"message": (
f"近 {TREND_DAYS} 天查询成功率 {query_rate:.1f}%"
f"({query_sample} 次已完成查询)"
),
}
)
login_sample = sum(
int(row["success"]) + int(row["failed"])
for row in trends["login"]
)
login_success = sum(int(row["success"]) for row in trends["login"])
login_rate = _rate(login_success, login_sample)
if login_sample >= 10 and login_rate < 60:
alerts.append(
{
"code": "login_success_rate_low",
"severity": "critical" if login_rate < 40 else "warning",
"count": login_sample,
"message": (
f"近 {TREND_DAYS} 天登录成功率 {login_rate:.1f}%"
f"({login_sample} 次已完成尝试)"
),
}
)
relogin_ok = sum(int(row["relogin_ok"]) for row in trends["lifecycle"])
relogin_failed = sum(
int(row["relogin_failed"]) for row in trends["lifecycle"]
)
if relogin_failed >= 5 and relogin_failed > relogin_ok:
alerts.append(
{
"code": "relogin_failure_trend_high",
"severity": "warning",
"count": relogin_failed,
"message": (
f"近 {TREND_DAYS} 天重登失败 {relogin_failed} 次,"
f"成功 {relogin_ok} 次"
),
}
)
return {
"checked_at": moment,
"accounts": {
@@ -425,7 +481,7 @@ def account_platform_status(
},
"supply_targets": supply_targets,
"alerts": alerts,
"trends": _platform_trends(db, moment),
"trends": trends,
}
@@ -13,7 +13,9 @@
from __future__ import annotations
import math
import random
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -21,7 +23,14 @@ from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from ....database import WorkerSessionLocal
from ....models import Account, AutomationJob, LoginQuotaSetting, LoginTask, User
from ....models import (
Account,
AutomationJob,
LoginAttempt,
LoginQuotaSetting,
LoginTask,
User,
)
from ....models import ProxyConfig as ProxyConfigModel
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
from ...account_lifecycle import (
@@ -62,6 +71,23 @@ AUTO_MAX_TOTAL_TIME = 240
QUOTA_JITTER_MINUTES = 30
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
DEFAULT_ADAPTIVE_INITIAL_BATCH = 10
DEFAULT_ADAPTIVE_WINDOW_DAYS = 7
DEFAULT_ADAPTIVE_MIN_SAMPLE = 10
DEFAULT_ADAPTIVE_GROW_RATE = 90.0
DEFAULT_ADAPTIVE_SHRINK_RATE = 70.0
DEFAULT_ADAPTIVE_GROW_FACTOR = 1.25
@dataclass(frozen=True)
class AdaptiveBatchDecision:
batch_size: int
configured_max: int
previous_batch_size: int
success_rate: float
sample_size: int
pending: int
action: str
def _cutoff(now: datetime) -> datetime:
@@ -75,6 +101,7 @@ def build_select_query(
*,
max_attempts: int = 2,
system_user_id: int | None = None,
batch_limit: int | None = None,
):
"""构造本轮重登候选的查询(与取行分开,便于按方言编译做回归)。
@@ -111,12 +138,13 @@ def build_select_query(
pass # 预留:将来若需要限制到某个归属人,在这里加过滤
# 名额先给第 1、2 档(两套还能换 / 还剩一套 / 积分≥300)。
# 同一档里仍按退避时间,避免把还没到点的高价值号提前拉起来。
limit = int(batch_limit if batch_limit is not None else job.batch_size or 0)
return query.order_by(
value_tier_expr().asc(),
Account.relogin_next_at.is_(None).desc(),
Account.relogin_next_at.asc(),
Account.id.asc(),
).limit(max(int(job.batch_size or 0), 1))
).limit(max(limit, 1))
def _load_batch_state(
@@ -271,6 +299,138 @@ def resolve_failure_circuit_config(
return resolved
def _recent_relogin_success(
db: Session,
*,
now: datetime,
window_days: int,
) -> tuple[int, float]:
start_day = now.date() - timedelta(days=max(window_days, 1) - 1)
rows = (
db.query(
LoginAttempt.status,
LoginAttempt.failure_kind,
func.count(LoginAttempt.id),
)
.filter(
LoginAttempt.quota_day >= start_day,
LoginAttempt.category.in_(
("relogin_high_value", "relogin_ordinary")
),
)
.group_by(LoginAttempt.status, LoginAttempt.failure_kind)
.all()
)
success = 0
failed = 0
for status, failure_kind, count in rows:
count_value = int(count or 0)
if status == "success":
success += count_value
elif status in {"failed", "error", "stopped"} and failure_kind != "email_quota":
failed += count_value
sample = success + failed
rate = success * 100 / sample if sample else 0.0
return sample, round(rate, 1)
def resolve_adaptive_batch_size(
db: Session,
job: AutomationJob,
*,
now: datetime,
pending: int,
) -> AdaptiveBatchDecision:
"""Scale the effective relogin batch inside the operator-configured ceiling."""
config = dict(job.config or {})
configured_max = max(int(job.batch_size or 1), 1)
if not bool(config.get("adaptive_batch_enabled", True)):
decision = AdaptiveBatchDecision(
batch_size=configured_max,
configured_max=configured_max,
previous_batch_size=configured_max,
success_rate=0.0,
sample_size=0,
pending=max(0, int(pending)),
action="fixed",
)
config["adaptive_batch_size"] = decision.batch_size
job.config = config
return decision
window_days = max(
1,
int(config.get("adaptive_window_days", DEFAULT_ADAPTIVE_WINDOW_DAYS) or 1),
)
min_sample = max(
1,
int(config.get("adaptive_min_sample", DEFAULT_ADAPTIVE_MIN_SAMPLE) or 1),
)
grow_rate = float(
config.get("adaptive_grow_rate", DEFAULT_ADAPTIVE_GROW_RATE)
)
shrink_rate = float(
config.get("adaptive_shrink_rate", DEFAULT_ADAPTIVE_SHRINK_RATE)
)
grow_factor = max(
1.01,
float(config.get("adaptive_grow_factor", DEFAULT_ADAPTIVE_GROW_FACTOR)),
)
initial = min(
configured_max,
max(
1,
int(
config.get(
"adaptive_initial_batch",
DEFAULT_ADAPTIVE_INITIAL_BATCH,
)
or 1
),
),
)
previous = min(
configured_max,
max(1, int(config.get("adaptive_batch_size", initial) or initial)),
)
sample, success_rate = _recent_relogin_success(
db,
now=now,
window_days=window_days,
)
target = previous
action = "hold"
if sample >= min_sample and success_rate < shrink_rate:
target = max(1, previous // 2)
action = "shrink"
elif sample >= min_sample and success_rate >= grow_rate:
target = min(
configured_max,
max(previous + 1, math.ceil(previous * grow_factor)),
)
action = "grow" if target > previous else "hold"
elif sample < min_sample:
action = "cold_start"
if pending > 0:
target = min(target, max(1, int(pending)))
decision = AdaptiveBatchDecision(
batch_size=max(1, target),
configured_max=configured_max,
previous_batch_size=previous,
success_rate=success_rate,
sample_size=sample,
pending=max(0, int(pending)),
action=action,
)
config["adaptive_batch_size"] = decision.batch_size
config["adaptive_last_success_rate"] = decision.success_rate
config["adaptive_last_sample"] = decision.sample_size
job.config = config
return decision
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
now = datetime.now(UTC)
config = resolve_failure_circuit_config(db, job.config)
@@ -286,8 +446,22 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
"message": "库里没有可用用户,无法以系统身份发起重登",
}
accounts = build_select_query(db, job, now, max_attempts=max_attempts).all()
pending = count_pending_relogin(db, max_attempts=max_attempts)
adaptive = resolve_adaptive_batch_size(
db,
job,
now=now,
pending=pending,
)
accounts = build_select_query(
db,
job,
now,
max_attempts=max_attempts,
batch_limit=adaptive.batch_size,
).all()
if not accounts:
db.commit()
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
selected_account_ids = tuple(int(account.id) for account in accounts)
@@ -370,7 +544,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
job.updated_at = now
parts = [
f"本轮重登 {counters['scanned']} 条",
f"本轮重登 {counters['scanned']} 条(有效批量 {adaptive.batch_size})",
f"成功 {counters['valid']}",
f"失败 {counters['invalid']}",
]
@@ -384,6 +558,11 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
if paused:
parts.append("失败率过高,已自动暂停作业")
if adaptive.action != "hold":
parts.append(
f"批量策略 {adaptive.previous_batch_size}->{adaptive.batch_size}"
f"({adaptive.action},近 7 天成功率 {adaptive.success_rate:.1f}%)"
)
return {
"scanned": counters["scanned"],
"valid": counters["valid"],
@@ -400,6 +579,15 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
"hard_failures": hard_failures,
"hard_failure_ratio": round(hard_failure_ratio, 4),
"paused": paused,
"adaptive_batch": {
"batch_size": adaptive.batch_size,
"configured_max": adaptive.configured_max,
"previous_batch_size": adaptive.previous_batch_size,
"success_rate": adaptive.success_rate,
"sample_size": adaptive.sample_size,
"pending": adaptive.pending,
"action": adaptive.action,
},
},
}