feat(automation): 按成功率灰度扩缩自动重登
This commit is contained in:
+18
-2
@@ -11,7 +11,7 @@
|
||||
|
||||
更新时间:2026-11-21
|
||||
|
||||
整体完成度约 **75%**。
|
||||
整体完成度约 **82%**。
|
||||
|
||||
| 阶段 | 完成度 | 已落地 | 主要缺口 |
|
||||
| --- | ---: | --- | --- |
|
||||
@@ -19,7 +19,7 @@
|
||||
| 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 |
|
||||
| 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 |
|
||||
| 阶段 3:商品与供给 | 100% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警、7 天查询/登录/CK 趋势 | 无 |
|
||||
| 阶段 4:安全恢复自动化 | 50% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效 | 灰度批次自动扩缩和长期趋势告警 |
|
||||
| 阶段 4:安全恢复自动化 | 90% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效,按 7 天成功率自动扩缩批次,长期低成功率告警 | 生产灰度观察与动态预算参数校准 |
|
||||
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
|
||||
|
||||
## 为什么重构
|
||||
@@ -530,6 +530,21 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。
|
||||
- 逐步恢复资料同步
|
||||
- 按成功率和风控调整预算
|
||||
|
||||
自动重登的 `batch_size` 是人工配置的安全上限,实际有效批量独立保存在作业配置中:
|
||||
|
||||
- 冷启动从 `min(batch_size, 10)` 开始;
|
||||
- 最近 7 天完成样本不少于 10 条时参与调整;
|
||||
- 成功率 `>= 90%` 时每次最多增长 25%,但不超过安全上限和积压数;
|
||||
- 成功率 `< 70%` 时批量减半;
|
||||
- 邮箱配额满不计入成功率分母,避免外部供给限制误触发缩容;
|
||||
- 现有单轮硬失败率熔断继续优先级最高,触发后直接暂停作业。
|
||||
|
||||
平台状态还会对长期趋势告警:
|
||||
|
||||
- 近 7 天查询样本不少于 20 且成功率低于 80%;
|
||||
- 近 7 天登录样本不少于 10 且成功率低于 60%;
|
||||
- 近 7 天重登失败至少 5 次且失败数高于成功数。
|
||||
|
||||
### 阶段 5:清理旧逻辑
|
||||
|
||||
删除或降级:
|
||||
@@ -620,3 +635,4 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。
|
||||
| 2026-11-21 | 增加保守默认供给目标、账号平台供给看板和库存/CK/工作队列阈值告警 |
|
||||
| 2026-11-21 | 增加供给目标管理 UI,支持新增、编辑、启停和调整安全库存 |
|
||||
| 2026-11-21 | 完成阶段 3:增加 7 天查询、登录和 CK 生命周期趋势,并补充同步时间索引 |
|
||||
| 2026-11-21 | 阶段 4:自动重登按 7 天成功率灰度扩缩批次,并增加长期低成功率告警 |
|
||||
|
||||
@@ -344,3 +344,63 @@ def test_account_platform_status_builds_seven_day_trends(db):
|
||||
assert login_today["failed"] == 1
|
||||
assert login_today["success_rate"] == 50.0
|
||||
assert lifecycle_today["ck_invalid"] == 1
|
||||
|
||||
|
||||
def test_account_platform_status_alerts_on_persistently_low_trends(db):
|
||||
account = _account("platform-trend-alerts")
|
||||
db.add(account)
|
||||
db.flush()
|
||||
now = datetime.now(UTC)
|
||||
db.add_all(
|
||||
[
|
||||
AccountSyncRun(
|
||||
account_id=account.id,
|
||||
domain="storehome",
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
status="failed",
|
||||
duration_ms=100,
|
||||
)
|
||||
for _ in range(20)
|
||||
]
|
||||
)
|
||||
db.add_all(
|
||||
[
|
||||
LoginAttempt(
|
||||
account_id=account.id,
|
||||
category="relogin_ordinary",
|
||||
trigger_source="automation",
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
status="failed",
|
||||
failure_kind="network",
|
||||
quota_day=now.date(),
|
||||
)
|
||||
for _ in range(10)
|
||||
]
|
||||
)
|
||||
db.add_all(
|
||||
[
|
||||
AccountEvent(
|
||||
account_id=account.id,
|
||||
event_type="relogin_failed",
|
||||
category="auth",
|
||||
occurred_at=now,
|
||||
title="重登失败",
|
||||
source="pytest",
|
||||
actor_type="system",
|
||||
source_ref=f"pytest-relogin-alert:{account.id}:{index}",
|
||||
)
|
||||
for index in range(5)
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
status = account_platform_status(db, now=now)
|
||||
codes = {alert["code"] for alert in status["alerts"]}
|
||||
|
||||
assert {
|
||||
"query_success_rate_low",
|
||||
"login_success_rate_low",
|
||||
"relogin_failure_trend_high",
|
||||
} <= codes
|
||||
|
||||
@@ -23,6 +23,7 @@ from tests.support.db import get_test_engine, make_test_session
|
||||
from web.backend.models import (
|
||||
Account,
|
||||
AutomationJob,
|
||||
LoginAttempt,
|
||||
LoginQuotaSetting,
|
||||
LoginTask,
|
||||
User,
|
||||
@@ -230,6 +231,100 @@ class TestFailureCircuit(_Base):
|
||||
assert paused is False
|
||||
|
||||
|
||||
class TestAdaptiveBatch(_Base):
|
||||
def _attempts(
|
||||
self,
|
||||
account: Account,
|
||||
*,
|
||||
success: int,
|
||||
failed: int,
|
||||
failure_kind: str = "network",
|
||||
):
|
||||
now = datetime.now(UTC)
|
||||
rows = [
|
||||
LoginAttempt(
|
||||
account_id=account.id,
|
||||
category="relogin_ordinary",
|
||||
trigger_source="automation",
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
status="success",
|
||||
quota_day=now.date(),
|
||||
)
|
||||
for _ in range(success)
|
||||
]
|
||||
rows.extend(
|
||||
LoginAttempt(
|
||||
account_id=account.id,
|
||||
category="relogin_ordinary",
|
||||
trigger_source="automation",
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
status="failed",
|
||||
failure_kind=failure_kind,
|
||||
quota_day=now.date(),
|
||||
)
|
||||
for _ in range(failed)
|
||||
)
|
||||
self.session.add_all(rows)
|
||||
self.session.commit()
|
||||
|
||||
def test_cold_start_uses_initial_batch_not_configured_ceiling(self):
|
||||
job = self.job()
|
||||
job.batch_size = 50
|
||||
job.config = {}
|
||||
self.session.commit()
|
||||
|
||||
decision = ck_relogin.resolve_adaptive_batch_size(
|
||||
self.session,
|
||||
job,
|
||||
now=datetime.now(UTC),
|
||||
pending=100,
|
||||
)
|
||||
|
||||
assert decision.batch_size == 10
|
||||
assert decision.action == "cold_start"
|
||||
|
||||
def test_high_recent_success_grows_by_at_most_twenty_five_percent(self):
|
||||
account = self.make_account("adaptive-success")
|
||||
self._attempts(account, success=9, failed=1)
|
||||
job = self.job()
|
||||
job.batch_size = 50
|
||||
job.config = {"adaptive_batch_size": 10}
|
||||
self.session.commit()
|
||||
|
||||
decision = ck_relogin.resolve_adaptive_batch_size(
|
||||
self.session,
|
||||
job,
|
||||
now=datetime.now(UTC),
|
||||
pending=100,
|
||||
)
|
||||
|
||||
assert decision.success_rate == 90.0
|
||||
assert decision.batch_size == 13
|
||||
assert decision.action == "grow"
|
||||
assert job.config["adaptive_batch_size"] == 13
|
||||
|
||||
def test_low_recent_success_halves_batch(self):
|
||||
account = self.make_account("adaptive-shrink")
|
||||
self._attempts(account, success=2, failed=8)
|
||||
job = self.job()
|
||||
job.batch_size = 50
|
||||
job.config = {"adaptive_batch_size": 20}
|
||||
self.session.commit()
|
||||
|
||||
decision = ck_relogin.resolve_adaptive_batch_size(
|
||||
self.session,
|
||||
job,
|
||||
now=datetime.now(UTC),
|
||||
pending=100,
|
||||
)
|
||||
|
||||
assert decision.success_rate == 20.0
|
||||
assert decision.batch_size == 10
|
||||
assert decision.action == "shrink"
|
||||
|
||||
|
||||
class TestBackoff(_Base):
|
||||
def test_ladder_is_escalating(self):
|
||||
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
|
||||
|
||||
@@ -403,6 +403,62 @@ def account_platform_status(
|
||||
}
|
||||
)
|
||||
|
||||
trends = _platform_trends(db, moment)
|
||||
query_sample = sum(
|
||||
int(row["success"]) + int(row["failed"])
|
||||
for row in trends["query"]
|
||||
)
|
||||
query_success = sum(int(row["success"]) for row in trends["query"])
|
||||
query_rate = _rate(query_success, query_sample)
|
||||
if query_sample >= 20 and query_rate < 80:
|
||||
alerts.append(
|
||||
{
|
||||
"code": "query_success_rate_low",
|
||||
"severity": "critical" if query_rate < 50 else "warning",
|
||||
"count": query_sample,
|
||||
"message": (
|
||||
f"近 {TREND_DAYS} 天查询成功率 {query_rate:.1f}%"
|
||||
f"({query_sample} 次已完成查询)"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
login_sample = sum(
|
||||
int(row["success"]) + int(row["failed"])
|
||||
for row in trends["login"]
|
||||
)
|
||||
login_success = sum(int(row["success"]) for row in trends["login"])
|
||||
login_rate = _rate(login_success, login_sample)
|
||||
if login_sample >= 10 and login_rate < 60:
|
||||
alerts.append(
|
||||
{
|
||||
"code": "login_success_rate_low",
|
||||
"severity": "critical" if login_rate < 40 else "warning",
|
||||
"count": login_sample,
|
||||
"message": (
|
||||
f"近 {TREND_DAYS} 天登录成功率 {login_rate:.1f}%"
|
||||
f"({login_sample} 次已完成尝试)"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
relogin_ok = sum(int(row["relogin_ok"]) for row in trends["lifecycle"])
|
||||
relogin_failed = sum(
|
||||
int(row["relogin_failed"]) for row in trends["lifecycle"]
|
||||
)
|
||||
if relogin_failed >= 5 and relogin_failed > relogin_ok:
|
||||
alerts.append(
|
||||
{
|
||||
"code": "relogin_failure_trend_high",
|
||||
"severity": "warning",
|
||||
"count": relogin_failed,
|
||||
"message": (
|
||||
f"近 {TREND_DAYS} 天重登失败 {relogin_failed} 次,"
|
||||
f"成功 {relogin_ok} 次"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"checked_at": moment,
|
||||
"accounts": {
|
||||
@@ -425,7 +481,7 @@ def account_platform_status(
|
||||
},
|
||||
"supply_targets": supply_targets,
|
||||
"alerts": alerts,
|
||||
"trends": _platform_trends(db, moment),
|
||||
"trends": trends,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
@@ -21,7 +23,14 @@ from sqlalchemy import func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....database import WorkerSessionLocal
|
||||
from ....models import Account, AutomationJob, LoginQuotaSetting, LoginTask, User
|
||||
from ....models import (
|
||||
Account,
|
||||
AutomationJob,
|
||||
LoginAttempt,
|
||||
LoginQuotaSetting,
|
||||
LoginTask,
|
||||
User,
|
||||
)
|
||||
from ....models import ProxyConfig as ProxyConfigModel
|
||||
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
|
||||
from ...account_lifecycle import (
|
||||
@@ -62,6 +71,23 @@ AUTO_MAX_TOTAL_TIME = 240
|
||||
QUOTA_JITTER_MINUTES = 30
|
||||
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
|
||||
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
|
||||
DEFAULT_ADAPTIVE_INITIAL_BATCH = 10
|
||||
DEFAULT_ADAPTIVE_WINDOW_DAYS = 7
|
||||
DEFAULT_ADAPTIVE_MIN_SAMPLE = 10
|
||||
DEFAULT_ADAPTIVE_GROW_RATE = 90.0
|
||||
DEFAULT_ADAPTIVE_SHRINK_RATE = 70.0
|
||||
DEFAULT_ADAPTIVE_GROW_FACTOR = 1.25
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AdaptiveBatchDecision:
|
||||
batch_size: int
|
||||
configured_max: int
|
||||
previous_batch_size: int
|
||||
success_rate: float
|
||||
sample_size: int
|
||||
pending: int
|
||||
action: str
|
||||
|
||||
|
||||
def _cutoff(now: datetime) -> datetime:
|
||||
@@ -75,6 +101,7 @@ def build_select_query(
|
||||
*,
|
||||
max_attempts: int = 2,
|
||||
system_user_id: int | None = None,
|
||||
batch_limit: int | None = None,
|
||||
):
|
||||
"""构造本轮重登候选的查询(与取行分开,便于按方言编译做回归)。
|
||||
|
||||
@@ -111,12 +138,13 @@ def build_select_query(
|
||||
pass # 预留:将来若需要限制到某个归属人,在这里加过滤
|
||||
# 名额先给第 1、2 档(两套还能换 / 还剩一套 / 积分≥300)。
|
||||
# 同一档里仍按退避时间,避免把还没到点的高价值号提前拉起来。
|
||||
limit = int(batch_limit if batch_limit is not None else job.batch_size or 0)
|
||||
return query.order_by(
|
||||
value_tier_expr().asc(),
|
||||
Account.relogin_next_at.is_(None).desc(),
|
||||
Account.relogin_next_at.asc(),
|
||||
Account.id.asc(),
|
||||
).limit(max(int(job.batch_size or 0), 1))
|
||||
).limit(max(limit, 1))
|
||||
|
||||
|
||||
def _load_batch_state(
|
||||
@@ -271,6 +299,138 @@ def resolve_failure_circuit_config(
|
||||
return resolved
|
||||
|
||||
|
||||
def _recent_relogin_success(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime,
|
||||
window_days: int,
|
||||
) -> tuple[int, float]:
|
||||
start_day = now.date() - timedelta(days=max(window_days, 1) - 1)
|
||||
rows = (
|
||||
db.query(
|
||||
LoginAttempt.status,
|
||||
LoginAttempt.failure_kind,
|
||||
func.count(LoginAttempt.id),
|
||||
)
|
||||
.filter(
|
||||
LoginAttempt.quota_day >= start_day,
|
||||
LoginAttempt.category.in_(
|
||||
("relogin_high_value", "relogin_ordinary")
|
||||
),
|
||||
)
|
||||
.group_by(LoginAttempt.status, LoginAttempt.failure_kind)
|
||||
.all()
|
||||
)
|
||||
success = 0
|
||||
failed = 0
|
||||
for status, failure_kind, count in rows:
|
||||
count_value = int(count or 0)
|
||||
if status == "success":
|
||||
success += count_value
|
||||
elif status in {"failed", "error", "stopped"} and failure_kind != "email_quota":
|
||||
failed += count_value
|
||||
sample = success + failed
|
||||
rate = success * 100 / sample if sample else 0.0
|
||||
return sample, round(rate, 1)
|
||||
|
||||
|
||||
def resolve_adaptive_batch_size(
|
||||
db: Session,
|
||||
job: AutomationJob,
|
||||
*,
|
||||
now: datetime,
|
||||
pending: int,
|
||||
) -> AdaptiveBatchDecision:
|
||||
"""Scale the effective relogin batch inside the operator-configured ceiling."""
|
||||
config = dict(job.config or {})
|
||||
configured_max = max(int(job.batch_size or 1), 1)
|
||||
if not bool(config.get("adaptive_batch_enabled", True)):
|
||||
decision = AdaptiveBatchDecision(
|
||||
batch_size=configured_max,
|
||||
configured_max=configured_max,
|
||||
previous_batch_size=configured_max,
|
||||
success_rate=0.0,
|
||||
sample_size=0,
|
||||
pending=max(0, int(pending)),
|
||||
action="fixed",
|
||||
)
|
||||
config["adaptive_batch_size"] = decision.batch_size
|
||||
job.config = config
|
||||
return decision
|
||||
|
||||
window_days = max(
|
||||
1,
|
||||
int(config.get("adaptive_window_days", DEFAULT_ADAPTIVE_WINDOW_DAYS) or 1),
|
||||
)
|
||||
min_sample = max(
|
||||
1,
|
||||
int(config.get("adaptive_min_sample", DEFAULT_ADAPTIVE_MIN_SAMPLE) or 1),
|
||||
)
|
||||
grow_rate = float(
|
||||
config.get("adaptive_grow_rate", DEFAULT_ADAPTIVE_GROW_RATE)
|
||||
)
|
||||
shrink_rate = float(
|
||||
config.get("adaptive_shrink_rate", DEFAULT_ADAPTIVE_SHRINK_RATE)
|
||||
)
|
||||
grow_factor = max(
|
||||
1.01,
|
||||
float(config.get("adaptive_grow_factor", DEFAULT_ADAPTIVE_GROW_FACTOR)),
|
||||
)
|
||||
initial = min(
|
||||
configured_max,
|
||||
max(
|
||||
1,
|
||||
int(
|
||||
config.get(
|
||||
"adaptive_initial_batch",
|
||||
DEFAULT_ADAPTIVE_INITIAL_BATCH,
|
||||
)
|
||||
or 1
|
||||
),
|
||||
),
|
||||
)
|
||||
previous = min(
|
||||
configured_max,
|
||||
max(1, int(config.get("adaptive_batch_size", initial) or initial)),
|
||||
)
|
||||
sample, success_rate = _recent_relogin_success(
|
||||
db,
|
||||
now=now,
|
||||
window_days=window_days,
|
||||
)
|
||||
|
||||
target = previous
|
||||
action = "hold"
|
||||
if sample >= min_sample and success_rate < shrink_rate:
|
||||
target = max(1, previous // 2)
|
||||
action = "shrink"
|
||||
elif sample >= min_sample and success_rate >= grow_rate:
|
||||
target = min(
|
||||
configured_max,
|
||||
max(previous + 1, math.ceil(previous * grow_factor)),
|
||||
)
|
||||
action = "grow" if target > previous else "hold"
|
||||
elif sample < min_sample:
|
||||
action = "cold_start"
|
||||
|
||||
if pending > 0:
|
||||
target = min(target, max(1, int(pending)))
|
||||
decision = AdaptiveBatchDecision(
|
||||
batch_size=max(1, target),
|
||||
configured_max=configured_max,
|
||||
previous_batch_size=previous,
|
||||
success_rate=success_rate,
|
||||
sample_size=sample,
|
||||
pending=max(0, int(pending)),
|
||||
action=action,
|
||||
)
|
||||
config["adaptive_batch_size"] = decision.batch_size
|
||||
config["adaptive_last_success_rate"] = decision.success_rate
|
||||
config["adaptive_last_sample"] = decision.sample_size
|
||||
job.config = config
|
||||
return decision
|
||||
|
||||
|
||||
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
config = resolve_failure_circuit_config(db, job.config)
|
||||
@@ -286,8 +446,22 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
"message": "库里没有可用用户,无法以系统身份发起重登",
|
||||
}
|
||||
|
||||
accounts = build_select_query(db, job, now, max_attempts=max_attempts).all()
|
||||
pending = count_pending_relogin(db, max_attempts=max_attempts)
|
||||
adaptive = resolve_adaptive_batch_size(
|
||||
db,
|
||||
job,
|
||||
now=now,
|
||||
pending=pending,
|
||||
)
|
||||
accounts = build_select_query(
|
||||
db,
|
||||
job,
|
||||
now,
|
||||
max_attempts=max_attempts,
|
||||
batch_limit=adaptive.batch_size,
|
||||
).all()
|
||||
if not accounts:
|
||||
db.commit()
|
||||
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
|
||||
|
||||
selected_account_ids = tuple(int(account.id) for account in accounts)
|
||||
@@ -370,7 +544,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
job.updated_at = now
|
||||
|
||||
parts = [
|
||||
f"本轮重登 {counters['scanned']} 条",
|
||||
f"本轮重登 {counters['scanned']} 条(有效批量 {adaptive.batch_size})",
|
||||
f"成功 {counters['valid']}",
|
||||
f"失败 {counters['invalid']}",
|
||||
]
|
||||
@@ -384,6 +558,11 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
|
||||
if paused:
|
||||
parts.append("失败率过高,已自动暂停作业")
|
||||
if adaptive.action != "hold":
|
||||
parts.append(
|
||||
f"批量策略 {adaptive.previous_batch_size}->{adaptive.batch_size}"
|
||||
f"({adaptive.action},近 7 天成功率 {adaptive.success_rate:.1f}%)"
|
||||
)
|
||||
return {
|
||||
"scanned": counters["scanned"],
|
||||
"valid": counters["valid"],
|
||||
@@ -400,6 +579,15 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
"hard_failures": hard_failures,
|
||||
"hard_failure_ratio": round(hard_failure_ratio, 4),
|
||||
"paused": paused,
|
||||
"adaptive_batch": {
|
||||
"batch_size": adaptive.batch_size,
|
||||
"configured_max": adaptive.configured_max,
|
||||
"previous_batch_size": adaptive.previous_batch_size,
|
||||
"success_rate": adaptive.success_rate,
|
||||
"sample_size": adaptive.sample_size,
|
||||
"pending": adaptive.pending,
|
||||
"action": adaptive.action,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user