feat(automation): 增加重登熔断与登录预算配置
This commit is contained in:
@@ -11,15 +11,15 @@
|
||||
|
||||
更新时间:2026-11-21
|
||||
|
||||
整体完成度约 **62%**。
|
||||
整体完成度约 **65%**。
|
||||
|
||||
| 阶段 | 完成度 | 已落地 | 主要缺口 |
|
||||
| --- | ---: | --- | --- |
|
||||
| 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 |
|
||||
| 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 |
|
||||
| 阶段 2:账号与 CK | 85% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队 | 更多自动重登灰度指标 |
|
||||
| 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 |
|
||||
| 阶段 3:商品与供给 | 60% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API、统一平台状态快照 | 默认供给目标、供给看板、阈值告警 |
|
||||
| 阶段 4:安全恢复自动化 | 30% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列 | 自动重登灰度参数、成功率熔断和动态预算 |
|
||||
| 阶段 4:安全恢复自动化 | 45% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置 | 灰度批次自动扩缩和长期趋势告警 |
|
||||
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
|
||||
|
||||
## 为什么重构
|
||||
|
||||
@@ -13,6 +13,12 @@ from web.backend.models import (
|
||||
AccountSyncState,
|
||||
AccountWorkItem,
|
||||
LoginQuotaSetting,
|
||||
User,
|
||||
)
|
||||
from web.backend.routers.automation import (
|
||||
LoginQuotaUpdate,
|
||||
get_login_quota,
|
||||
update_login_quota,
|
||||
)
|
||||
from web.backend.services import account_sync_service
|
||||
from web.backend.services.account_lifecycle import on_ck_verdict
|
||||
@@ -446,3 +452,37 @@ def test_login_quota_enforces_per_account_and_hourly_budget(db):
|
||||
assert second_hour.allowed
|
||||
assert not third_hour.allowed
|
||||
assert "每小时" in third_hour.reason
|
||||
|
||||
|
||||
def test_login_quota_configuration_route_updates_all_categories(db):
|
||||
admin = User(
|
||||
username="quota-admin",
|
||||
password_hash="hash",
|
||||
role="super_admin",
|
||||
is_active=True,
|
||||
)
|
||||
db.add(admin)
|
||||
db.commit()
|
||||
|
||||
updated = update_login_quota(
|
||||
LoginQuotaUpdate(
|
||||
daily_attempt_limit=1200,
|
||||
hourly_attempt_limit=80,
|
||||
per_account_daily_limit=3,
|
||||
onboarding_daily_limit=120,
|
||||
relogin_high_value_daily_limit=240,
|
||||
relogin_ordinary_daily_limit=360,
|
||||
manual_daily_limit=200,
|
||||
login_concurrency=3,
|
||||
failure_pause_ratio=35,
|
||||
enabled=True,
|
||||
),
|
||||
db=db,
|
||||
current=admin,
|
||||
)
|
||||
loaded = get_login_quota(db=db, current=admin)
|
||||
|
||||
assert updated["daily_attempt_limit"] == 1200
|
||||
assert updated["relogin_high_value_daily_limit"] == 240
|
||||
assert updated["failure_pause_ratio"] == 35
|
||||
assert loaded["login_concurrency"] == 3
|
||||
|
||||
@@ -161,6 +161,48 @@ class TestFailureClassification(_Base):
|
||||
assert ck_relogin.classify_failure("") == "network"
|
||||
|
||||
|
||||
class TestFailureCircuit(_Base):
|
||||
def test_hard_failure_ratio_pauses_job(self):
|
||||
paused, hard, ratio = ck_relogin.evaluate_failure_circuit(
|
||||
{
|
||||
"scanned": 10,
|
||||
"invalid": 7,
|
||||
"email_quota": 0,
|
||||
},
|
||||
{},
|
||||
)
|
||||
|
||||
assert paused is True
|
||||
assert hard == 7
|
||||
assert ratio == 0.7
|
||||
|
||||
def test_email_quota_does_not_trigger_circuit(self):
|
||||
paused, hard, ratio = ck_relogin.evaluate_failure_circuit(
|
||||
{
|
||||
"scanned": 10,
|
||||
"invalid": 10,
|
||||
"email_quota": 10,
|
||||
},
|
||||
{},
|
||||
)
|
||||
|
||||
assert paused is False
|
||||
assert hard == 0
|
||||
assert ratio == 0
|
||||
|
||||
def test_minimum_sample_size_is_respected(self):
|
||||
paused, _, _ = ck_relogin.evaluate_failure_circuit(
|
||||
{
|
||||
"scanned": 5,
|
||||
"invalid": 5,
|
||||
"email_quota": 0,
|
||||
},
|
||||
{"failure_pause_min_scanned": 10},
|
||||
)
|
||||
|
||||
assert paused is False
|
||||
|
||||
|
||||
class TestBackoff(_Base):
|
||||
def test_ladder_is_escalating(self):
|
||||
now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
|
||||
|
||||
@@ -17,7 +17,14 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import get_db
|
||||
from ..deps import require_permission
|
||||
from ..models import Account, AutomationJob, AutomationRun, LoginTask, User
|
||||
from ..models import (
|
||||
Account,
|
||||
AutomationJob,
|
||||
AutomationRun,
|
||||
LoginQuotaSetting,
|
||||
LoginTask,
|
||||
User,
|
||||
)
|
||||
from ..services.audit_service import record_audit
|
||||
from ..services.automation import (
|
||||
ensure_jobs,
|
||||
@@ -43,6 +50,39 @@ class JobUpdate(BaseModel):
|
||||
concurrency: int | None = Field(default=None, ge=1, le=16)
|
||||
|
||||
|
||||
class LoginQuotaUpdate(BaseModel):
|
||||
daily_attempt_limit: int = Field(..., ge=1, le=1_000_000)
|
||||
hourly_attempt_limit: int = Field(..., ge=1, le=100_000)
|
||||
per_account_daily_limit: int = Field(..., ge=1, le=100)
|
||||
onboarding_daily_limit: int = Field(..., ge=0, le=100_000)
|
||||
relogin_high_value_daily_limit: int = Field(..., ge=0, le=100_000)
|
||||
relogin_ordinary_daily_limit: int = Field(..., ge=0, le=100_000)
|
||||
manual_daily_limit: int = Field(..., ge=0, le=100_000)
|
||||
login_concurrency: int = Field(..., ge=1, le=10)
|
||||
failure_pause_ratio: int = Field(..., ge=0, le=100)
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
def _login_quota_out(row: LoginQuotaSetting) -> dict[str, Any]:
|
||||
return {
|
||||
"daily_attempt_limit": int(row.daily_attempt_limit),
|
||||
"hourly_attempt_limit": int(row.hourly_attempt_limit),
|
||||
"per_account_daily_limit": int(row.per_account_daily_limit),
|
||||
"onboarding_daily_limit": int(row.onboarding_daily_limit),
|
||||
"relogin_high_value_daily_limit": int(
|
||||
row.relogin_high_value_daily_limit
|
||||
),
|
||||
"relogin_ordinary_daily_limit": int(
|
||||
row.relogin_ordinary_daily_limit
|
||||
),
|
||||
"manual_daily_limit": int(row.manual_daily_limit),
|
||||
"login_concurrency": int(row.login_concurrency),
|
||||
"failure_pause_ratio": int(row.failure_pause_ratio),
|
||||
"enabled": bool(row.enabled),
|
||||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _job_out(job: AutomationJob) -> dict[str, Any]:
|
||||
return {
|
||||
"key": job.key,
|
||||
@@ -125,6 +165,58 @@ def list_jobs(
|
||||
}
|
||||
|
||||
|
||||
@router.get("/login-quota")
|
||||
def get_login_quota(
|
||||
db: Session = Depends(get_db),
|
||||
current: User = Depends(require_permission("automation:view")),
|
||||
):
|
||||
"""查看登录与自动重登的数据库级预算。"""
|
||||
row = db.query(LoginQuotaSetting).filter(LoginQuotaSetting.id == 1).first()
|
||||
if row is None:
|
||||
row = LoginQuotaSetting(id=1)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
return _login_quota_out(row)
|
||||
|
||||
|
||||
@router.put("/login-quota")
|
||||
def update_login_quota(
|
||||
payload: LoginQuotaUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
current: User = Depends(require_permission("automation:manage")),
|
||||
):
|
||||
"""更新登录、首次登录、重登和账号检测预算。"""
|
||||
row = db.query(LoginQuotaSetting).filter(LoginQuotaSetting.id == 1).first()
|
||||
if row is None:
|
||||
row = LoginQuotaSetting(id=1)
|
||||
db.add(row)
|
||||
before = _login_quota_out(row) if row.updated_at else {}
|
||||
for field in (
|
||||
"daily_attempt_limit",
|
||||
"hourly_attempt_limit",
|
||||
"per_account_daily_limit",
|
||||
"onboarding_daily_limit",
|
||||
"relogin_high_value_daily_limit",
|
||||
"relogin_ordinary_daily_limit",
|
||||
"manual_daily_limit",
|
||||
"login_concurrency",
|
||||
"failure_pause_ratio",
|
||||
"enabled",
|
||||
):
|
||||
setattr(row, field, getattr(payload, field))
|
||||
row.updated_by = current.id
|
||||
row.updated_at = datetime.now(UTC)
|
||||
record_audit(
|
||||
db,
|
||||
current,
|
||||
action="automation:login-quota:update",
|
||||
target="login_quota_settings:1",
|
||||
detail={"before": before, "after": _login_quota_out(row)},
|
||||
)
|
||||
db.commit()
|
||||
return _login_quota_out(row)
|
||||
|
||||
|
||||
@router.patch("/jobs/{key}")
|
||||
def update_job(
|
||||
key: str,
|
||||
|
||||
@@ -60,6 +60,8 @@ AUTO_MAX_TOTAL_TIME = 240
|
||||
|
||||
# 邮箱配额打满后退避到次日零点,再叠加随机抖动,避免整点齐发。
|
||||
QUOTA_JITTER_MINUTES = 30
|
||||
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
|
||||
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
|
||||
|
||||
|
||||
def _cutoff(now: datetime) -> datetime:
|
||||
@@ -223,6 +225,33 @@ def _next_retry_at(kind: str, attempts: int, now: datetime) -> datetime:
|
||||
return now + timedelta(minutes=step)
|
||||
|
||||
|
||||
def evaluate_failure_circuit(
|
||||
counters: dict[str, int],
|
||||
config: dict[str, Any],
|
||||
) -> tuple[bool, int, float]:
|
||||
"""评估硬失败率;邮箱配额满属于预期供给限制,不计入熔断。"""
|
||||
scanned = max(0, int(counters.get("scanned", 0) or 0))
|
||||
invalid = max(0, int(counters.get("invalid", 0) or 0))
|
||||
email_quota = max(0, int(counters.get("email_quota", 0) or 0))
|
||||
hard_failures = max(0, invalid - email_quota)
|
||||
minimum = max(
|
||||
1,
|
||||
int(
|
||||
config.get(
|
||||
"failure_pause_min_scanned",
|
||||
DEFAULT_FAILURE_PAUSE_MIN_SCANNED,
|
||||
)
|
||||
),
|
||||
)
|
||||
raw_ratio = float(
|
||||
config.get("failure_pause_ratio", DEFAULT_FAILURE_PAUSE_RATIO)
|
||||
)
|
||||
threshold = raw_ratio / 100 if raw_ratio > 1 else raw_ratio
|
||||
threshold = max(0.0, min(threshold, 1.0))
|
||||
ratio = hard_failures / scanned if scanned else 0.0
|
||||
return scanned >= minimum and ratio > threshold, hard_failures, ratio
|
||||
|
||||
|
||||
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
config = job.config or {}
|
||||
@@ -313,6 +342,13 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
|
||||
# runner 在另一个会话里提交了结果:必须用新事务读回(见 settle_fresh_results)。
|
||||
counters = settle_fresh_results(batch_id, now=now, max_attempts=max_attempts)
|
||||
paused, hard_failures, hard_failure_ratio = evaluate_failure_circuit(
|
||||
counters,
|
||||
config,
|
||||
)
|
||||
if paused:
|
||||
job.enabled = False
|
||||
job.updated_at = now
|
||||
|
||||
parts = [
|
||||
f"本轮重登 {counters['scanned']} 条",
|
||||
@@ -325,6 +361,10 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
parts.append(f"其中配额满 {counters['email_quota']}(退避到次日)")
|
||||
if counters["manual"]:
|
||||
parts.append(f"转人工 {counters['manual']}")
|
||||
if hard_failures:
|
||||
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
|
||||
if paused:
|
||||
parts.append("失败率过高,已自动暂停作业")
|
||||
return {
|
||||
"scanned": counters["scanned"],
|
||||
"valid": counters["valid"],
|
||||
@@ -338,6 +378,9 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
"manual": counters["manual"],
|
||||
"retry": counters["retry"],
|
||||
"email_quota": counters["email_quota"],
|
||||
"hard_failures": hard_failures,
|
||||
"hard_failure_ratio": round(hard_failure_ratio, 4),
|
||||
"paused": paused,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user