diff --git a/docs/斗鱼/斗鱼账号平台最终实施设计.md b/docs/斗鱼/斗鱼账号平台最终实施设计.md index 64f9c1d..95be4fc 100644 --- a/docs/斗鱼/斗鱼账号平台最终实施设计.md +++ b/docs/斗鱼/斗鱼账号平台最终实施设计.md @@ -11,15 +11,15 @@ 更新时间:2026-11-21 -整体完成度约 **62%**。 +整体完成度约 **65%**。 | 阶段 | 完成度 | 已落地 | 主要缺口 | | --- | ---: | --- | --- | | 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 | | 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 | -| 阶段 2:账号与 CK | 85% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队 | 更多自动重登灰度指标 | +| 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 | | 阶段 3:商品与供给 | 60% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API、统一平台状态快照 | 默认供给目标、供给看板、阈值告警 | -| 阶段 4:安全恢复自动化 | 30% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列 | 自动重登灰度参数、成功率熔断和动态预算 | +| 阶段 4:安全恢复自动化 | 45% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置 | 灰度批次自动扩缩和长期趋势告警 | | 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 | ## 为什么重构 diff --git a/tests/test_account_platform_foundation.py b/tests/test_account_platform_foundation.py index 567b11d..48ddd4b 100644 --- a/tests/test_account_platform_foundation.py +++ b/tests/test_account_platform_foundation.py @@ -13,6 +13,12 @@ from web.backend.models import ( AccountSyncState, AccountWorkItem, LoginQuotaSetting, + User, +) +from web.backend.routers.automation import ( + LoginQuotaUpdate, + get_login_quota, + update_login_quota, ) from web.backend.services import account_sync_service from web.backend.services.account_lifecycle import on_ck_verdict @@ -446,3 +452,37 @@ def test_login_quota_enforces_per_account_and_hourly_budget(db): assert second_hour.allowed assert not third_hour.allowed assert "每小时" in third_hour.reason + + +def test_login_quota_configuration_route_updates_all_categories(db): + admin = User( + username="quota-admin", + password_hash="hash", + role="super_admin", + is_active=True, + ) + db.add(admin) + db.commit() + + updated = update_login_quota( + LoginQuotaUpdate( + daily_attempt_limit=1200, + hourly_attempt_limit=80, + per_account_daily_limit=3, + onboarding_daily_limit=120, + relogin_high_value_daily_limit=240, + relogin_ordinary_daily_limit=360, + manual_daily_limit=200, + login_concurrency=3, + failure_pause_ratio=35, + enabled=True, + ), + db=db, + current=admin, + ) + loaded = get_login_quota(db=db, current=admin) + + assert updated["daily_attempt_limit"] == 1200 + assert updated["relogin_high_value_daily_limit"] == 240 + assert updated["failure_pause_ratio"] == 35 + assert loaded["login_concurrency"] == 3 diff --git a/tests/test_automation_ck_relogin.py b/tests/test_automation_ck_relogin.py index 8838d52..c1e585a 100644 --- a/tests/test_automation_ck_relogin.py +++ b/tests/test_automation_ck_relogin.py @@ -161,6 +161,48 @@ class TestFailureClassification(_Base): assert ck_relogin.classify_failure("") == "network" +class TestFailureCircuit(_Base): + def test_hard_failure_ratio_pauses_job(self): + paused, hard, ratio = ck_relogin.evaluate_failure_circuit( + { + "scanned": 10, + "invalid": 7, + "email_quota": 0, + }, + {}, + ) + + assert paused is True + assert hard == 7 + assert ratio == 0.7 + + def test_email_quota_does_not_trigger_circuit(self): + paused, hard, ratio = ck_relogin.evaluate_failure_circuit( + { + "scanned": 10, + "invalid": 10, + "email_quota": 10, + }, + {}, + ) + + assert paused is False + assert hard == 0 + assert ratio == 0 + + def test_minimum_sample_size_is_respected(self): + paused, _, _ = ck_relogin.evaluate_failure_circuit( + { + "scanned": 5, + "invalid": 5, + "email_quota": 0, + }, + {"failure_pause_min_scanned": 10}, + ) + + assert paused is False + + class TestBackoff(_Base): def test_ladder_is_escalating(self): now = datetime(2026, 10, 2, 12, 0, tzinfo=UTC) diff --git a/web/backend/routers/automation.py b/web/backend/routers/automation.py index 0d3f896..7c6322f 100644 --- a/web/backend/routers/automation.py +++ b/web/backend/routers/automation.py @@ -17,7 +17,14 @@ from sqlalchemy.orm import Session from ..database import get_db from ..deps import require_permission -from ..models import Account, AutomationJob, AutomationRun, LoginTask, User +from ..models import ( + Account, + AutomationJob, + AutomationRun, + LoginQuotaSetting, + LoginTask, + User, +) from ..services.audit_service import record_audit from ..services.automation import ( ensure_jobs, @@ -43,6 +50,39 @@ class JobUpdate(BaseModel): concurrency: int | None = Field(default=None, ge=1, le=16) +class LoginQuotaUpdate(BaseModel): + daily_attempt_limit: int = Field(..., ge=1, le=1_000_000) + hourly_attempt_limit: int = Field(..., ge=1, le=100_000) + per_account_daily_limit: int = Field(..., ge=1, le=100) + onboarding_daily_limit: int = Field(..., ge=0, le=100_000) + relogin_high_value_daily_limit: int = Field(..., ge=0, le=100_000) + relogin_ordinary_daily_limit: int = Field(..., ge=0, le=100_000) + manual_daily_limit: int = Field(..., ge=0, le=100_000) + login_concurrency: int = Field(..., ge=1, le=10) + failure_pause_ratio: int = Field(..., ge=0, le=100) + enabled: bool = True + + +def _login_quota_out(row: LoginQuotaSetting) -> dict[str, Any]: + return { + "daily_attempt_limit": int(row.daily_attempt_limit), + "hourly_attempt_limit": int(row.hourly_attempt_limit), + "per_account_daily_limit": int(row.per_account_daily_limit), + "onboarding_daily_limit": int(row.onboarding_daily_limit), + "relogin_high_value_daily_limit": int( + row.relogin_high_value_daily_limit + ), + "relogin_ordinary_daily_limit": int( + row.relogin_ordinary_daily_limit + ), + "manual_daily_limit": int(row.manual_daily_limit), + "login_concurrency": int(row.login_concurrency), + "failure_pause_ratio": int(row.failure_pause_ratio), + "enabled": bool(row.enabled), + "updated_at": row.updated_at.isoformat() if row.updated_at else None, + } + + def _job_out(job: AutomationJob) -> dict[str, Any]: return { "key": job.key, @@ -125,6 +165,58 @@ def list_jobs( } +@router.get("/login-quota") +def get_login_quota( + db: Session = Depends(get_db), + current: User = Depends(require_permission("automation:view")), +): + """查看登录与自动重登的数据库级预算。""" + row = db.query(LoginQuotaSetting).filter(LoginQuotaSetting.id == 1).first() + if row is None: + row = LoginQuotaSetting(id=1) + db.add(row) + db.commit() + return _login_quota_out(row) + + +@router.put("/login-quota") +def update_login_quota( + payload: LoginQuotaUpdate, + db: Session = Depends(get_db), + current: User = Depends(require_permission("automation:manage")), +): + """更新登录、首次登录、重登和账号检测预算。""" + row = db.query(LoginQuotaSetting).filter(LoginQuotaSetting.id == 1).first() + if row is None: + row = LoginQuotaSetting(id=1) + db.add(row) + before = _login_quota_out(row) if row.updated_at else {} + for field in ( + "daily_attempt_limit", + "hourly_attempt_limit", + "per_account_daily_limit", + "onboarding_daily_limit", + "relogin_high_value_daily_limit", + "relogin_ordinary_daily_limit", + "manual_daily_limit", + "login_concurrency", + "failure_pause_ratio", + "enabled", + ): + setattr(row, field, getattr(payload, field)) + row.updated_by = current.id + row.updated_at = datetime.now(UTC) + record_audit( + db, + current, + action="automation:login-quota:update", + target="login_quota_settings:1", + detail={"before": before, "after": _login_quota_out(row)}, + ) + db.commit() + return _login_quota_out(row) + + @router.patch("/jobs/{key}") def update_job( key: str, diff --git a/web/backend/services/automation/jobs/ck_relogin.py b/web/backend/services/automation/jobs/ck_relogin.py index 21e023d..312d62c 100644 --- a/web/backend/services/automation/jobs/ck_relogin.py +++ b/web/backend/services/automation/jobs/ck_relogin.py @@ -60,6 +60,8 @@ AUTO_MAX_TOTAL_TIME = 240 # 邮箱配额打满后退避到次日零点,再叠加随机抖动,避免整点齐发。 QUOTA_JITTER_MINUTES = 30 +DEFAULT_FAILURE_PAUSE_RATIO = 0.4 +DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10 def _cutoff(now: datetime) -> datetime: @@ -223,6 +225,33 @@ def _next_retry_at(kind: str, attempts: int, now: datetime) -> datetime: return now + timedelta(minutes=step) +def evaluate_failure_circuit( + counters: dict[str, int], + config: dict[str, Any], +) -> tuple[bool, int, float]: + """评估硬失败率;邮箱配额满属于预期供给限制,不计入熔断。""" + scanned = max(0, int(counters.get("scanned", 0) or 0)) + invalid = max(0, int(counters.get("invalid", 0) or 0)) + email_quota = max(0, int(counters.get("email_quota", 0) or 0)) + hard_failures = max(0, invalid - email_quota) + minimum = max( + 1, + int( + config.get( + "failure_pause_min_scanned", + DEFAULT_FAILURE_PAUSE_MIN_SCANNED, + ) + ), + ) + raw_ratio = float( + config.get("failure_pause_ratio", DEFAULT_FAILURE_PAUSE_RATIO) + ) + threshold = raw_ratio / 100 if raw_ratio > 1 else raw_ratio + threshold = max(0.0, min(threshold, 1.0)) + ratio = hard_failures / scanned if scanned else 0.0 + return scanned >= minimum and ratio > threshold, hard_failures, ratio + + def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: now = datetime.now(UTC) config = job.config or {} @@ -313,6 +342,13 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: # runner 在另一个会话里提交了结果:必须用新事务读回(见 settle_fresh_results)。 counters = settle_fresh_results(batch_id, now=now, max_attempts=max_attempts) + paused, hard_failures, hard_failure_ratio = evaluate_failure_circuit( + counters, + config, + ) + if paused: + job.enabled = False + job.updated_at = now parts = [ f"本轮重登 {counters['scanned']} 条", @@ -325,6 +361,10 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: parts.append(f"其中配额满 {counters['email_quota']}(退避到次日)") if counters["manual"]: parts.append(f"转人工 {counters['manual']}") + if hard_failures: + parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})") + if paused: + parts.append("失败率过高,已自动暂停作业") return { "scanned": counters["scanned"], "valid": counters["valid"], @@ -338,6 +378,9 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: "manual": counters["manual"], "retry": counters["retry"], "email_quota": counters["email_quota"], + "hard_failures": hard_failures, + "hard_failure_ratio": round(hard_failure_ratio, 4), + "paused": paused, }, }