feat(automation): 增加重登熔断与登录预算配置

This commit is contained in:
yml
2026-11-21 22:01:34 +08:00
parent 3b2d4f89fc
commit 7172ed2f94
5 changed files with 221 additions and 4 deletions
+93 -1
View File
@@ -17,7 +17,14 @@ from sqlalchemy.orm import Session
from ..database import get_db
from ..deps import require_permission
from ..models import Account, AutomationJob, AutomationRun, LoginTask, User
from ..models import (
Account,
AutomationJob,
AutomationRun,
LoginQuotaSetting,
LoginTask,
User,
)
from ..services.audit_service import record_audit
from ..services.automation import (
ensure_jobs,
@@ -43,6 +50,39 @@ class JobUpdate(BaseModel):
concurrency: int | None = Field(default=None, ge=1, le=16)
class LoginQuotaUpdate(BaseModel):
daily_attempt_limit: int = Field(..., ge=1, le=1_000_000)
hourly_attempt_limit: int = Field(..., ge=1, le=100_000)
per_account_daily_limit: int = Field(..., ge=1, le=100)
onboarding_daily_limit: int = Field(..., ge=0, le=100_000)
relogin_high_value_daily_limit: int = Field(..., ge=0, le=100_000)
relogin_ordinary_daily_limit: int = Field(..., ge=0, le=100_000)
manual_daily_limit: int = Field(..., ge=0, le=100_000)
login_concurrency: int = Field(..., ge=1, le=10)
failure_pause_ratio: int = Field(..., ge=0, le=100)
enabled: bool = True
def _login_quota_out(row: LoginQuotaSetting) -> dict[str, Any]:
return {
"daily_attempt_limit": int(row.daily_attempt_limit),
"hourly_attempt_limit": int(row.hourly_attempt_limit),
"per_account_daily_limit": int(row.per_account_daily_limit),
"onboarding_daily_limit": int(row.onboarding_daily_limit),
"relogin_high_value_daily_limit": int(
row.relogin_high_value_daily_limit
),
"relogin_ordinary_daily_limit": int(
row.relogin_ordinary_daily_limit
),
"manual_daily_limit": int(row.manual_daily_limit),
"login_concurrency": int(row.login_concurrency),
"failure_pause_ratio": int(row.failure_pause_ratio),
"enabled": bool(row.enabled),
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
}
def _job_out(job: AutomationJob) -> dict[str, Any]:
return {
"key": job.key,
@@ -125,6 +165,58 @@ def list_jobs(
}
@router.get("/login-quota")
def get_login_quota(
db: Session = Depends(get_db),
current: User = Depends(require_permission("automation:view")),
):
"""查看登录与自动重登的数据库级预算。"""
row = db.query(LoginQuotaSetting).filter(LoginQuotaSetting.id == 1).first()
if row is None:
row = LoginQuotaSetting(id=1)
db.add(row)
db.commit()
return _login_quota_out(row)
@router.put("/login-quota")
def update_login_quota(
payload: LoginQuotaUpdate,
db: Session = Depends(get_db),
current: User = Depends(require_permission("automation:manage")),
):
"""更新登录、首次登录、重登和账号检测预算。"""
row = db.query(LoginQuotaSetting).filter(LoginQuotaSetting.id == 1).first()
if row is None:
row = LoginQuotaSetting(id=1)
db.add(row)
before = _login_quota_out(row) if row.updated_at else {}
for field in (
"daily_attempt_limit",
"hourly_attempt_limit",
"per_account_daily_limit",
"onboarding_daily_limit",
"relogin_high_value_daily_limit",
"relogin_ordinary_daily_limit",
"manual_daily_limit",
"login_concurrency",
"failure_pause_ratio",
"enabled",
):
setattr(row, field, getattr(payload, field))
row.updated_by = current.id
row.updated_at = datetime.now(UTC)
record_audit(
db,
current,
action="automation:login-quota:update",
target="login_quota_settings:1",
detail={"before": before, "after": _login_quota_out(row)},
)
db.commit()
return _login_quota_out(row)
@router.patch("/jobs/{key}")
def update_job(
key: str,
@@ -60,6 +60,8 @@ AUTO_MAX_TOTAL_TIME = 240
# 邮箱配额打满后退避到次日零点,再叠加随机抖动,避免整点齐发。
QUOTA_JITTER_MINUTES = 30
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
def _cutoff(now: datetime) -> datetime:
@@ -223,6 +225,33 @@ def _next_retry_at(kind: str, attempts: int, now: datetime) -> datetime:
return now + timedelta(minutes=step)
def evaluate_failure_circuit(
counters: dict[str, int],
config: dict[str, Any],
) -> tuple[bool, int, float]:
"""评估硬失败率;邮箱配额满属于预期供给限制,不计入熔断。"""
scanned = max(0, int(counters.get("scanned", 0) or 0))
invalid = max(0, int(counters.get("invalid", 0) or 0))
email_quota = max(0, int(counters.get("email_quota", 0) or 0))
hard_failures = max(0, invalid - email_quota)
minimum = max(
1,
int(
config.get(
"failure_pause_min_scanned",
DEFAULT_FAILURE_PAUSE_MIN_SCANNED,
)
),
)
raw_ratio = float(
config.get("failure_pause_ratio", DEFAULT_FAILURE_PAUSE_RATIO)
)
threshold = raw_ratio / 100 if raw_ratio > 1 else raw_ratio
threshold = max(0.0, min(threshold, 1.0))
ratio = hard_failures / scanned if scanned else 0.0
return scanned >= minimum and ratio > threshold, hard_failures, ratio
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
now = datetime.now(UTC)
config = job.config or {}
@@ -313,6 +342,13 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
# runner 在另一个会话里提交了结果:必须用新事务读回(见 settle_fresh_results)。
counters = settle_fresh_results(batch_id, now=now, max_attempts=max_attempts)
paused, hard_failures, hard_failure_ratio = evaluate_failure_circuit(
counters,
config,
)
if paused:
job.enabled = False
job.updated_at = now
parts = [
f"本轮重登 {counters['scanned']} 条",
@@ -325,6 +361,10 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
parts.append(f"其中配额满 {counters['email_quota']}(退避到次日)")
if counters["manual"]:
parts.append(f"转人工 {counters['manual']}")
if hard_failures:
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
if paused:
parts.append("失败率过高,已自动暂停作业")
return {
"scanned": counters["scanned"],
"valid": counters["valid"],
@@ -338,6 +378,9 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
"manual": counters["manual"],
"retry": counters["retry"],
"email_quota": counters["email_quota"],
"hard_failures": hard_failures,
"hard_failure_ratio": round(hard_failure_ratio, 4),
"paused": paused,
},
}