feat(automation): 增加重登熔断与登录预算配置
This commit is contained in:
@@ -60,6 +60,8 @@ AUTO_MAX_TOTAL_TIME = 240
|
||||
|
||||
# 邮箱配额打满后退避到次日零点,再叠加随机抖动,避免整点齐发。
|
||||
QUOTA_JITTER_MINUTES = 30
|
||||
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
|
||||
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
|
||||
|
||||
|
||||
def _cutoff(now: datetime) -> datetime:
|
||||
@@ -223,6 +225,33 @@ def _next_retry_at(kind: str, attempts: int, now: datetime) -> datetime:
|
||||
return now + timedelta(minutes=step)
|
||||
|
||||
|
||||
def evaluate_failure_circuit(
|
||||
counters: dict[str, int],
|
||||
config: dict[str, Any],
|
||||
) -> tuple[bool, int, float]:
|
||||
"""评估硬失败率;邮箱配额满属于预期供给限制,不计入熔断。"""
|
||||
scanned = max(0, int(counters.get("scanned", 0) or 0))
|
||||
invalid = max(0, int(counters.get("invalid", 0) or 0))
|
||||
email_quota = max(0, int(counters.get("email_quota", 0) or 0))
|
||||
hard_failures = max(0, invalid - email_quota)
|
||||
minimum = max(
|
||||
1,
|
||||
int(
|
||||
config.get(
|
||||
"failure_pause_min_scanned",
|
||||
DEFAULT_FAILURE_PAUSE_MIN_SCANNED,
|
||||
)
|
||||
),
|
||||
)
|
||||
raw_ratio = float(
|
||||
config.get("failure_pause_ratio", DEFAULT_FAILURE_PAUSE_RATIO)
|
||||
)
|
||||
threshold = raw_ratio / 100 if raw_ratio > 1 else raw_ratio
|
||||
threshold = max(0.0, min(threshold, 1.0))
|
||||
ratio = hard_failures / scanned if scanned else 0.0
|
||||
return scanned >= minimum and ratio > threshold, hard_failures, ratio
|
||||
|
||||
|
||||
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
config = job.config or {}
|
||||
@@ -313,6 +342,13 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
|
||||
# runner 在另一个会话里提交了结果:必须用新事务读回(见 settle_fresh_results)。
|
||||
counters = settle_fresh_results(batch_id, now=now, max_attempts=max_attempts)
|
||||
paused, hard_failures, hard_failure_ratio = evaluate_failure_circuit(
|
||||
counters,
|
||||
config,
|
||||
)
|
||||
if paused:
|
||||
job.enabled = False
|
||||
job.updated_at = now
|
||||
|
||||
parts = [
|
||||
f"本轮重登 {counters['scanned']} 条",
|
||||
@@ -325,6 +361,10 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
parts.append(f"其中配额满 {counters['email_quota']}(退避到次日)")
|
||||
if counters["manual"]:
|
||||
parts.append(f"转人工 {counters['manual']}")
|
||||
if hard_failures:
|
||||
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
|
||||
if paused:
|
||||
parts.append("失败率过高,已自动暂停作业")
|
||||
return {
|
||||
"scanned": counters["scanned"],
|
||||
"valid": counters["valid"],
|
||||
@@ -338,6 +378,9 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
"manual": counters["manual"],
|
||||
"retry": counters["retry"],
|
||||
"email_quota": counters["email_quota"],
|
||||
"hard_failures": hard_failures,
|
||||
"hard_failure_ratio": round(hard_failure_ratio, 4),
|
||||
"paused": paused,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user