feat(automation): 按成功率灰度扩缩自动重登

This commit is contained in:
yml
2026-11-21 23:04:08 +08:00
parent 3e618c1a88
commit 5fd7df27dc
5 changed files with 422 additions and 7 deletions
+57 -1
View File
@@ -403,6 +403,62 @@ def account_platform_status(
}
)
trends = _platform_trends(db, moment)
query_sample = sum(
int(row["success"]) + int(row["failed"])
for row in trends["query"]
)
query_success = sum(int(row["success"]) for row in trends["query"])
query_rate = _rate(query_success, query_sample)
if query_sample >= 20 and query_rate < 80:
alerts.append(
{
"code": "query_success_rate_low",
"severity": "critical" if query_rate < 50 else "warning",
"count": query_sample,
"message": (
f"近 {TREND_DAYS} 天查询成功率 {query_rate:.1f}%"
f"({query_sample} 次已完成查询)"
),
}
)
login_sample = sum(
int(row["success"]) + int(row["failed"])
for row in trends["login"]
)
login_success = sum(int(row["success"]) for row in trends["login"])
login_rate = _rate(login_success, login_sample)
if login_sample >= 10 and login_rate < 60:
alerts.append(
{
"code": "login_success_rate_low",
"severity": "critical" if login_rate < 40 else "warning",
"count": login_sample,
"message": (
f"近 {TREND_DAYS} 天登录成功率 {login_rate:.1f}%"
f"({login_sample} 次已完成尝试)"
),
}
)
relogin_ok = sum(int(row["relogin_ok"]) for row in trends["lifecycle"])
relogin_failed = sum(
int(row["relogin_failed"]) for row in trends["lifecycle"]
)
if relogin_failed >= 5 and relogin_failed > relogin_ok:
alerts.append(
{
"code": "relogin_failure_trend_high",
"severity": "warning",
"count": relogin_failed,
"message": (
f"近 {TREND_DAYS} 天重登失败 {relogin_failed} 次,"
f"成功 {relogin_ok} 次"
),
}
)
return {
"checked_at": moment,
"accounts": {
@@ -425,7 +481,7 @@ def account_platform_status(
},
"supply_targets": supply_targets,
"alerts": alerts,
"trends": _platform_trends(db, moment),
"trends": trends,
}
@@ -13,7 +13,9 @@
from __future__ import annotations
import math
import random
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -21,7 +23,14 @@ from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from ....database import WorkerSessionLocal
from ....models import Account, AutomationJob, LoginQuotaSetting, LoginTask, User
from ....models import (
Account,
AutomationJob,
LoginAttempt,
LoginQuotaSetting,
LoginTask,
User,
)
from ....models import ProxyConfig as ProxyConfigModel
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
from ...account_lifecycle import (
@@ -62,6 +71,23 @@ AUTO_MAX_TOTAL_TIME = 240
QUOTA_JITTER_MINUTES = 30
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
DEFAULT_ADAPTIVE_INITIAL_BATCH = 10
DEFAULT_ADAPTIVE_WINDOW_DAYS = 7
DEFAULT_ADAPTIVE_MIN_SAMPLE = 10
DEFAULT_ADAPTIVE_GROW_RATE = 90.0
DEFAULT_ADAPTIVE_SHRINK_RATE = 70.0
DEFAULT_ADAPTIVE_GROW_FACTOR = 1.25
@dataclass(frozen=True)
class AdaptiveBatchDecision:
batch_size: int
configured_max: int
previous_batch_size: int
success_rate: float
sample_size: int
pending: int
action: str
def _cutoff(now: datetime) -> datetime:
@@ -75,6 +101,7 @@ def build_select_query(
*,
max_attempts: int = 2,
system_user_id: int | None = None,
batch_limit: int | None = None,
):
"""构造本轮重登候选的查询(与取行分开,便于按方言编译做回归)。
@@ -111,12 +138,13 @@ def build_select_query(
pass # 预留:将来若需要限制到某个归属人,在这里加过滤
# 名额先给第 1、2 档(两套还能换 / 还剩一套 / 积分≥300)。
# 同一档里仍按退避时间,避免把还没到点的高价值号提前拉起来。
limit = int(batch_limit if batch_limit is not None else job.batch_size or 0)
return query.order_by(
value_tier_expr().asc(),
Account.relogin_next_at.is_(None).desc(),
Account.relogin_next_at.asc(),
Account.id.asc(),
).limit(max(int(job.batch_size or 0), 1))
).limit(max(limit, 1))
def _load_batch_state(
@@ -271,6 +299,138 @@ def resolve_failure_circuit_config(
return resolved
def _recent_relogin_success(
db: Session,
*,
now: datetime,
window_days: int,
) -> tuple[int, float]:
start_day = now.date() - timedelta(days=max(window_days, 1) - 1)
rows = (
db.query(
LoginAttempt.status,
LoginAttempt.failure_kind,
func.count(LoginAttempt.id),
)
.filter(
LoginAttempt.quota_day >= start_day,
LoginAttempt.category.in_(
("relogin_high_value", "relogin_ordinary")
),
)
.group_by(LoginAttempt.status, LoginAttempt.failure_kind)
.all()
)
success = 0
failed = 0
for status, failure_kind, count in rows:
count_value = int(count or 0)
if status == "success":
success += count_value
elif status in {"failed", "error", "stopped"} and failure_kind != "email_quota":
failed += count_value
sample = success + failed
rate = success * 100 / sample if sample else 0.0
return sample, round(rate, 1)
def resolve_adaptive_batch_size(
db: Session,
job: AutomationJob,
*,
now: datetime,
pending: int,
) -> AdaptiveBatchDecision:
"""Scale the effective relogin batch inside the operator-configured ceiling."""
config = dict(job.config or {})
configured_max = max(int(job.batch_size or 1), 1)
if not bool(config.get("adaptive_batch_enabled", True)):
decision = AdaptiveBatchDecision(
batch_size=configured_max,
configured_max=configured_max,
previous_batch_size=configured_max,
success_rate=0.0,
sample_size=0,
pending=max(0, int(pending)),
action="fixed",
)
config["adaptive_batch_size"] = decision.batch_size
job.config = config
return decision
window_days = max(
1,
int(config.get("adaptive_window_days", DEFAULT_ADAPTIVE_WINDOW_DAYS) or 1),
)
min_sample = max(
1,
int(config.get("adaptive_min_sample", DEFAULT_ADAPTIVE_MIN_SAMPLE) or 1),
)
grow_rate = float(
config.get("adaptive_grow_rate", DEFAULT_ADAPTIVE_GROW_RATE)
)
shrink_rate = float(
config.get("adaptive_shrink_rate", DEFAULT_ADAPTIVE_SHRINK_RATE)
)
grow_factor = max(
1.01,
float(config.get("adaptive_grow_factor", DEFAULT_ADAPTIVE_GROW_FACTOR)),
)
initial = min(
configured_max,
max(
1,
int(
config.get(
"adaptive_initial_batch",
DEFAULT_ADAPTIVE_INITIAL_BATCH,
)
or 1
),
),
)
previous = min(
configured_max,
max(1, int(config.get("adaptive_batch_size", initial) or initial)),
)
sample, success_rate = _recent_relogin_success(
db,
now=now,
window_days=window_days,
)
target = previous
action = "hold"
if sample >= min_sample and success_rate < shrink_rate:
target = max(1, previous // 2)
action = "shrink"
elif sample >= min_sample and success_rate >= grow_rate:
target = min(
configured_max,
max(previous + 1, math.ceil(previous * grow_factor)),
)
action = "grow" if target > previous else "hold"
elif sample < min_sample:
action = "cold_start"
if pending > 0:
target = min(target, max(1, int(pending)))
decision = AdaptiveBatchDecision(
batch_size=max(1, target),
configured_max=configured_max,
previous_batch_size=previous,
success_rate=success_rate,
sample_size=sample,
pending=max(0, int(pending)),
action=action,
)
config["adaptive_batch_size"] = decision.batch_size
config["adaptive_last_success_rate"] = decision.success_rate
config["adaptive_last_sample"] = decision.sample_size
job.config = config
return decision
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
now = datetime.now(UTC)
config = resolve_failure_circuit_config(db, job.config)
@@ -286,8 +446,22 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
"message": "库里没有可用用户,无法以系统身份发起重登",
}
accounts = build_select_query(db, job, now, max_attempts=max_attempts).all()
pending = count_pending_relogin(db, max_attempts=max_attempts)
adaptive = resolve_adaptive_batch_size(
db,
job,
now=now,
pending=pending,
)
accounts = build_select_query(
db,
job,
now,
max_attempts=max_attempts,
batch_limit=adaptive.batch_size,
).all()
if not accounts:
db.commit()
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
selected_account_ids = tuple(int(account.id) for account in accounts)
@@ -370,7 +544,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
job.updated_at = now
parts = [
f"本轮重登 {counters['scanned']} 条",
f"本轮重登 {counters['scanned']} 条(有效批量 {adaptive.batch_size})",
f"成功 {counters['valid']}",
f"失败 {counters['invalid']}",
]
@@ -384,6 +558,11 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
if paused:
parts.append("失败率过高,已自动暂停作业")
if adaptive.action != "hold":
parts.append(
f"批量策略 {adaptive.previous_batch_size}->{adaptive.batch_size}"
f"({adaptive.action},近 7 天成功率 {adaptive.success_rate:.1f}%)"
)
return {
"scanned": counters["scanned"],
"valid": counters["valid"],
@@ -400,6 +579,15 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
"hard_failures": hard_failures,
"hard_failure_ratio": round(hard_failure_ratio, 4),
"paused": paused,
"adaptive_batch": {
"batch_size": adaptive.batch_size,
"configured_max": adaptive.configured_max,
"previous_batch_size": adaptive.previous_batch_size,
"success_rate": adaptive.success_rate,
"sample_size": adaptive.sample_size,
"pending": adaptive.pending,
"action": adaptive.action,
},
},
}