feat(automation): 按成功率灰度扩缩自动重登
This commit is contained in:
@@ -403,6 +403,62 @@ def account_platform_status(
|
||||
}
|
||||
)
|
||||
|
||||
trends = _platform_trends(db, moment)
|
||||
query_sample = sum(
|
||||
int(row["success"]) + int(row["failed"])
|
||||
for row in trends["query"]
|
||||
)
|
||||
query_success = sum(int(row["success"]) for row in trends["query"])
|
||||
query_rate = _rate(query_success, query_sample)
|
||||
if query_sample >= 20 and query_rate < 80:
|
||||
alerts.append(
|
||||
{
|
||||
"code": "query_success_rate_low",
|
||||
"severity": "critical" if query_rate < 50 else "warning",
|
||||
"count": query_sample,
|
||||
"message": (
|
||||
f"近 {TREND_DAYS} 天查询成功率 {query_rate:.1f}%"
|
||||
f"({query_sample} 次已完成查询)"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
login_sample = sum(
|
||||
int(row["success"]) + int(row["failed"])
|
||||
for row in trends["login"]
|
||||
)
|
||||
login_success = sum(int(row["success"]) for row in trends["login"])
|
||||
login_rate = _rate(login_success, login_sample)
|
||||
if login_sample >= 10 and login_rate < 60:
|
||||
alerts.append(
|
||||
{
|
||||
"code": "login_success_rate_low",
|
||||
"severity": "critical" if login_rate < 40 else "warning",
|
||||
"count": login_sample,
|
||||
"message": (
|
||||
f"近 {TREND_DAYS} 天登录成功率 {login_rate:.1f}%"
|
||||
f"({login_sample} 次已完成尝试)"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
relogin_ok = sum(int(row["relogin_ok"]) for row in trends["lifecycle"])
|
||||
relogin_failed = sum(
|
||||
int(row["relogin_failed"]) for row in trends["lifecycle"]
|
||||
)
|
||||
if relogin_failed >= 5 and relogin_failed > relogin_ok:
|
||||
alerts.append(
|
||||
{
|
||||
"code": "relogin_failure_trend_high",
|
||||
"severity": "warning",
|
||||
"count": relogin_failed,
|
||||
"message": (
|
||||
f"近 {TREND_DAYS} 天重登失败 {relogin_failed} 次,"
|
||||
f"成功 {relogin_ok} 次"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"checked_at": moment,
|
||||
"accounts": {
|
||||
@@ -425,7 +481,7 @@ def account_platform_status(
|
||||
},
|
||||
"supply_targets": supply_targets,
|
||||
"alerts": alerts,
|
||||
"trends": _platform_trends(db, moment),
|
||||
"trends": trends,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
@@ -21,7 +23,14 @@ from sqlalchemy import func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....database import WorkerSessionLocal
|
||||
from ....models import Account, AutomationJob, LoginQuotaSetting, LoginTask, User
|
||||
from ....models import (
|
||||
Account,
|
||||
AutomationJob,
|
||||
LoginAttempt,
|
||||
LoginQuotaSetting,
|
||||
LoginTask,
|
||||
User,
|
||||
)
|
||||
from ....models import ProxyConfig as ProxyConfigModel
|
||||
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
|
||||
from ...account_lifecycle import (
|
||||
@@ -62,6 +71,23 @@ AUTO_MAX_TOTAL_TIME = 240
|
||||
QUOTA_JITTER_MINUTES = 30
|
||||
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
|
||||
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
|
||||
DEFAULT_ADAPTIVE_INITIAL_BATCH = 10
|
||||
DEFAULT_ADAPTIVE_WINDOW_DAYS = 7
|
||||
DEFAULT_ADAPTIVE_MIN_SAMPLE = 10
|
||||
DEFAULT_ADAPTIVE_GROW_RATE = 90.0
|
||||
DEFAULT_ADAPTIVE_SHRINK_RATE = 70.0
|
||||
DEFAULT_ADAPTIVE_GROW_FACTOR = 1.25
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AdaptiveBatchDecision:
|
||||
batch_size: int
|
||||
configured_max: int
|
||||
previous_batch_size: int
|
||||
success_rate: float
|
||||
sample_size: int
|
||||
pending: int
|
||||
action: str
|
||||
|
||||
|
||||
def _cutoff(now: datetime) -> datetime:
|
||||
@@ -75,6 +101,7 @@ def build_select_query(
|
||||
*,
|
||||
max_attempts: int = 2,
|
||||
system_user_id: int | None = None,
|
||||
batch_limit: int | None = None,
|
||||
):
|
||||
"""构造本轮重登候选的查询(与取行分开,便于按方言编译做回归)。
|
||||
|
||||
@@ -111,12 +138,13 @@ def build_select_query(
|
||||
pass # 预留:将来若需要限制到某个归属人,在这里加过滤
|
||||
# 名额先给第 1、2 档(两套还能换 / 还剩一套 / 积分≥300)。
|
||||
# 同一档里仍按退避时间,避免把还没到点的高价值号提前拉起来。
|
||||
limit = int(batch_limit if batch_limit is not None else job.batch_size or 0)
|
||||
return query.order_by(
|
||||
value_tier_expr().asc(),
|
||||
Account.relogin_next_at.is_(None).desc(),
|
||||
Account.relogin_next_at.asc(),
|
||||
Account.id.asc(),
|
||||
).limit(max(int(job.batch_size or 0), 1))
|
||||
).limit(max(limit, 1))
|
||||
|
||||
|
||||
def _load_batch_state(
|
||||
@@ -271,6 +299,138 @@ def resolve_failure_circuit_config(
|
||||
return resolved
|
||||
|
||||
|
||||
def _recent_relogin_success(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime,
|
||||
window_days: int,
|
||||
) -> tuple[int, float]:
|
||||
start_day = now.date() - timedelta(days=max(window_days, 1) - 1)
|
||||
rows = (
|
||||
db.query(
|
||||
LoginAttempt.status,
|
||||
LoginAttempt.failure_kind,
|
||||
func.count(LoginAttempt.id),
|
||||
)
|
||||
.filter(
|
||||
LoginAttempt.quota_day >= start_day,
|
||||
LoginAttempt.category.in_(
|
||||
("relogin_high_value", "relogin_ordinary")
|
||||
),
|
||||
)
|
||||
.group_by(LoginAttempt.status, LoginAttempt.failure_kind)
|
||||
.all()
|
||||
)
|
||||
success = 0
|
||||
failed = 0
|
||||
for status, failure_kind, count in rows:
|
||||
count_value = int(count or 0)
|
||||
if status == "success":
|
||||
success += count_value
|
||||
elif status in {"failed", "error", "stopped"} and failure_kind != "email_quota":
|
||||
failed += count_value
|
||||
sample = success + failed
|
||||
rate = success * 100 / sample if sample else 0.0
|
||||
return sample, round(rate, 1)
|
||||
|
||||
|
||||
def resolve_adaptive_batch_size(
|
||||
db: Session,
|
||||
job: AutomationJob,
|
||||
*,
|
||||
now: datetime,
|
||||
pending: int,
|
||||
) -> AdaptiveBatchDecision:
|
||||
"""Scale the effective relogin batch inside the operator-configured ceiling."""
|
||||
config = dict(job.config or {})
|
||||
configured_max = max(int(job.batch_size or 1), 1)
|
||||
if not bool(config.get("adaptive_batch_enabled", True)):
|
||||
decision = AdaptiveBatchDecision(
|
||||
batch_size=configured_max,
|
||||
configured_max=configured_max,
|
||||
previous_batch_size=configured_max,
|
||||
success_rate=0.0,
|
||||
sample_size=0,
|
||||
pending=max(0, int(pending)),
|
||||
action="fixed",
|
||||
)
|
||||
config["adaptive_batch_size"] = decision.batch_size
|
||||
job.config = config
|
||||
return decision
|
||||
|
||||
window_days = max(
|
||||
1,
|
||||
int(config.get("adaptive_window_days", DEFAULT_ADAPTIVE_WINDOW_DAYS) or 1),
|
||||
)
|
||||
min_sample = max(
|
||||
1,
|
||||
int(config.get("adaptive_min_sample", DEFAULT_ADAPTIVE_MIN_SAMPLE) or 1),
|
||||
)
|
||||
grow_rate = float(
|
||||
config.get("adaptive_grow_rate", DEFAULT_ADAPTIVE_GROW_RATE)
|
||||
)
|
||||
shrink_rate = float(
|
||||
config.get("adaptive_shrink_rate", DEFAULT_ADAPTIVE_SHRINK_RATE)
|
||||
)
|
||||
grow_factor = max(
|
||||
1.01,
|
||||
float(config.get("adaptive_grow_factor", DEFAULT_ADAPTIVE_GROW_FACTOR)),
|
||||
)
|
||||
initial = min(
|
||||
configured_max,
|
||||
max(
|
||||
1,
|
||||
int(
|
||||
config.get(
|
||||
"adaptive_initial_batch",
|
||||
DEFAULT_ADAPTIVE_INITIAL_BATCH,
|
||||
)
|
||||
or 1
|
||||
),
|
||||
),
|
||||
)
|
||||
previous = min(
|
||||
configured_max,
|
||||
max(1, int(config.get("adaptive_batch_size", initial) or initial)),
|
||||
)
|
||||
sample, success_rate = _recent_relogin_success(
|
||||
db,
|
||||
now=now,
|
||||
window_days=window_days,
|
||||
)
|
||||
|
||||
target = previous
|
||||
action = "hold"
|
||||
if sample >= min_sample and success_rate < shrink_rate:
|
||||
target = max(1, previous // 2)
|
||||
action = "shrink"
|
||||
elif sample >= min_sample and success_rate >= grow_rate:
|
||||
target = min(
|
||||
configured_max,
|
||||
max(previous + 1, math.ceil(previous * grow_factor)),
|
||||
)
|
||||
action = "grow" if target > previous else "hold"
|
||||
elif sample < min_sample:
|
||||
action = "cold_start"
|
||||
|
||||
if pending > 0:
|
||||
target = min(target, max(1, int(pending)))
|
||||
decision = AdaptiveBatchDecision(
|
||||
batch_size=max(1, target),
|
||||
configured_max=configured_max,
|
||||
previous_batch_size=previous,
|
||||
success_rate=success_rate,
|
||||
sample_size=sample,
|
||||
pending=max(0, int(pending)),
|
||||
action=action,
|
||||
)
|
||||
config["adaptive_batch_size"] = decision.batch_size
|
||||
config["adaptive_last_success_rate"] = decision.success_rate
|
||||
config["adaptive_last_sample"] = decision.sample_size
|
||||
job.config = config
|
||||
return decision
|
||||
|
||||
|
||||
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
config = resolve_failure_circuit_config(db, job.config)
|
||||
@@ -286,8 +446,22 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
"message": "库里没有可用用户,无法以系统身份发起重登",
|
||||
}
|
||||
|
||||
accounts = build_select_query(db, job, now, max_attempts=max_attempts).all()
|
||||
pending = count_pending_relogin(db, max_attempts=max_attempts)
|
||||
adaptive = resolve_adaptive_batch_size(
|
||||
db,
|
||||
job,
|
||||
now=now,
|
||||
pending=pending,
|
||||
)
|
||||
accounts = build_select_query(
|
||||
db,
|
||||
job,
|
||||
now,
|
||||
max_attempts=max_attempts,
|
||||
batch_limit=adaptive.batch_size,
|
||||
).all()
|
||||
if not accounts:
|
||||
db.commit()
|
||||
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
|
||||
|
||||
selected_account_ids = tuple(int(account.id) for account in accounts)
|
||||
@@ -370,7 +544,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
job.updated_at = now
|
||||
|
||||
parts = [
|
||||
f"本轮重登 {counters['scanned']} 条",
|
||||
f"本轮重登 {counters['scanned']} 条(有效批量 {adaptive.batch_size})",
|
||||
f"成功 {counters['valid']}",
|
||||
f"失败 {counters['invalid']}",
|
||||
]
|
||||
@@ -384,6 +558,11 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
|
||||
if paused:
|
||||
parts.append("失败率过高,已自动暂停作业")
|
||||
if adaptive.action != "hold":
|
||||
parts.append(
|
||||
f"批量策略 {adaptive.previous_batch_size}->{adaptive.batch_size}"
|
||||
f"({adaptive.action},近 7 天成功率 {adaptive.success_rate:.1f}%)"
|
||||
)
|
||||
return {
|
||||
"scanned": counters["scanned"],
|
||||
"valid": counters["valid"],
|
||||
@@ -400,6 +579,15 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
"hard_failures": hard_failures,
|
||||
"hard_failure_ratio": round(hard_failure_ratio, 4),
|
||||
"paused": paused,
|
||||
"adaptive_batch": {
|
||||
"batch_size": adaptive.batch_size,
|
||||
"configured_max": adaptive.configured_max,
|
||||
"previous_batch_size": adaptive.previous_batch_size,
|
||||
"success_rate": adaptive.success_rate,
|
||||
"sample_size": adaptive.sample_size,
|
||||
"pending": adaptive.pending,
|
||||
"action": adaptive.action,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user