740 lines
26 KiB
Python
740 lines
26 KiB
Python
"""CK 自动重登作业(P1b)。
|
||
|
||
和巡检(P1a)是一对:巡检只产出「谁失效了」的清单,本作业负责把能救的救回来。
|
||
线上实测(2026-10-02)20 个失效账号重登 **20/20 成功** —— 失效是会话级过期,
|
||
不是凭证问题。
|
||
|
||
**核心设计:失败必须分型。** 线上数据表明 415 个重登失败里只有 35% 是真正需要
|
||
人工的(极验滑块),另外 64%(取码失败 + 重启中断)其实可自动重试;不分型就会
|
||
把可救名额一起推给人工。分型结果落 ``login_tasks.relogin_failure_kind``。
|
||
|
||
本作业**默认关闭**,且按规划 §7.4.7 分三段上线(小规模实测已完成 → 限量 → 全量)。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import math
|
||
import random
|
||
from dataclasses import dataclass
|
||
from datetime import UTC, datetime, timedelta
|
||
from typing import Any
|
||
|
||
from sqlalchemy import case, func, or_
|
||
from sqlalchemy.orm import Session
|
||
|
||
from ....database import WorkerSessionLocal
|
||
from ....models import (
|
||
Account,
|
||
AutomationJob,
|
||
LoginAttempt,
|
||
LoginQuotaSetting,
|
||
LoginTask,
|
||
User,
|
||
)
|
||
from ....models import ProxyConfig as ProxyConfigModel
|
||
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
|
||
from ...account_lifecycle import (
|
||
MANUAL_KINDS,
|
||
MANUAL_STATES,
|
||
WORK_LOGIN_RELOGIN,
|
||
on_login_success,
|
||
sync_status,
|
||
)
|
||
from ...account_protection_service import (
|
||
ProtectionSnapshot,
|
||
load_protection_snapshot,
|
||
)
|
||
from ...account_value import value_tier_expr
|
||
from ...account_work_queue import (
|
||
claim_account_work,
|
||
enqueue_account_work,
|
||
fail_account_work_type,
|
||
reschedule_account_work_type,
|
||
)
|
||
from ...douyu_service import cookie_holding_filter, cookie_operable_filter
|
||
from ...login_failure import classify_failure
|
||
from ...login_service import LoginBatchRunner, get_relogin_limits
|
||
|
||
JOB_KEY = "ck_relogin"
|
||
|
||
# 失败分型 → 后续动作。
|
||
# `MANUAL_KINDS`(极验 / 凭证 → 转人工)已挪到 `services/account_lifecycle.py`:
|
||
# 生命周期状态推导要用它,service 不该反过来依赖 job。
|
||
# 可自动重试的:取码失败/超时(多为邮件延迟)、中断(服务重启或人工停止)、
|
||
# 网络异常。email_quota 单列,因为它要退避到次日而不是按小时退避。
|
||
RETRYABLE_KINDS = frozenset({"email_code", "interrupted", "network"})
|
||
|
||
# 退避阶梯(分钟):1h → 3h → 12h → 24h,超出取最后一级。
|
||
BACKOFF_MINUTES = (60, 180, 720, 1440)
|
||
|
||
# 自动重登时给 runner 的总时长上限。默认的 600 秒会长时间占住调度线程,
|
||
# 自动化场景压到 240 秒:一轮跑不完的账号留着下一轮(它们本来就有退避)。
|
||
AUTO_MAX_TOTAL_TIME = 240
|
||
|
||
# 邮箱配额打满后退避到次日零点,再叠加随机抖动,避免整点齐发。
|
||
QUOTA_JITTER_MINUTES = 30
|
||
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
|
||
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
|
||
DEFAULT_ADAPTIVE_INITIAL_BATCH = 10
|
||
DEFAULT_ADAPTIVE_WINDOW_DAYS = 7
|
||
DEFAULT_ADAPTIVE_MIN_SAMPLE = 10
|
||
DEFAULT_ADAPTIVE_GROW_RATE = 90.0
|
||
DEFAULT_ADAPTIVE_SHRINK_RATE = 70.0
|
||
DEFAULT_ADAPTIVE_GROW_FACTOR = 1.25
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class AdaptiveBatchDecision:
|
||
batch_size: int
|
||
configured_max: int
|
||
previous_batch_size: int
|
||
success_rate: float
|
||
sample_size: int
|
||
pending: int
|
||
action: str
|
||
|
||
|
||
def _cutoff(now: datetime) -> datetime:
|
||
return now
|
||
|
||
|
||
def build_select_query(
|
||
db: Session,
|
||
job: AutomationJob,
|
||
now: datetime,
|
||
*,
|
||
max_attempts: int = 2,
|
||
system_user_id: int | None = None,
|
||
batch_limit: int | None = None,
|
||
protection: ProtectionSnapshot | None = None,
|
||
):
|
||
"""构造本轮重登候选的查询(与取行分开,便于按方言编译做回归)。
|
||
|
||
解耦后**按账号选行**(2026-10-04):CK 与 ``ck_check_status`` / ``relogin_*``
|
||
都在 ``accounts`` 上。以前是 ``db.query(LoginTask).join(Account)`` —— 那样
|
||
「没有登录历史的账号」永远进不了重登队列,而重登本来只关心账号。
|
||
|
||
「在途尝试」仍然要查 ``login_tasks``(那是某次尝试的属性),由
|
||
``cookie_operable_filter`` 的 NOT EXISTS 子查询完成,走迁移 0042 新增的
|
||
``(account_id, attempt_status)`` 索引。
|
||
"""
|
||
query = (
|
||
db.query(Account)
|
||
.filter(Account.ck_check_status == "invalid")
|
||
.filter(cookie_operable_filter())
|
||
.filter(Account.auto_managed.is_(True))
|
||
# 停用/归档的账号一律不碰(生命周期状态机,见 account_lifecycle)。
|
||
.filter(Account.status.notin_(MANUAL_STATES))
|
||
.filter(Account.relogin_attempts < max_attempts)
|
||
.filter(
|
||
or_(
|
||
Account.relogin_next_at.is_(None),
|
||
Account.relogin_next_at <= _cutoff(now),
|
||
)
|
||
)
|
||
)
|
||
# 已判定需要人工的类型不再自动重试(极验/凭证)。空值用 coalesce 兜住历史值。
|
||
query = query.filter(
|
||
func.coalesce(Account.relogin_failure_kind, "").notin_(
|
||
tuple(sorted(MANUAL_KINDS))
|
||
)
|
||
)
|
||
snapshot = protection or load_protection_snapshot(db)
|
||
suppressed_ids = snapshot.suppressed_relogin_ids
|
||
if suppressed_ids:
|
||
# 保护组可以分别关闭自动重登。账号同时属于任一已开启自动重登的组时,
|
||
# 仍按允许处理;只有完全被抑制的账号才排除。
|
||
query = query.filter(~Account.id.in_(suppressed_ids))
|
||
relogin_ids = snapshot.relogin_ids
|
||
if system_user_id is not None:
|
||
pass # 预留:将来若需要限制到某个归属人,在这里加过滤
|
||
# 名额先给第 1、2 档(两套还能换 / 还剩一套 / 积分≥300)。
|
||
# 同一档里仍按退避时间,避免把还没到点的高价值号提前拉起来。
|
||
limit = int(batch_limit if batch_limit is not None else job.batch_size or 0)
|
||
ordering = [
|
||
Account.relogin_next_at.is_(None).desc(),
|
||
Account.relogin_next_at.asc(),
|
||
Account.id.asc(),
|
||
]
|
||
if relogin_ids:
|
||
ordering.insert(
|
||
0,
|
||
case((Account.id.in_(relogin_ids), 0), else_=1),
|
||
)
|
||
ordering.insert(
|
||
1 if relogin_ids else 0,
|
||
value_tier_expr().asc(),
|
||
)
|
||
return query.order_by(*ordering).limit(max(limit, 1))
|
||
|
||
|
||
def _load_batch_state(
|
||
session: Session, batch_id: str
|
||
) -> tuple[dict[int, tuple[Account, LoginTask]], list[int]]:
|
||
"""读回某一批次产生的尝试记录,并配对其所属账号。
|
||
|
||
按 ``attempt_batch_id`` 找 —— ``_begin_attempt`` 会给每一条被本批次接管的记录
|
||
打上批次号。所以即使记录是**复用**的旧行,也能被准确对应回本轮。
|
||
"""
|
||
tasks = (
|
||
session.query(LoginTask)
|
||
.filter(LoginTask.attempt_batch_id == batch_id)
|
||
.order_by(LoginTask.id.asc())
|
||
.all()
|
||
)
|
||
state: dict[int, tuple[Account, LoginTask]] = {}
|
||
order: list[int] = []
|
||
for task in tasks:
|
||
account = session.get(Account, task.account_id)
|
||
if account is None:
|
||
continue
|
||
state[account.id] = (account, task)
|
||
order.append(account.id)
|
||
return state, order
|
||
|
||
|
||
def settle_fresh_results(
|
||
batch_id: str,
|
||
*,
|
||
now: datetime,
|
||
max_attempts: int = 2,
|
||
expected_account_ids: list[int] | tuple[int, ...] | None = None,
|
||
) -> dict[str, int]:
|
||
"""在**新会话(新事务)**里读回批次结果并结算。
|
||
|
||
必须新开会话,不能复用调度器传进来的 ``db``:那个会话在挑候选时就已经开了
|
||
事务,而 MySQL 默认隔离级别是 **REPEATABLE READ** —— 在同一事务里重新查询
|
||
只会看到事务开始时的快照,看不到 runner 在另一个会话里提交的成功结果。
|
||
|
||
线上为此踩过一次:10 条重登**实际全部成功**,结算却按失败写回,给它们加了
|
||
退避时间,其中 2 条还被标成「极验」进了待人工队列。SQLite 的读一致性与 MySQL
|
||
不同,这类问题在 SQLite 上复现不出来。
|
||
"""
|
||
session = WorkerSessionLocal()
|
||
try:
|
||
fresh, order = _load_batch_state(session, batch_id)
|
||
# runner 可能因登录预算、账号预检等原因没有生成 LoginTask。此时也必须
|
||
# 结算本轮已经 claim 的账号,否则工作项会永久停在 running,并阻塞后续
|
||
# 所有重登候选。2026-11-22 生产曾因此卡住 5 条、堵住 261 条积压。
|
||
settle_account_ids = list(
|
||
dict.fromkeys(
|
||
[
|
||
*order,
|
||
*(int(account_id) for account_id in expected_account_ids or ()),
|
||
]
|
||
)
|
||
)
|
||
counters = settle_results(
|
||
session,
|
||
fresh,
|
||
settle_account_ids,
|
||
now=now,
|
||
max_attempts=max_attempts,
|
||
)
|
||
session.commit()
|
||
return counters
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
def count_pending_relogin(db: Session, *, max_attempts: int = 2) -> int:
|
||
"""待自动重登的**积压**条数(给页面显示用)。
|
||
|
||
刻意不复用 ``build_select_query``:那个查询带 ``LIMIT batch_size``,用它取
|
||
count 只会得到批次上限(页面上显示恒等于 10),完全看不出积压。这里也**不
|
||
按退避窗口过滤** —— 退避是「什么时候试」的细节,不影响「还有多少要处理」。
|
||
"""
|
||
snapshot = load_protection_snapshot(db)
|
||
query = (
|
||
db.query(func.count(Account.id))
|
||
.filter(Account.ck_check_status == "invalid")
|
||
.filter(cookie_holding_filter())
|
||
.filter(Account.auto_managed.is_(True))
|
||
# 停用/归档不计入积压(它们不会被自动处理,显示出来只会误导)
|
||
.filter(Account.status.notin_(MANUAL_STATES))
|
||
.filter(Account.relogin_attempts < max_attempts)
|
||
.filter(
|
||
func.coalesce(Account.relogin_failure_kind, "").notin_(
|
||
tuple(sorted(MANUAL_KINDS))
|
||
)
|
||
)
|
||
)
|
||
if snapshot.suppressed_relogin_ids:
|
||
query = query.filter(~Account.id.in_(snapshot.suppressed_relogin_ids))
|
||
return int(query.scalar() or 0)
|
||
|
||
|
||
def _system_user_id(db: Session) -> int | None:
|
||
"""自动重登要用一个真实用户身份落 ``created_by``(外键非空,且便于追责)。
|
||
|
||
取最早创建的超管作为「系统操作人」;库是空的就返回 None,作业会跳过本轮。
|
||
"""
|
||
row = (
|
||
db.query(User.id)
|
||
.filter(User.role == "super_admin")
|
||
.order_by(User.id.asc())
|
||
.first()
|
||
)
|
||
if row:
|
||
return int(row[0])
|
||
row = db.query(User.id).order_by(User.id.asc()).first()
|
||
return int(row[0]) if row else None
|
||
|
||
|
||
def _next_retry_at(kind: str, attempts: int, now: datetime) -> datetime:
|
||
if kind == "email_quota":
|
||
tomorrow = (now + timedelta(days=1)).replace(
|
||
hour=0, minute=0, second=0, microsecond=0
|
||
)
|
||
return tomorrow + timedelta(minutes=random.uniform(0, QUOTA_JITTER_MINUTES))
|
||
step = BACKOFF_MINUTES[min(max(attempts, 1), len(BACKOFF_MINUTES)) - 1]
|
||
return now + timedelta(minutes=step)
|
||
|
||
|
||
def evaluate_failure_circuit(
|
||
counters: dict[str, int],
|
||
config: dict[str, Any],
|
||
) -> tuple[bool, int, float]:
|
||
"""评估硬失败率;邮箱配额满属于预期供给限制,不计入熔断。"""
|
||
scanned = max(0, int(counters.get("scanned", 0) or 0))
|
||
invalid = max(0, int(counters.get("invalid", 0) or 0))
|
||
email_quota = max(0, int(counters.get("email_quota", 0) or 0))
|
||
hard_failures = max(0, invalid - email_quota)
|
||
minimum = max(
|
||
1,
|
||
int(
|
||
config.get(
|
||
"failure_pause_min_scanned",
|
||
DEFAULT_FAILURE_PAUSE_MIN_SCANNED,
|
||
)
|
||
),
|
||
)
|
||
raw_ratio = float(
|
||
config.get("failure_pause_ratio", DEFAULT_FAILURE_PAUSE_RATIO)
|
||
)
|
||
threshold = raw_ratio / 100 if raw_ratio > 1 else raw_ratio
|
||
threshold = max(0.0, min(threshold, 1.0))
|
||
ratio = hard_failures / scanned if scanned else 0.0
|
||
return scanned >= minimum and ratio > threshold, hard_failures, ratio
|
||
|
||
|
||
def resolve_failure_circuit_config(
|
||
db: Session,
|
||
config: dict[str, Any] | None,
|
||
) -> dict[str, Any]:
|
||
"""合并作业配置和在线登录预算里的失败暂停阈值。"""
|
||
resolved = dict(config or {})
|
||
if "failure_pause_ratio" not in resolved:
|
||
setting = (
|
||
db.query(LoginQuotaSetting)
|
||
.filter(LoginQuotaSetting.id == 1)
|
||
.first()
|
||
)
|
||
if setting is not None:
|
||
resolved["failure_pause_ratio"] = int(
|
||
setting.failure_pause_ratio
|
||
)
|
||
return resolved
|
||
|
||
|
||
def _recent_relogin_success(
|
||
db: Session,
|
||
*,
|
||
now: datetime,
|
||
window_days: int,
|
||
) -> tuple[int, float]:
|
||
start_day = now.date() - timedelta(days=max(window_days, 1) - 1)
|
||
rows = (
|
||
db.query(
|
||
LoginAttempt.status,
|
||
LoginAttempt.failure_kind,
|
||
func.count(LoginAttempt.id),
|
||
)
|
||
.filter(
|
||
LoginAttempt.quota_day >= start_day,
|
||
LoginAttempt.category.in_(
|
||
("relogin_high_value", "relogin_ordinary")
|
||
),
|
||
)
|
||
.group_by(LoginAttempt.status, LoginAttempt.failure_kind)
|
||
.all()
|
||
)
|
||
success = 0
|
||
failed = 0
|
||
for status, failure_kind, count in rows:
|
||
count_value = int(count or 0)
|
||
if status == "success":
|
||
success += count_value
|
||
elif status in {"failed", "error", "stopped"} and failure_kind != "email_quota":
|
||
failed += count_value
|
||
sample = success + failed
|
||
rate = success * 100 / sample if sample else 0.0
|
||
return sample, round(rate, 1)
|
||
|
||
|
||
def resolve_adaptive_batch_size(
|
||
db: Session,
|
||
job: AutomationJob,
|
||
*,
|
||
now: datetime,
|
||
pending: int,
|
||
) -> AdaptiveBatchDecision:
|
||
"""Scale the effective relogin batch inside the operator-configured ceiling."""
|
||
config = dict(job.config or {})
|
||
configured_max = max(int(job.batch_size or 1), 1)
|
||
if not bool(config.get("adaptive_batch_enabled", True)):
|
||
decision = AdaptiveBatchDecision(
|
||
batch_size=configured_max,
|
||
configured_max=configured_max,
|
||
previous_batch_size=configured_max,
|
||
success_rate=0.0,
|
||
sample_size=0,
|
||
pending=max(0, int(pending)),
|
||
action="fixed",
|
||
)
|
||
config["adaptive_batch_size"] = decision.batch_size
|
||
job.config = config
|
||
return decision
|
||
|
||
window_days = max(
|
||
1,
|
||
int(config.get("adaptive_window_days", DEFAULT_ADAPTIVE_WINDOW_DAYS) or 1),
|
||
)
|
||
min_sample = max(
|
||
1,
|
||
int(config.get("adaptive_min_sample", DEFAULT_ADAPTIVE_MIN_SAMPLE) or 1),
|
||
)
|
||
grow_rate = float(
|
||
config.get("adaptive_grow_rate", DEFAULT_ADAPTIVE_GROW_RATE)
|
||
)
|
||
shrink_rate = float(
|
||
config.get("adaptive_shrink_rate", DEFAULT_ADAPTIVE_SHRINK_RATE)
|
||
)
|
||
grow_factor = max(
|
||
1.01,
|
||
float(config.get("adaptive_grow_factor", DEFAULT_ADAPTIVE_GROW_FACTOR)),
|
||
)
|
||
initial = min(
|
||
configured_max,
|
||
max(
|
||
1,
|
||
int(
|
||
config.get(
|
||
"adaptive_initial_batch",
|
||
DEFAULT_ADAPTIVE_INITIAL_BATCH,
|
||
)
|
||
or 1
|
||
),
|
||
),
|
||
)
|
||
previous = min(
|
||
configured_max,
|
||
max(1, int(config.get("adaptive_batch_size", initial) or initial)),
|
||
)
|
||
sample, success_rate = _recent_relogin_success(
|
||
db,
|
||
now=now,
|
||
window_days=window_days,
|
||
)
|
||
|
||
target = previous
|
||
action = "hold"
|
||
if sample >= min_sample and success_rate < shrink_rate:
|
||
target = max(1, previous // 2)
|
||
action = "shrink"
|
||
elif sample >= min_sample and success_rate >= grow_rate:
|
||
target = min(
|
||
configured_max,
|
||
max(previous + 1, math.ceil(previous * grow_factor)),
|
||
)
|
||
action = "grow" if target > previous else "hold"
|
||
elif sample < min_sample:
|
||
action = "cold_start"
|
||
|
||
if pending > 0:
|
||
target = min(target, max(1, int(pending)))
|
||
decision = AdaptiveBatchDecision(
|
||
batch_size=max(1, target),
|
||
configured_max=configured_max,
|
||
previous_batch_size=previous,
|
||
success_rate=success_rate,
|
||
sample_size=sample,
|
||
pending=max(0, int(pending)),
|
||
action=action,
|
||
)
|
||
config["adaptive_batch_size"] = decision.batch_size
|
||
config["adaptive_last_success_rate"] = decision.success_rate
|
||
config["adaptive_last_sample"] = decision.sample_size
|
||
job.config = config
|
||
return decision
|
||
|
||
|
||
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||
now = datetime.now(UTC)
|
||
config = resolve_failure_circuit_config(db, job.config)
|
||
max_attempts = int(config.get("max_attempts_per_day", 2) or 2)
|
||
|
||
system_user_id = _system_user_id(db)
|
||
if system_user_id is None:
|
||
return {
|
||
"scanned": 0,
|
||
"valid": 0,
|
||
"invalid": 0,
|
||
"skipped": 0,
|
||
"message": "库里没有可用用户,无法以系统身份发起重登",
|
||
}
|
||
|
||
pending = count_pending_relogin(db, max_attempts=max_attempts)
|
||
adaptive = resolve_adaptive_batch_size(
|
||
db,
|
||
job,
|
||
now=now,
|
||
pending=pending,
|
||
)
|
||
protection = load_protection_snapshot(db)
|
||
accounts = build_select_query(
|
||
db,
|
||
job,
|
||
now,
|
||
max_attempts=max_attempts,
|
||
batch_limit=adaptive.batch_size,
|
||
protection=protection,
|
||
).all()
|
||
if not accounts:
|
||
db.commit()
|
||
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
|
||
|
||
selected_account_ids = tuple(int(account.id) for account in accounts)
|
||
for account in accounts:
|
||
priority = 10 if int(account.id) in protection.relogin_ids else 30
|
||
enqueue_account_work(
|
||
db,
|
||
account_id=int(account.id),
|
||
work_type=WORK_LOGIN_RELOGIN,
|
||
priority=priority,
|
||
max_attempts=20,
|
||
due_at=now,
|
||
reason="CK 失效,等待自动重登",
|
||
)
|
||
db.commit()
|
||
work_items = claim_account_work(
|
||
db,
|
||
work_types=(WORK_LOGIN_RELOGIN,),
|
||
account_ids=selected_account_ids,
|
||
limit=len(selected_account_ids),
|
||
now=now,
|
||
)
|
||
db.commit()
|
||
claimed_account_ids = [int(item.account_id) for item in work_items]
|
||
if not claimed_account_ids:
|
||
return {
|
||
"scanned": 0,
|
||
"valid": 0,
|
||
"invalid": 0,
|
||
"skipped": len(selected_account_ids),
|
||
"message": "候选账号的重登工作项已被其他执行器领取",
|
||
}
|
||
account_map = {
|
||
int(account.id): account
|
||
for account in db.query(Account)
|
||
.filter(Account.id.in_(claimed_account_ids))
|
||
.all()
|
||
}
|
||
accounts = [
|
||
account_map[account_id]
|
||
for account_id in claimed_account_ids
|
||
if account_id in account_map
|
||
]
|
||
|
||
# 调度单位是**账号**(CK 在账号上);登录记录只是这次尝试的历史。
|
||
account_ids = [account.id for account in accounts]
|
||
proxy = db.query(ProxyConfigModel).first()
|
||
_, max_total_time = get_relogin_limits()
|
||
|
||
# 批次用独立会话(runner 会在自己的线程/会话里写库),跑完再回来读结果。
|
||
runner_db = WorkerSessionLocal()
|
||
runner = LoginBatchRunner(
|
||
db=runner_db,
|
||
account_ids=account_ids,
|
||
created_by=system_user_id,
|
||
# 自动化要对全库生效,不受「只能操作分配给自己的账号」限制。
|
||
creator_permissions=["login:view_all", "cookie:operate"],
|
||
max_login_retries=int(config.get("max_login_retries", 5) or 5),
|
||
max_total_time=min(int(max_total_time), AUTO_MAX_TOTAL_TIME),
|
||
proxy_config=proxy,
|
||
log_queue=None,
|
||
loop=None,
|
||
concurrency=max(min(int(job.concurrency or 3), 8), 1),
|
||
mode="relogin",
|
||
trigger_source="automation",
|
||
)
|
||
batch_id = runner.batch_id
|
||
try:
|
||
runner.run()
|
||
finally:
|
||
runner_db.close()
|
||
|
||
# runner 在另一个会话里提交了结果:必须用新事务读回(见 settle_fresh_results)。
|
||
counters = settle_fresh_results(
|
||
batch_id,
|
||
now=now,
|
||
max_attempts=max_attempts,
|
||
expected_account_ids=account_ids,
|
||
)
|
||
paused, hard_failures, hard_failure_ratio = evaluate_failure_circuit(
|
||
counters,
|
||
config,
|
||
)
|
||
if paused:
|
||
job.enabled = False
|
||
job.updated_at = now
|
||
|
||
parts = [
|
||
f"本轮重登 {counters['scanned']} 条(有效批量 {adaptive.batch_size})",
|
||
f"成功 {counters['valid']}",
|
||
f"失败 {counters['invalid']}",
|
||
]
|
||
if counters["retry"]:
|
||
parts.append(f"待重试 {counters['retry']}")
|
||
if counters["email_quota"]:
|
||
parts.append(f"其中配额满 {counters['email_quota']}(退避到次日)")
|
||
if counters["manual"]:
|
||
parts.append(f"转人工 {counters['manual']}")
|
||
if hard_failures:
|
||
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
|
||
if paused:
|
||
parts.append("失败率过高,已自动暂停作业")
|
||
if adaptive.action != "hold":
|
||
parts.append(
|
||
f"批量策略 {adaptive.previous_batch_size}->{adaptive.batch_size}"
|
||
f"({adaptive.action},近 7 天成功率 {adaptive.success_rate:.1f}%)"
|
||
)
|
||
return {
|
||
"scanned": counters["scanned"],
|
||
"valid": counters["valid"],
|
||
"invalid": counters["invalid"],
|
||
"relogin_started": counters["scanned"],
|
||
"relogin_success": counters["valid"],
|
||
"skipped": 0,
|
||
"message": ",".join(parts),
|
||
"detail": {
|
||
"batch_id": batch_id,
|
||
"manual": counters["manual"],
|
||
"retry": counters["retry"],
|
||
"email_quota": counters["email_quota"],
|
||
"hard_failures": hard_failures,
|
||
"hard_failure_ratio": round(hard_failure_ratio, 4),
|
||
"paused": paused,
|
||
"adaptive_batch": {
|
||
"batch_size": adaptive.batch_size,
|
||
"configured_max": adaptive.configured_max,
|
||
"previous_batch_size": adaptive.previous_batch_size,
|
||
"success_rate": adaptive.success_rate,
|
||
"sample_size": adaptive.sample_size,
|
||
"pending": adaptive.pending,
|
||
"action": adaptive.action,
|
||
},
|
||
},
|
||
}
|
||
|
||
|
||
def settle_results(
|
||
db: Session,
|
||
fresh: dict[int, tuple[Account, LoginTask]],
|
||
account_ids: list[int],
|
||
*,
|
||
now: datetime,
|
||
max_attempts: int = 2,
|
||
) -> dict[str, int]:
|
||
"""批次跑完后结算:成功的清零,失败的按分型决定重试还是转人工。
|
||
|
||
解耦后 **决策看「账号 + 本轮的尝试记录」**(2026-10-04):
|
||
- 成败来自这次尝试(``LoginTask.attempt_status``);
|
||
- 退避进度写在**账号**上(``Account.relogin_*``)—— 它是账号的状态,
|
||
不该随某条历史记录一起被删掉。
|
||
|
||
单独抽出来是因为这段是 P1b 的核心决策逻辑,必须能脱离真实登录做单测。
|
||
只改 ``relogin_*`` 四个字段,**不碰 CK 本体**(CK 由 runner 在成功时替换)。
|
||
|
||
``db`` 用于写账号事件(转人工时记一条);事件是旁路,写失败不影响结算。
|
||
"""
|
||
counters = {
|
||
"scanned": len(account_ids),
|
||
"valid": 0,
|
||
"invalid": 0,
|
||
"manual": 0,
|
||
"retry": 0,
|
||
"email_quota": 0,
|
||
}
|
||
for account_id in account_ids:
|
||
entry = fresh.get(account_id)
|
||
if entry is None:
|
||
account = db.get(Account, int(account_id))
|
||
retry_at = now + timedelta(hours=1)
|
||
if account is not None:
|
||
account.relogin_next_at = retry_at
|
||
reschedule_account_work_type(
|
||
db,
|
||
account_id=int(account_id),
|
||
work_type=WORK_LOGIN_RELOGIN,
|
||
due_at=retry_at,
|
||
reason="自动重登未产生账号任务,重新排队",
|
||
)
|
||
counters["retry"] += 1
|
||
continue
|
||
account, task = entry
|
||
if task.attempt_status == "success":
|
||
counters["valid"] += 1
|
||
# 重登成功:清退避痕迹 + 重算状态(与人工登录成功同一套语义)
|
||
on_login_success(db, account, occurred_at=now)
|
||
continue
|
||
|
||
counters["invalid"] += 1
|
||
kind = classify_failure(task.attempt_message)
|
||
account.relogin_failure_kind = kind
|
||
account.relogin_last_error = (task.attempt_message or "")[:500]
|
||
account.relogin_attempts = int(account.relogin_attempts or 0) + 1
|
||
if kind in MANUAL_KINDS or account.relogin_attempts >= max_attempts:
|
||
# 转人工:极验/凭证问题重试无意义,次数用尽也不该无限撞。
|
||
counters["manual"] += 1
|
||
account.relogin_next_at = None
|
||
# 只在「转人工」时记事件:还要退避重试的不记,否则同一账号每轮都刷一条。
|
||
emit(
|
||
db,
|
||
account_id=account.id,
|
||
event_type=EVENT_RELOGIN_FAILED,
|
||
title=f"自动重登转人工({kind})",
|
||
occurred_at=now,
|
||
detail={
|
||
"kind": kind,
|
||
"attempts": account.relogin_attempts,
|
||
"error": (task.attempt_message or "")[:200],
|
||
},
|
||
source="login",
|
||
source_ref=f"relogin_failed:{task.id}:{int(now.timestamp())}",
|
||
)
|
||
# 状态跟着落到 inactive 的分支(invalid 或 manual)——推导函数负责判定
|
||
sync_status(db, account, occurred_at=now)
|
||
fail_account_work_type(
|
||
db,
|
||
account_id=int(account.id),
|
||
work_type=WORK_LOGIN_RELOGIN,
|
||
reason=f"自动重登转人工({kind})",
|
||
now=now,
|
||
)
|
||
continue
|
||
if kind == "email_quota":
|
||
counters["email_quota"] += 1
|
||
counters["retry"] += 1
|
||
account.relogin_next_at = _next_retry_at(kind, account.relogin_attempts, now)
|
||
reschedule_account_work_type(
|
||
db,
|
||
account_id=int(account.id),
|
||
work_type=WORK_LOGIN_RELOGIN,
|
||
due_at=account.relogin_next_at,
|
||
reason=f"自动重登未成功({kind}),按账号退避时间重试",
|
||
)
|
||
# 排了下一次重试 → 仍是 invalid(不是 manual)
|
||
sync_status(db, account, occurred_at=now)
|
||
return counters
|