Files
live-hub-py/web/backend/services/automation/jobs/ck_relogin.py
T

740 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""CK 自动重登作业(P1b)。
和巡检(P1a)是一对:巡检只产出「谁失效了」的清单,本作业负责把能救的救回来。
线上实测(2026-10-02)20 个失效账号重登 **20/20 成功** —— 失效是会话级过期,
不是凭证问题。
**核心设计:失败必须分型。** 线上数据表明 415 个重登失败里只有 35% 是真正需要
人工的(极验滑块),另外 64%(取码失败 + 重启中断)其实可自动重试;不分型就会
把可救名额一起推给人工。分型结果落 ``login_tasks.relogin_failure_kind``。
本作业**默认关闭**,且按规划 §7.4.7 分三段上线(小规模实测已完成 → 限量 → 全量)。
"""
from __future__ import annotations
import math
import random
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import case, func, or_
from sqlalchemy.orm import Session
from ....database import WorkerSessionLocal
from ....models import (
Account,
AutomationJob,
LoginAttempt,
LoginQuotaSetting,
LoginTask,
User,
)
from ....models import ProxyConfig as ProxyConfigModel
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
from ...account_lifecycle import (
MANUAL_KINDS,
MANUAL_STATES,
WORK_LOGIN_RELOGIN,
on_login_success,
sync_status,
)
from ...account_protection_service import (
ProtectionSnapshot,
load_protection_snapshot,
)
from ...account_value import value_tier_expr
from ...account_work_queue import (
claim_account_work,
enqueue_account_work,
fail_account_work_type,
reschedule_account_work_type,
)
from ...douyu_service import cookie_holding_filter, cookie_operable_filter
from ...login_failure import classify_failure
from ...login_service import LoginBatchRunner, get_relogin_limits
JOB_KEY = "ck_relogin"
# 失败分型 → 后续动作。
# `MANUAL_KINDS`(极验 / 凭证 → 转人工)已挪到 `services/account_lifecycle.py`:
# 生命周期状态推导要用它,service 不该反过来依赖 job。
# 可自动重试的:取码失败/超时(多为邮件延迟)、中断(服务重启或人工停止)、
# 网络异常。email_quota 单列,因为它要退避到次日而不是按小时退避。
RETRYABLE_KINDS = frozenset({"email_code", "interrupted", "network"})
# 退避阶梯(分钟):1h → 3h → 12h → 24h,超出取最后一级。
BACKOFF_MINUTES = (60, 180, 720, 1440)
# 自动重登时给 runner 的总时长上限。默认的 600 秒会长时间占住调度线程,
# 自动化场景压到 240 秒:一轮跑不完的账号留着下一轮(它们本来就有退避)。
AUTO_MAX_TOTAL_TIME = 240
# 邮箱配额打满后退避到次日零点,再叠加随机抖动,避免整点齐发。
QUOTA_JITTER_MINUTES = 30
DEFAULT_FAILURE_PAUSE_RATIO = 0.4
DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10
DEFAULT_ADAPTIVE_INITIAL_BATCH = 10
DEFAULT_ADAPTIVE_WINDOW_DAYS = 7
DEFAULT_ADAPTIVE_MIN_SAMPLE = 10
DEFAULT_ADAPTIVE_GROW_RATE = 90.0
DEFAULT_ADAPTIVE_SHRINK_RATE = 70.0
DEFAULT_ADAPTIVE_GROW_FACTOR = 1.25
@dataclass(frozen=True)
class AdaptiveBatchDecision:
batch_size: int
configured_max: int
previous_batch_size: int
success_rate: float
sample_size: int
pending: int
action: str
def _cutoff(now: datetime) -> datetime:
return now
def build_select_query(
db: Session,
job: AutomationJob,
now: datetime,
*,
max_attempts: int = 2,
system_user_id: int | None = None,
batch_limit: int | None = None,
protection: ProtectionSnapshot | None = None,
):
"""构造本轮重登候选的查询(与取行分开,便于按方言编译做回归)。
解耦后**按账号选行**(2026-10-04):CK 与 ``ck_check_status`` / ``relogin_*``
都在 ``accounts`` 上。以前是 ``db.query(LoginTask).join(Account)`` —— 那样
「没有登录历史的账号」永远进不了重登队列,而重登本来只关心账号。
「在途尝试」仍然要查 ``login_tasks``(那是某次尝试的属性),由
``cookie_operable_filter`` 的 NOT EXISTS 子查询完成,走迁移 0042 新增的
``(account_id, attempt_status)`` 索引。
"""
query = (
db.query(Account)
.filter(Account.ck_check_status == "invalid")
.filter(cookie_operable_filter())
.filter(Account.auto_managed.is_(True))
# 停用/归档的账号一律不碰(生命周期状态机,见 account_lifecycle)。
.filter(Account.status.notin_(MANUAL_STATES))
.filter(Account.relogin_attempts < max_attempts)
.filter(
or_(
Account.relogin_next_at.is_(None),
Account.relogin_next_at <= _cutoff(now),
)
)
)
# 已判定需要人工的类型不再自动重试(极验/凭证)。空值用 coalesce 兜住历史值。
query = query.filter(
func.coalesce(Account.relogin_failure_kind, "").notin_(
tuple(sorted(MANUAL_KINDS))
)
)
snapshot = protection or load_protection_snapshot(db)
suppressed_ids = snapshot.suppressed_relogin_ids
if suppressed_ids:
# 保护组可以分别关闭自动重登。账号同时属于任一已开启自动重登的组时,
# 仍按允许处理;只有完全被抑制的账号才排除。
query = query.filter(~Account.id.in_(suppressed_ids))
relogin_ids = snapshot.relogin_ids
if system_user_id is not None:
pass # 预留:将来若需要限制到某个归属人,在这里加过滤
# 名额先给第 1、2 档(两套还能换 / 还剩一套 / 积分≥300)。
# 同一档里仍按退避时间,避免把还没到点的高价值号提前拉起来。
limit = int(batch_limit if batch_limit is not None else job.batch_size or 0)
ordering = [
Account.relogin_next_at.is_(None).desc(),
Account.relogin_next_at.asc(),
Account.id.asc(),
]
if relogin_ids:
ordering.insert(
0,
case((Account.id.in_(relogin_ids), 0), else_=1),
)
ordering.insert(
1 if relogin_ids else 0,
value_tier_expr().asc(),
)
return query.order_by(*ordering).limit(max(limit, 1))
def _load_batch_state(
session: Session, batch_id: str
) -> tuple[dict[int, tuple[Account, LoginTask]], list[int]]:
"""读回某一批次产生的尝试记录,并配对其所属账号。
按 ``attempt_batch_id`` 找 —— ``_begin_attempt`` 会给每一条被本批次接管的记录
打上批次号。所以即使记录是**复用**的旧行,也能被准确对应回本轮。
"""
tasks = (
session.query(LoginTask)
.filter(LoginTask.attempt_batch_id == batch_id)
.order_by(LoginTask.id.asc())
.all()
)
state: dict[int, tuple[Account, LoginTask]] = {}
order: list[int] = []
for task in tasks:
account = session.get(Account, task.account_id)
if account is None:
continue
state[account.id] = (account, task)
order.append(account.id)
return state, order
def settle_fresh_results(
batch_id: str,
*,
now: datetime,
max_attempts: int = 2,
expected_account_ids: list[int] | tuple[int, ...] | None = None,
) -> dict[str, int]:
"""在**新会话(新事务)**里读回批次结果并结算。
必须新开会话,不能复用调度器传进来的 ``db``:那个会话在挑候选时就已经开了
事务,而 MySQL 默认隔离级别是 **REPEATABLE READ** —— 在同一事务里重新查询
只会看到事务开始时的快照,看不到 runner 在另一个会话里提交的成功结果。
线上为此踩过一次:10 条重登**实际全部成功**,结算却按失败写回,给它们加了
退避时间,其中 2 条还被标成「极验」进了待人工队列。SQLite 的读一致性与 MySQL
不同,这类问题在 SQLite 上复现不出来。
"""
session = WorkerSessionLocal()
try:
fresh, order = _load_batch_state(session, batch_id)
# runner 可能因登录预算、账号预检等原因没有生成 LoginTask。此时也必须
# 结算本轮已经 claim 的账号,否则工作项会永久停在 running,并阻塞后续
# 所有重登候选。2026-11-22 生产曾因此卡住 5 条、堵住 261 条积压。
settle_account_ids = list(
dict.fromkeys(
[
*order,
*(int(account_id) for account_id in expected_account_ids or ()),
]
)
)
counters = settle_results(
session,
fresh,
settle_account_ids,
now=now,
max_attempts=max_attempts,
)
session.commit()
return counters
finally:
session.close()
def count_pending_relogin(db: Session, *, max_attempts: int = 2) -> int:
"""待自动重登的**积压**条数(给页面显示用)。
刻意不复用 ``build_select_query``:那个查询带 ``LIMIT batch_size``,用它取
count 只会得到批次上限(页面上显示恒等于 10),完全看不出积压。这里也**不
按退避窗口过滤** —— 退避是「什么时候试」的细节,不影响「还有多少要处理」。
"""
snapshot = load_protection_snapshot(db)
query = (
db.query(func.count(Account.id))
.filter(Account.ck_check_status == "invalid")
.filter(cookie_holding_filter())
.filter(Account.auto_managed.is_(True))
# 停用/归档不计入积压(它们不会被自动处理,显示出来只会误导)
.filter(Account.status.notin_(MANUAL_STATES))
.filter(Account.relogin_attempts < max_attempts)
.filter(
func.coalesce(Account.relogin_failure_kind, "").notin_(
tuple(sorted(MANUAL_KINDS))
)
)
)
if snapshot.suppressed_relogin_ids:
query = query.filter(~Account.id.in_(snapshot.suppressed_relogin_ids))
return int(query.scalar() or 0)
def _system_user_id(db: Session) -> int | None:
"""自动重登要用一个真实用户身份落 ``created_by``(外键非空,且便于追责)。
取最早创建的超管作为「系统操作人」;库是空的就返回 None,作业会跳过本轮。
"""
row = (
db.query(User.id)
.filter(User.role == "super_admin")
.order_by(User.id.asc())
.first()
)
if row:
return int(row[0])
row = db.query(User.id).order_by(User.id.asc()).first()
return int(row[0]) if row else None
def _next_retry_at(kind: str, attempts: int, now: datetime) -> datetime:
if kind == "email_quota":
tomorrow = (now + timedelta(days=1)).replace(
hour=0, minute=0, second=0, microsecond=0
)
return tomorrow + timedelta(minutes=random.uniform(0, QUOTA_JITTER_MINUTES))
step = BACKOFF_MINUTES[min(max(attempts, 1), len(BACKOFF_MINUTES)) - 1]
return now + timedelta(minutes=step)
def evaluate_failure_circuit(
counters: dict[str, int],
config: dict[str, Any],
) -> tuple[bool, int, float]:
"""评估硬失败率;邮箱配额满属于预期供给限制,不计入熔断。"""
scanned = max(0, int(counters.get("scanned", 0) or 0))
invalid = max(0, int(counters.get("invalid", 0) or 0))
email_quota = max(0, int(counters.get("email_quota", 0) or 0))
hard_failures = max(0, invalid - email_quota)
minimum = max(
1,
int(
config.get(
"failure_pause_min_scanned",
DEFAULT_FAILURE_PAUSE_MIN_SCANNED,
)
),
)
raw_ratio = float(
config.get("failure_pause_ratio", DEFAULT_FAILURE_PAUSE_RATIO)
)
threshold = raw_ratio / 100 if raw_ratio > 1 else raw_ratio
threshold = max(0.0, min(threshold, 1.0))
ratio = hard_failures / scanned if scanned else 0.0
return scanned >= minimum and ratio > threshold, hard_failures, ratio
def resolve_failure_circuit_config(
db: Session,
config: dict[str, Any] | None,
) -> dict[str, Any]:
"""合并作业配置和在线登录预算里的失败暂停阈值。"""
resolved = dict(config or {})
if "failure_pause_ratio" not in resolved:
setting = (
db.query(LoginQuotaSetting)
.filter(LoginQuotaSetting.id == 1)
.first()
)
if setting is not None:
resolved["failure_pause_ratio"] = int(
setting.failure_pause_ratio
)
return resolved
def _recent_relogin_success(
db: Session,
*,
now: datetime,
window_days: int,
) -> tuple[int, float]:
start_day = now.date() - timedelta(days=max(window_days, 1) - 1)
rows = (
db.query(
LoginAttempt.status,
LoginAttempt.failure_kind,
func.count(LoginAttempt.id),
)
.filter(
LoginAttempt.quota_day >= start_day,
LoginAttempt.category.in_(
("relogin_high_value", "relogin_ordinary")
),
)
.group_by(LoginAttempt.status, LoginAttempt.failure_kind)
.all()
)
success = 0
failed = 0
for status, failure_kind, count in rows:
count_value = int(count or 0)
if status == "success":
success += count_value
elif status in {"failed", "error", "stopped"} and failure_kind != "email_quota":
failed += count_value
sample = success + failed
rate = success * 100 / sample if sample else 0.0
return sample, round(rate, 1)
def resolve_adaptive_batch_size(
db: Session,
job: AutomationJob,
*,
now: datetime,
pending: int,
) -> AdaptiveBatchDecision:
"""Scale the effective relogin batch inside the operator-configured ceiling."""
config = dict(job.config or {})
configured_max = max(int(job.batch_size or 1), 1)
if not bool(config.get("adaptive_batch_enabled", True)):
decision = AdaptiveBatchDecision(
batch_size=configured_max,
configured_max=configured_max,
previous_batch_size=configured_max,
success_rate=0.0,
sample_size=0,
pending=max(0, int(pending)),
action="fixed",
)
config["adaptive_batch_size"] = decision.batch_size
job.config = config
return decision
window_days = max(
1,
int(config.get("adaptive_window_days", DEFAULT_ADAPTIVE_WINDOW_DAYS) or 1),
)
min_sample = max(
1,
int(config.get("adaptive_min_sample", DEFAULT_ADAPTIVE_MIN_SAMPLE) or 1),
)
grow_rate = float(
config.get("adaptive_grow_rate", DEFAULT_ADAPTIVE_GROW_RATE)
)
shrink_rate = float(
config.get("adaptive_shrink_rate", DEFAULT_ADAPTIVE_SHRINK_RATE)
)
grow_factor = max(
1.01,
float(config.get("adaptive_grow_factor", DEFAULT_ADAPTIVE_GROW_FACTOR)),
)
initial = min(
configured_max,
max(
1,
int(
config.get(
"adaptive_initial_batch",
DEFAULT_ADAPTIVE_INITIAL_BATCH,
)
or 1
),
),
)
previous = min(
configured_max,
max(1, int(config.get("adaptive_batch_size", initial) or initial)),
)
sample, success_rate = _recent_relogin_success(
db,
now=now,
window_days=window_days,
)
target = previous
action = "hold"
if sample >= min_sample and success_rate < shrink_rate:
target = max(1, previous // 2)
action = "shrink"
elif sample >= min_sample and success_rate >= grow_rate:
target = min(
configured_max,
max(previous + 1, math.ceil(previous * grow_factor)),
)
action = "grow" if target > previous else "hold"
elif sample < min_sample:
action = "cold_start"
if pending > 0:
target = min(target, max(1, int(pending)))
decision = AdaptiveBatchDecision(
batch_size=max(1, target),
configured_max=configured_max,
previous_batch_size=previous,
success_rate=success_rate,
sample_size=sample,
pending=max(0, int(pending)),
action=action,
)
config["adaptive_batch_size"] = decision.batch_size
config["adaptive_last_success_rate"] = decision.success_rate
config["adaptive_last_sample"] = decision.sample_size
job.config = config
return decision
def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
now = datetime.now(UTC)
config = resolve_failure_circuit_config(db, job.config)
max_attempts = int(config.get("max_attempts_per_day", 2) or 2)
system_user_id = _system_user_id(db)
if system_user_id is None:
return {
"scanned": 0,
"valid": 0,
"invalid": 0,
"skipped": 0,
"message": "库里没有可用用户,无法以系统身份发起重登",
}
pending = count_pending_relogin(db, max_attempts=max_attempts)
adaptive = resolve_adaptive_batch_size(
db,
job,
now=now,
pending=pending,
)
protection = load_protection_snapshot(db)
accounts = build_select_query(
db,
job,
now,
max_attempts=max_attempts,
batch_limit=adaptive.batch_size,
protection=protection,
).all()
if not accounts:
db.commit()
return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0}
selected_account_ids = tuple(int(account.id) for account in accounts)
for account in accounts:
priority = 10 if int(account.id) in protection.relogin_ids else 30
enqueue_account_work(
db,
account_id=int(account.id),
work_type=WORK_LOGIN_RELOGIN,
priority=priority,
max_attempts=20,
due_at=now,
reason="CK 失效,等待自动重登",
)
db.commit()
work_items = claim_account_work(
db,
work_types=(WORK_LOGIN_RELOGIN,),
account_ids=selected_account_ids,
limit=len(selected_account_ids),
now=now,
)
db.commit()
claimed_account_ids = [int(item.account_id) for item in work_items]
if not claimed_account_ids:
return {
"scanned": 0,
"valid": 0,
"invalid": 0,
"skipped": len(selected_account_ids),
"message": "候选账号的重登工作项已被其他执行器领取",
}
account_map = {
int(account.id): account
for account in db.query(Account)
.filter(Account.id.in_(claimed_account_ids))
.all()
}
accounts = [
account_map[account_id]
for account_id in claimed_account_ids
if account_id in account_map
]
# 调度单位是**账号**(CK 在账号上);登录记录只是这次尝试的历史。
account_ids = [account.id for account in accounts]
proxy = db.query(ProxyConfigModel).first()
_, max_total_time = get_relogin_limits()
# 批次用独立会话(runner 会在自己的线程/会话里写库),跑完再回来读结果。
runner_db = WorkerSessionLocal()
runner = LoginBatchRunner(
db=runner_db,
account_ids=account_ids,
created_by=system_user_id,
# 自动化要对全库生效,不受「只能操作分配给自己的账号」限制。
creator_permissions=["login:view_all", "cookie:operate"],
max_login_retries=int(config.get("max_login_retries", 5) or 5),
max_total_time=min(int(max_total_time), AUTO_MAX_TOTAL_TIME),
proxy_config=proxy,
log_queue=None,
loop=None,
concurrency=max(min(int(job.concurrency or 3), 8), 1),
mode="relogin",
trigger_source="automation",
)
batch_id = runner.batch_id
try:
runner.run()
finally:
runner_db.close()
# runner 在另一个会话里提交了结果:必须用新事务读回(见 settle_fresh_results)。
counters = settle_fresh_results(
batch_id,
now=now,
max_attempts=max_attempts,
expected_account_ids=account_ids,
)
paused, hard_failures, hard_failure_ratio = evaluate_failure_circuit(
counters,
config,
)
if paused:
job.enabled = False
job.updated_at = now
parts = [
f"本轮重登 {counters['scanned']} 条(有效批量 {adaptive.batch_size})",
f"成功 {counters['valid']}",
f"失败 {counters['invalid']}",
]
if counters["retry"]:
parts.append(f"待重试 {counters['retry']}")
if counters["email_quota"]:
parts.append(f"其中配额满 {counters['email_quota']}(退避到次日)")
if counters["manual"]:
parts.append(f"转人工 {counters['manual']}")
if hard_failures:
parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})")
if paused:
parts.append("失败率过高,已自动暂停作业")
if adaptive.action != "hold":
parts.append(
f"批量策略 {adaptive.previous_batch_size}->{adaptive.batch_size}"
f"({adaptive.action},近 7 天成功率 {adaptive.success_rate:.1f}%)"
)
return {
"scanned": counters["scanned"],
"valid": counters["valid"],
"invalid": counters["invalid"],
"relogin_started": counters["scanned"],
"relogin_success": counters["valid"],
"skipped": 0,
"message": ",".join(parts),
"detail": {
"batch_id": batch_id,
"manual": counters["manual"],
"retry": counters["retry"],
"email_quota": counters["email_quota"],
"hard_failures": hard_failures,
"hard_failure_ratio": round(hard_failure_ratio, 4),
"paused": paused,
"adaptive_batch": {
"batch_size": adaptive.batch_size,
"configured_max": adaptive.configured_max,
"previous_batch_size": adaptive.previous_batch_size,
"success_rate": adaptive.success_rate,
"sample_size": adaptive.sample_size,
"pending": adaptive.pending,
"action": adaptive.action,
},
},
}
def settle_results(
db: Session,
fresh: dict[int, tuple[Account, LoginTask]],
account_ids: list[int],
*,
now: datetime,
max_attempts: int = 2,
) -> dict[str, int]:
"""批次跑完后结算:成功的清零,失败的按分型决定重试还是转人工。
解耦后 **决策看「账号 + 本轮的尝试记录」**(2026-10-04):
- 成败来自这次尝试(``LoginTask.attempt_status``);
- 退避进度写在**账号**上(``Account.relogin_*``)—— 它是账号的状态,
不该随某条历史记录一起被删掉。
单独抽出来是因为这段是 P1b 的核心决策逻辑,必须能脱离真实登录做单测。
只改 ``relogin_*`` 四个字段,**不碰 CK 本体**(CK 由 runner 在成功时替换)。
``db`` 用于写账号事件(转人工时记一条);事件是旁路,写失败不影响结算。
"""
counters = {
"scanned": len(account_ids),
"valid": 0,
"invalid": 0,
"manual": 0,
"retry": 0,
"email_quota": 0,
}
for account_id in account_ids:
entry = fresh.get(account_id)
if entry is None:
account = db.get(Account, int(account_id))
retry_at = now + timedelta(hours=1)
if account is not None:
account.relogin_next_at = retry_at
reschedule_account_work_type(
db,
account_id=int(account_id),
work_type=WORK_LOGIN_RELOGIN,
due_at=retry_at,
reason="自动重登未产生账号任务,重新排队",
)
counters["retry"] += 1
continue
account, task = entry
if task.attempt_status == "success":
counters["valid"] += 1
# 重登成功:清退避痕迹 + 重算状态(与人工登录成功同一套语义)
on_login_success(db, account, occurred_at=now)
continue
counters["invalid"] += 1
kind = classify_failure(task.attempt_message)
account.relogin_failure_kind = kind
account.relogin_last_error = (task.attempt_message or "")[:500]
account.relogin_attempts = int(account.relogin_attempts or 0) + 1
if kind in MANUAL_KINDS or account.relogin_attempts >= max_attempts:
# 转人工:极验/凭证问题重试无意义,次数用尽也不该无限撞。
counters["manual"] += 1
account.relogin_next_at = None
# 只在「转人工」时记事件:还要退避重试的不记,否则同一账号每轮都刷一条。
emit(
db,
account_id=account.id,
event_type=EVENT_RELOGIN_FAILED,
title=f"自动重登转人工({kind})",
occurred_at=now,
detail={
"kind": kind,
"attempts": account.relogin_attempts,
"error": (task.attempt_message or "")[:200],
},
source="login",
source_ref=f"relogin_failed:{task.id}:{int(now.timestamp())}",
)
# 状态跟着落到 inactive 的分支(invalid 或 manual)——推导函数负责判定
sync_status(db, account, occurred_at=now)
fail_account_work_type(
db,
account_id=int(account.id),
work_type=WORK_LOGIN_RELOGIN,
reason=f"自动重登转人工({kind})",
now=now,
)
continue
if kind == "email_quota":
counters["email_quota"] += 1
counters["retry"] += 1
account.relogin_next_at = _next_retry_at(kind, account.relogin_attempts, now)
reschedule_account_work_type(
db,
account_id=int(account.id),
work_type=WORK_LOGIN_RELOGIN,
due_at=account.relogin_next_at,
reason=f"自动重登未成功({kind}),按账号退避时间重试",
)
# 排了下一次重试 → 仍是 invalid(不是 manual)
sync_status(db, account, occurred_at=now)
return counters