"""CK 自动重登作业(P1b)。 和巡检(P1a)是一对:巡检只产出「谁失效了」的清单,本作业负责把能救的救回来。 线上实测(2026-10-02)20 个失效账号重登 **20/20 成功** —— 失效是会话级过期, 不是凭证问题。 **核心设计:失败必须分型。** 线上数据表明 415 个重登失败里只有 35% 是真正需要 人工的(极验滑块),另外 64%(取码失败 + 重启中断)其实可自动重试;不分型就会 把可救名额一起推给人工。分型结果落 ``login_tasks.relogin_failure_kind``。 本作业**默认关闭**,且按规划 §7.4.7 分三段上线(小规模实测已完成 → 限量 → 全量)。 """ from __future__ import annotations import math import random from dataclasses import dataclass from datetime import UTC, datetime, timedelta from typing import Any from sqlalchemy import case, func, or_ from sqlalchemy.orm import Session from ....database import WorkerSessionLocal from ....models import ( Account, AutomationJob, LoginAttempt, LoginQuotaSetting, LoginTask, User, ) from ....models import ProxyConfig as ProxyConfigModel from ...account_events_service import EVENT_RELOGIN_FAILED, emit from ...account_lifecycle import ( MANUAL_KINDS, MANUAL_STATES, WORK_LOGIN_RELOGIN, on_login_success, sync_status, ) from ...account_protection_service import ( ProtectionSnapshot, load_protection_snapshot, ) from ...account_value import value_tier_expr from ...account_work_queue import ( claim_account_work, enqueue_account_work, fail_account_work_type, reschedule_account_work_type, ) from ...douyu_service import cookie_holding_filter, cookie_operable_filter from ...login_failure import classify_failure from ...login_service import LoginBatchRunner, get_relogin_limits JOB_KEY = "ck_relogin" # 失败分型 → 后续动作。 # `MANUAL_KINDS`(极验 / 凭证 → 转人工)已挪到 `services/account_lifecycle.py`: # 生命周期状态推导要用它,service 不该反过来依赖 job。 # 可自动重试的:取码失败/超时(多为邮件延迟)、中断(服务重启或人工停止)、 # 网络异常。email_quota 单列,因为它要退避到次日而不是按小时退避。 RETRYABLE_KINDS = frozenset({"email_code", "interrupted", "network"}) # 退避阶梯(分钟):1h → 3h → 12h → 24h,超出取最后一级。 BACKOFF_MINUTES = (60, 180, 720, 1440) # 自动重登时给 runner 的总时长上限。默认的 600 秒会长时间占住调度线程, # 自动化场景压到 240 秒:一轮跑不完的账号留着下一轮(它们本来就有退避)。 AUTO_MAX_TOTAL_TIME = 240 # 邮箱配额打满后退避到次日零点,再叠加随机抖动,避免整点齐发。 QUOTA_JITTER_MINUTES = 30 DEFAULT_FAILURE_PAUSE_RATIO = 0.4 DEFAULT_FAILURE_PAUSE_MIN_SCANNED = 10 DEFAULT_ADAPTIVE_INITIAL_BATCH = 10 DEFAULT_ADAPTIVE_WINDOW_DAYS = 7 DEFAULT_ADAPTIVE_MIN_SAMPLE = 10 DEFAULT_ADAPTIVE_GROW_RATE = 90.0 DEFAULT_ADAPTIVE_SHRINK_RATE = 70.0 DEFAULT_ADAPTIVE_GROW_FACTOR = 1.25 @dataclass(frozen=True) class AdaptiveBatchDecision: batch_size: int configured_max: int previous_batch_size: int success_rate: float sample_size: int pending: int action: str def _cutoff(now: datetime) -> datetime: return now def build_select_query( db: Session, job: AutomationJob, now: datetime, *, max_attempts: int = 2, system_user_id: int | None = None, batch_limit: int | None = None, protection: ProtectionSnapshot | None = None, ): """构造本轮重登候选的查询(与取行分开,便于按方言编译做回归)。 解耦后**按账号选行**(2026-10-04):CK 与 ``ck_check_status`` / ``relogin_*`` 都在 ``accounts`` 上。以前是 ``db.query(LoginTask).join(Account)`` —— 那样 「没有登录历史的账号」永远进不了重登队列,而重登本来只关心账号。 「在途尝试」仍然要查 ``login_tasks``(那是某次尝试的属性),由 ``cookie_operable_filter`` 的 NOT EXISTS 子查询完成,走迁移 0042 新增的 ``(account_id, attempt_status)`` 索引。 """ query = ( db.query(Account) .filter(Account.ck_check_status == "invalid") .filter(cookie_operable_filter()) .filter(Account.auto_managed.is_(True)) # 停用/归档的账号一律不碰(生命周期状态机,见 account_lifecycle)。 .filter(Account.status.notin_(MANUAL_STATES)) .filter(Account.relogin_attempts < max_attempts) .filter( or_( Account.relogin_next_at.is_(None), Account.relogin_next_at <= _cutoff(now), ) ) ) # 已判定需要人工的类型不再自动重试(极验/凭证)。空值用 coalesce 兜住历史值。 query = query.filter( func.coalesce(Account.relogin_failure_kind, "").notin_( tuple(sorted(MANUAL_KINDS)) ) ) snapshot = protection or load_protection_snapshot(db) suppressed_ids = snapshot.suppressed_relogin_ids if suppressed_ids: # 保护组可以分别关闭自动重登。账号同时属于任一已开启自动重登的组时, # 仍按允许处理;只有完全被抑制的账号才排除。 query = query.filter(~Account.id.in_(suppressed_ids)) relogin_ids = snapshot.relogin_ids if system_user_id is not None: pass # 预留:将来若需要限制到某个归属人,在这里加过滤 # 名额先给第 1、2 档(两套还能换 / 还剩一套 / 积分≥300)。 # 同一档里仍按退避时间,避免把还没到点的高价值号提前拉起来。 limit = int(batch_limit if batch_limit is not None else job.batch_size or 0) ordering = [ Account.relogin_next_at.is_(None).desc(), Account.relogin_next_at.asc(), Account.id.asc(), ] if relogin_ids: ordering.insert( 0, case((Account.id.in_(relogin_ids), 0), else_=1), ) ordering.insert( 1 if relogin_ids else 0, value_tier_expr().asc(), ) return query.order_by(*ordering).limit(max(limit, 1)) def _load_batch_state( session: Session, batch_id: str ) -> tuple[dict[int, tuple[Account, LoginTask]], list[int]]: """读回某一批次产生的尝试记录,并配对其所属账号。 按 ``attempt_batch_id`` 找 —— ``_begin_attempt`` 会给每一条被本批次接管的记录 打上批次号。所以即使记录是**复用**的旧行,也能被准确对应回本轮。 """ tasks = ( session.query(LoginTask) .filter(LoginTask.attempt_batch_id == batch_id) .order_by(LoginTask.id.asc()) .all() ) state: dict[int, tuple[Account, LoginTask]] = {} order: list[int] = [] for task in tasks: account = session.get(Account, task.account_id) if account is None: continue state[account.id] = (account, task) order.append(account.id) return state, order def settle_fresh_results( batch_id: str, *, now: datetime, max_attempts: int = 2, expected_account_ids: list[int] | tuple[int, ...] | None = None, ) -> dict[str, int]: """在**新会话(新事务)**里读回批次结果并结算。 必须新开会话,不能复用调度器传进来的 ``db``:那个会话在挑候选时就已经开了 事务,而 MySQL 默认隔离级别是 **REPEATABLE READ** —— 在同一事务里重新查询 只会看到事务开始时的快照,看不到 runner 在另一个会话里提交的成功结果。 线上为此踩过一次:10 条重登**实际全部成功**,结算却按失败写回,给它们加了 退避时间,其中 2 条还被标成「极验」进了待人工队列。SQLite 的读一致性与 MySQL 不同,这类问题在 SQLite 上复现不出来。 """ session = WorkerSessionLocal() try: fresh, order = _load_batch_state(session, batch_id) # runner 可能因登录预算、账号预检等原因没有生成 LoginTask。此时也必须 # 结算本轮已经 claim 的账号,否则工作项会永久停在 running,并阻塞后续 # 所有重登候选。2026-11-22 生产曾因此卡住 5 条、堵住 261 条积压。 settle_account_ids = list( dict.fromkeys( [ *order, *(int(account_id) for account_id in expected_account_ids or ()), ] ) ) counters = settle_results( session, fresh, settle_account_ids, now=now, max_attempts=max_attempts, ) session.commit() return counters finally: session.close() def count_pending_relogin(db: Session, *, max_attempts: int = 2) -> int: """待自动重登的**积压**条数(给页面显示用)。 刻意不复用 ``build_select_query``:那个查询带 ``LIMIT batch_size``,用它取 count 只会得到批次上限(页面上显示恒等于 10),完全看不出积压。这里也**不 按退避窗口过滤** —— 退避是「什么时候试」的细节,不影响「还有多少要处理」。 """ snapshot = load_protection_snapshot(db) query = ( db.query(func.count(Account.id)) .filter(Account.ck_check_status == "invalid") .filter(cookie_holding_filter()) .filter(Account.auto_managed.is_(True)) # 停用/归档不计入积压(它们不会被自动处理,显示出来只会误导) .filter(Account.status.notin_(MANUAL_STATES)) .filter(Account.relogin_attempts < max_attempts) .filter( func.coalesce(Account.relogin_failure_kind, "").notin_( tuple(sorted(MANUAL_KINDS)) ) ) ) if snapshot.suppressed_relogin_ids: query = query.filter(~Account.id.in_(snapshot.suppressed_relogin_ids)) return int(query.scalar() or 0) def _system_user_id(db: Session) -> int | None: """自动重登要用一个真实用户身份落 ``created_by``(外键非空,且便于追责)。 取最早创建的超管作为「系统操作人」;库是空的就返回 None,作业会跳过本轮。 """ row = ( db.query(User.id) .filter(User.role == "super_admin") .order_by(User.id.asc()) .first() ) if row: return int(row[0]) row = db.query(User.id).order_by(User.id.asc()).first() return int(row[0]) if row else None def _next_retry_at(kind: str, attempts: int, now: datetime) -> datetime: if kind == "email_quota": tomorrow = (now + timedelta(days=1)).replace( hour=0, minute=0, second=0, microsecond=0 ) return tomorrow + timedelta(minutes=random.uniform(0, QUOTA_JITTER_MINUTES)) step = BACKOFF_MINUTES[min(max(attempts, 1), len(BACKOFF_MINUTES)) - 1] return now + timedelta(minutes=step) def evaluate_failure_circuit( counters: dict[str, int], config: dict[str, Any], ) -> tuple[bool, int, float]: """评估硬失败率;邮箱配额满属于预期供给限制,不计入熔断。""" scanned = max(0, int(counters.get("scanned", 0) or 0)) invalid = max(0, int(counters.get("invalid", 0) or 0)) email_quota = max(0, int(counters.get("email_quota", 0) or 0)) hard_failures = max(0, invalid - email_quota) minimum = max( 1, int( config.get( "failure_pause_min_scanned", DEFAULT_FAILURE_PAUSE_MIN_SCANNED, ) ), ) raw_ratio = float( config.get("failure_pause_ratio", DEFAULT_FAILURE_PAUSE_RATIO) ) threshold = raw_ratio / 100 if raw_ratio > 1 else raw_ratio threshold = max(0.0, min(threshold, 1.0)) ratio = hard_failures / scanned if scanned else 0.0 return scanned >= minimum and ratio > threshold, hard_failures, ratio def resolve_failure_circuit_config( db: Session, config: dict[str, Any] | None, ) -> dict[str, Any]: """合并作业配置和在线登录预算里的失败暂停阈值。""" resolved = dict(config or {}) if "failure_pause_ratio" not in resolved: setting = ( db.query(LoginQuotaSetting) .filter(LoginQuotaSetting.id == 1) .first() ) if setting is not None: resolved["failure_pause_ratio"] = int( setting.failure_pause_ratio ) return resolved def _recent_relogin_success( db: Session, *, now: datetime, window_days: int, ) -> tuple[int, float]: start_day = now.date() - timedelta(days=max(window_days, 1) - 1) rows = ( db.query( LoginAttempt.status, LoginAttempt.failure_kind, func.count(LoginAttempt.id), ) .filter( LoginAttempt.quota_day >= start_day, LoginAttempt.category.in_( ("relogin_high_value", "relogin_ordinary") ), ) .group_by(LoginAttempt.status, LoginAttempt.failure_kind) .all() ) success = 0 failed = 0 for status, failure_kind, count in rows: count_value = int(count or 0) if status == "success": success += count_value elif status in {"failed", "error", "stopped"} and failure_kind != "email_quota": failed += count_value sample = success + failed rate = success * 100 / sample if sample else 0.0 return sample, round(rate, 1) def resolve_adaptive_batch_size( db: Session, job: AutomationJob, *, now: datetime, pending: int, ) -> AdaptiveBatchDecision: """Scale the effective relogin batch inside the operator-configured ceiling.""" config = dict(job.config or {}) configured_max = max(int(job.batch_size or 1), 1) if not bool(config.get("adaptive_batch_enabled", True)): decision = AdaptiveBatchDecision( batch_size=configured_max, configured_max=configured_max, previous_batch_size=configured_max, success_rate=0.0, sample_size=0, pending=max(0, int(pending)), action="fixed", ) config["adaptive_batch_size"] = decision.batch_size job.config = config return decision window_days = max( 1, int(config.get("adaptive_window_days", DEFAULT_ADAPTIVE_WINDOW_DAYS) or 1), ) min_sample = max( 1, int(config.get("adaptive_min_sample", DEFAULT_ADAPTIVE_MIN_SAMPLE) or 1), ) grow_rate = float( config.get("adaptive_grow_rate", DEFAULT_ADAPTIVE_GROW_RATE) ) shrink_rate = float( config.get("adaptive_shrink_rate", DEFAULT_ADAPTIVE_SHRINK_RATE) ) grow_factor = max( 1.01, float(config.get("adaptive_grow_factor", DEFAULT_ADAPTIVE_GROW_FACTOR)), ) initial = min( configured_max, max( 1, int( config.get( "adaptive_initial_batch", DEFAULT_ADAPTIVE_INITIAL_BATCH, ) or 1 ), ), ) previous = min( configured_max, max(1, int(config.get("adaptive_batch_size", initial) or initial)), ) sample, success_rate = _recent_relogin_success( db, now=now, window_days=window_days, ) target = previous action = "hold" if sample >= min_sample and success_rate < shrink_rate: target = max(1, previous // 2) action = "shrink" elif sample >= min_sample and success_rate >= grow_rate: target = min( configured_max, max(previous + 1, math.ceil(previous * grow_factor)), ) action = "grow" if target > previous else "hold" elif sample < min_sample: action = "cold_start" if pending > 0: target = min(target, max(1, int(pending))) decision = AdaptiveBatchDecision( batch_size=max(1, target), configured_max=configured_max, previous_batch_size=previous, success_rate=success_rate, sample_size=sample, pending=max(0, int(pending)), action=action, ) config["adaptive_batch_size"] = decision.batch_size config["adaptive_last_success_rate"] = decision.success_rate config["adaptive_last_sample"] = decision.sample_size job.config = config return decision def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]: now = datetime.now(UTC) config = resolve_failure_circuit_config(db, job.config) max_attempts = int(config.get("max_attempts_per_day", 2) or 2) system_user_id = _system_user_id(db) if system_user_id is None: return { "scanned": 0, "valid": 0, "invalid": 0, "skipped": 0, "message": "库里没有可用用户,无法以系统身份发起重登", } pending = count_pending_relogin(db, max_attempts=max_attempts) adaptive = resolve_adaptive_batch_size( db, job, now=now, pending=pending, ) protection = load_protection_snapshot(db) accounts = build_select_query( db, job, now, max_attempts=max_attempts, batch_limit=adaptive.batch_size, protection=protection, ).all() if not accounts: db.commit() return {"scanned": 0, "valid": 0, "invalid": 0, "skipped": 0} selected_account_ids = tuple(int(account.id) for account in accounts) for account in accounts: priority = 10 if int(account.id) in protection.relogin_ids else 30 enqueue_account_work( db, account_id=int(account.id), work_type=WORK_LOGIN_RELOGIN, priority=priority, max_attempts=20, due_at=now, reason="CK 失效,等待自动重登", ) db.commit() work_items = claim_account_work( db, work_types=(WORK_LOGIN_RELOGIN,), account_ids=selected_account_ids, limit=len(selected_account_ids), now=now, ) db.commit() claimed_account_ids = [int(item.account_id) for item in work_items] if not claimed_account_ids: return { "scanned": 0, "valid": 0, "invalid": 0, "skipped": len(selected_account_ids), "message": "候选账号的重登工作项已被其他执行器领取", } account_map = { int(account.id): account for account in db.query(Account) .filter(Account.id.in_(claimed_account_ids)) .all() } accounts = [ account_map[account_id] for account_id in claimed_account_ids if account_id in account_map ] # 调度单位是**账号**(CK 在账号上);登录记录只是这次尝试的历史。 account_ids = [account.id for account in accounts] proxy = db.query(ProxyConfigModel).first() _, max_total_time = get_relogin_limits() # 批次用独立会话(runner 会在自己的线程/会话里写库),跑完再回来读结果。 runner_db = WorkerSessionLocal() runner = LoginBatchRunner( db=runner_db, account_ids=account_ids, created_by=system_user_id, # 自动化要对全库生效,不受「只能操作分配给自己的账号」限制。 creator_permissions=["login:view_all", "cookie:operate"], max_login_retries=int(config.get("max_login_retries", 5) or 5), max_total_time=min(int(max_total_time), AUTO_MAX_TOTAL_TIME), proxy_config=proxy, log_queue=None, loop=None, concurrency=max(min(int(job.concurrency or 3), 8), 1), mode="relogin", trigger_source="automation", ) batch_id = runner.batch_id try: runner.run() finally: runner_db.close() # runner 在另一个会话里提交了结果:必须用新事务读回(见 settle_fresh_results)。 counters = settle_fresh_results( batch_id, now=now, max_attempts=max_attempts, expected_account_ids=account_ids, ) paused, hard_failures, hard_failure_ratio = evaluate_failure_circuit( counters, config, ) if paused: job.enabled = False job.updated_at = now parts = [ f"本轮重登 {counters['scanned']} 条(有效批量 {adaptive.batch_size})", f"成功 {counters['valid']}", f"失败 {counters['invalid']}", ] if counters["retry"]: parts.append(f"待重试 {counters['retry']}") if counters["email_quota"]: parts.append(f"其中配额满 {counters['email_quota']}(退避到次日)") if counters["manual"]: parts.append(f"转人工 {counters['manual']}") if hard_failures: parts.append(f"硬失败 {hard_failures}({hard_failure_ratio:.0%})") if paused: parts.append("失败率过高,已自动暂停作业") if adaptive.action != "hold": parts.append( f"批量策略 {adaptive.previous_batch_size}->{adaptive.batch_size}" f"({adaptive.action},近 7 天成功率 {adaptive.success_rate:.1f}%)" ) return { "scanned": counters["scanned"], "valid": counters["valid"], "invalid": counters["invalid"], "relogin_started": counters["scanned"], "relogin_success": counters["valid"], "skipped": 0, "message": ",".join(parts), "detail": { "batch_id": batch_id, "manual": counters["manual"], "retry": counters["retry"], "email_quota": counters["email_quota"], "hard_failures": hard_failures, "hard_failure_ratio": round(hard_failure_ratio, 4), "paused": paused, "adaptive_batch": { "batch_size": adaptive.batch_size, "configured_max": adaptive.configured_max, "previous_batch_size": adaptive.previous_batch_size, "success_rate": adaptive.success_rate, "sample_size": adaptive.sample_size, "pending": adaptive.pending, "action": adaptive.action, }, }, } def settle_results( db: Session, fresh: dict[int, tuple[Account, LoginTask]], account_ids: list[int], *, now: datetime, max_attempts: int = 2, ) -> dict[str, int]: """批次跑完后结算:成功的清零,失败的按分型决定重试还是转人工。 解耦后 **决策看「账号 + 本轮的尝试记录」**(2026-10-04): - 成败来自这次尝试(``LoginTask.attempt_status``); - 退避进度写在**账号**上(``Account.relogin_*``)—— 它是账号的状态, 不该随某条历史记录一起被删掉。 单独抽出来是因为这段是 P1b 的核心决策逻辑,必须能脱离真实登录做单测。 只改 ``relogin_*`` 四个字段,**不碰 CK 本体**(CK 由 runner 在成功时替换)。 ``db`` 用于写账号事件(转人工时记一条);事件是旁路,写失败不影响结算。 """ counters = { "scanned": len(account_ids), "valid": 0, "invalid": 0, "manual": 0, "retry": 0, "email_quota": 0, } for account_id in account_ids: entry = fresh.get(account_id) if entry is None: account = db.get(Account, int(account_id)) retry_at = now + timedelta(hours=1) if account is not None: account.relogin_next_at = retry_at reschedule_account_work_type( db, account_id=int(account_id), work_type=WORK_LOGIN_RELOGIN, due_at=retry_at, reason="自动重登未产生账号任务,重新排队", ) counters["retry"] += 1 continue account, task = entry if task.attempt_status == "success": counters["valid"] += 1 # 重登成功:清退避痕迹 + 重算状态(与人工登录成功同一套语义) on_login_success(db, account, occurred_at=now) continue counters["invalid"] += 1 kind = classify_failure(task.attempt_message) account.relogin_failure_kind = kind account.relogin_last_error = (task.attempt_message or "")[:500] account.relogin_attempts = int(account.relogin_attempts or 0) + 1 if kind in MANUAL_KINDS or account.relogin_attempts >= max_attempts: # 转人工:极验/凭证问题重试无意义,次数用尽也不该无限撞。 counters["manual"] += 1 account.relogin_next_at = None # 只在「转人工」时记事件:还要退避重试的不记,否则同一账号每轮都刷一条。 emit( db, account_id=account.id, event_type=EVENT_RELOGIN_FAILED, title=f"自动重登转人工({kind})", occurred_at=now, detail={ "kind": kind, "attempts": account.relogin_attempts, "error": (task.attempt_message or "")[:200], }, source="login", source_ref=f"relogin_failed:{task.id}:{int(now.timestamp())}", ) # 状态跟着落到 inactive 的分支(invalid 或 manual)——推导函数负责判定 sync_status(db, account, occurred_at=now) fail_account_work_type( db, account_id=int(account.id), work_type=WORK_LOGIN_RELOGIN, reason=f"自动重登转人工({kind})", now=now, ) continue if kind == "email_quota": counters["email_quota"] += 1 counters["retry"] += 1 account.relogin_next_at = _next_retry_at(kind, account.relogin_attempts, now) reschedule_account_work_type( db, account_id=int(account.id), work_type=WORK_LOGIN_RELOGIN, due_at=account.relogin_next_at, reason=f"自动重登未成功({kind}),按账号退避时间重试", ) # 排了下一次重试 → 仍是 invalid(不是 manual) sync_status(db, account, occurred_at=now) return counters