feat(douyu): 整合自动化中心并修复重登队列

This commit is contained in:
yml
2026-11-22 21:17:37 +08:00
parent af05c54c3f
commit 94de1d167b
18 changed files with 854 additions and 1029 deletions
@@ -455,7 +455,7 @@ AND 不违反 365 天规则
- `ck_health`:只生成/执行 `ck_probe`,不处理 invalid/manual
- `ck_relogin`:读取 `login_relogin` 工作,受登录配额
- `account_overview_refresh`:拆成 points/gold/role/exchange_records 同步域
- `account_overview_refresh`:已淘汰,改由 points/gold/role/exchange_records 各同步域按需维护
- `account_capability_refresh`:变为 `storehome_sync` 补货工作
- `platform_fulfillment_lease_reaper`:保持独立
- `douyu_tasks_retention`:保持独立
@@ -444,7 +444,6 @@ AND 没有 force_refresh
| --- | --- | --- | --- |
| `ck_health` | 开启 | 每 5 分钟一轮,每轮 200,并发 5 | 高价值账号约 6 小时重检,普通账号约 24 小时 |
| `account_capability_refresh` | 开启 | 每 5 分钟一轮,每轮 100,并发 3 | 刷新有 pending 履约商品能力的账号 |
| `account_overview_refresh` | 关闭 | 每轮 300,并发 3 | 曾因单轮 29 个账号全部失败并触发熔断 |
| `ck_relogin` | 关闭 | 每轮 30,并发 3 | 自动重登自 2026-10-06 后未再运行 |
| `douyu_tasks_retention` | 开启 | 每批 2000 | 清理斗鱼任务日志 |
| `points_expire_check` | 关闭 | 每天 | 积分到期补记 |
@@ -525,14 +524,11 @@ CK 有效账号中:
`ck_relogin` 当前关闭,CK 失效账号已经积压约 2 万。直接重新开启当前配置,可能每 5 分钟
拉起最多 30 个账号,瞬时并发 3,形成不可控的连续登录。
### 账号资料刷新因熔断关闭
### 账号资料全量定时刷新已淘汰
`account_overview_refresh` 曾因一轮 29 个账号全部失败而自动关闭。重新开启前必须先解决:
- CK 质量筛选;
- 活动登录态与基础 CK 的区分;
- 并发和失败熔断;
- 轻量商品快照与完整资料同步的拆分。
`account_overview_refresh` 已从默认作业注册表移除,生产配置也会在迁移中停用。
账号资料改由按需同步、账号目录补货和履约能力补货链路维护,避免全库周期性
重查产生风控、重复日志和无意义数据库写入。
## 登录工作分类
@@ -760,7 +756,6 @@ manual:停止自动化
2. 先以每日 30-50 个小批量开启 `ck_relogin`;
3. 监控成功率和失败分型;
4. 再逐步提升到计算预算;
5. 恢复账号资料刷新的小批量试运行。
### P2:供给库存驱动
+11 -3
View File
@@ -1,6 +1,6 @@
# 自动化与资料收集
更新时间:2026-11-06
更新时间:2026-11-22
本文描述当前已落地的自动化、外部订单读取和文件存储能力。具体实现以
`web/backend/services/automation/`、`external_order_source.py` 和 `file_storage.py` 为准。
@@ -8,7 +8,9 @@
## 自动化运行模型
- 调度器随应用启动,总开关为 `AUTOMATION_ENABLED`,默认关闭。
- 每个作业还有独立开关和运行参数,默认关闭,在 `/automation` 页面配置。
- 每个作业还有独立开关和运行参数,在 `/automation`「斗鱼自动化中心」统一配置。
- 平台状态、CK 趋势、供给目标、履约保护目标、登录预算、待人工账号和运行记录
都集中在该页面;账号检测页只保留批量账号检测本职功能。
- 部署固定单容器、单 worker;作业在进程内线程池运行,重启会把未结束记录标为中断。
- 作业状态和运行记录落 `automation_jobs`、`automation_runs`,不靠内存保存进度。
- 单个作业失败只影响本轮;失败率异常时作业会自动停用,避免继续消耗账号或触发风控。
@@ -19,10 +21,16 @@
| --- | --- | --- |
| `ck_health` | 检测持有 CK 的账号 | 区分 `valid`、`invalid`、`error`;`error` 不覆盖上次有效结论 |
| `ck_relogin` | 对失效 CK 自动重登 | 有退避、次数上限和失败分型;极验/凭证/必须绑手机转人工 |
| `account_overview_refresh` | 分批刷新账号资料和一键查询快照 | 只查 CK 有效且未停用账号;“没查过”优先于“过期重查” |
| `account_sync_queue` | 独立消费账号目录、积分、鱼翅和角色同步 | 使用独立执行槽,不占用订单选号 |
| `account_capability_refresh` | 为待履约商品补充账号能力目录 | 只做本地候选筛选,外部查询进入同步队列 |
| `account_supply_replenishment` | 按供给目标缺口补充账号 | 通用补货只占低水位,需求补货优先 |
| `platform_fulfillment_lease_reaper` | 回收过期履约租约 | 扫码超时未绑定账号按有界宽限期释放 |
| `douyu_tasks_retention` | 分批清理斗鱼执行日志 | 默认启用;噪声查询保留 7 天,其余任务保留 90 天,在途任务不删 |
| `points_expire` | 补记积分到期 | 只写账号事件,不修改外部平台 |
`account_overview_refresh` 已淘汰:账号资料改由按需同步、履约能力补货和 CK
生命周期链路维护,不再做全库周期性资料重查。
所有账号级作业都必须排除 `status` 为 `disabled` / `archived` 的账号,并尊重
`auto_managed=False`。停用是“账号不可用”,`auto_managed=False` 是“账号可用但不要自动操作”,
两者不能混用。
-349
View File
@@ -1,349 +0,0 @@
"""定时补全账号资料的作业(`account_overview_refresh`)。
**为什么不能一次性跑全量**:生产约 8,000 个待查账号,一次性发起 ≈ 8 万次斗鱼请求,
很容易触发风控 —— 而风控的代价是所有账号一起受影响。所以做成定时分批,
并复用「自动巡检」的两道保护(`cookie_operable_filter` + 错误率熔断)。
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from tests.support.db import get_test_engine, make_test_session
from web.backend.models import Account, AutomationJob, User
from web.backend.services.automation.jobs import account_overview_refresh as job
class _Base:
def setup_method(self):
self.engine = get_test_engine()
self.session = make_test_session(self.engine)
self.admin = User(username="ovr-admin", password_hash="h", role="super_admin")
self.session.add(self.admin)
self.session.flush()
self.job = AutomationJob(
key=job.JOB_KEY,
title="账号资料定时补全",
# 「同一账号多久重检」= 7 天(就是界面上那个字段)
interval_seconds=7 * 24 * 3600,
batch_size=10,
concurrency=3,
enabled=False,
)
self.session.add(self.job)
self.session.commit()
def teardown_method(self):
self.session.close()
def account(
self,
name: str,
*,
ck: str = "valid",
queried_days_ago: int | None = None,
auto_managed: bool = True,
status: str = "active",
hv_checked_days_ago: float | None = None,
hv_status: str | None = None,
) -> Account:
acc = Account(
username=name,
password="p",
email=f"{name}@e.com",
email_password="ep",
cookie="acf_uid=1; acf_ccn=c",
ck_check_status=ck,
auto_managed=auto_managed,
status=status,
)
if queried_days_ago is not None:
acc.overview_queried_at = datetime.now(UTC) - timedelta(
days=queried_days_ago
)
if hv_checked_days_ago is not None:
acc.high_value_checked_at = datetime.now(UTC) - timedelta(
days=hv_checked_days_ago
)
if hv_status is not None:
acc.high_value_bairimeng_status = hv_status
acc.high_value_mengtu_status = hv_status
self.session.add(acc)
self.session.commit()
return acc
def selected(self) -> list[Account]:
return job.build_select_query(self.session, self.job, datetime.now(UTC)).all()
class TestSelection(_Base):
def test_only_valid_ck_is_selected(self):
"""只查有效 CK —— 失效的查了也是白查,只会多一条失败记录。"""
self.account("ovr-valid", ck="valid")
self.account("ovr-invalid", ck="invalid")
self.account("ovr-error", ck="error")
names = [a.username for a in self.selected()]
assert "ovr-valid" in names
assert "ovr-invalid" not in names
assert "ovr-error" not in names
def test_never_queried_comes_first(self):
"""**没查过的优先** —— 否则周期性刷新会永远挤掉从没查过的新账号。"""
self.account("ovr-old", queried_days_ago=30)
self.account("ovr-never")
assert self.selected()[0].username == "ovr-never"
def test_assigned_and_recent_login_lead_within_the_same_group(self):
"""没查过的仍整体优先;同一档里已分配的在前,再按最近登录。"""
stale = self.account("ovr-stale-assigned", queried_days_ago=30)
stale.assigned_to = self.admin.id
recent = self.account("ovr-recent")
recent.cookie_updated_at = datetime.now(UTC)
older = self.account("ovr-older")
older.cookie_updated_at = datetime.now(UTC) - timedelta(days=10)
assigned = self.account("ovr-assigned")
assigned.assigned_to = self.admin.id
assigned.cookie_updated_at = datetime.now(UTC) - timedelta(days=1)
self.session.commit()
names = [a.username for a in self.selected()]
assert names.index("ovr-assigned") < names.index("ovr-recent")
assert names.index("ovr-recent") < names.index("ovr-older")
assert names.index("ovr-older") < names.index("ovr-stale-assigned")
def test_recently_queried_is_skipped(self):
"""刚查过的要跳过(`refresh_days` 之内不再查)。"""
self.account("ovr-fresh", queried_days_ago=1)
names = [a.username for a in self.selected()]
assert "ovr-fresh" not in names
def test_stale_is_selected_again(self):
self.account("ovr-stale", queried_days_ago=30)
assert "ovr-stale" in [a.username for a in self.selected()]
def test_recheck_interval_comes_from_the_ui_field(self):
"""**用界面上那个「同一账号多久重检」** —— 与其它批量作业同一套口径。
之前这里读的是 `job.config["refresh_days"]`,于是界面上那个字段
对本作业是**死的**(改了没反应)—— 正是用户批评过的
「看起来能配、其实配的是不相干的东西」。
"""
self.job.interval_seconds = 100 * 24 * 3600
self.session.commit()
self.account("ovr-30d", queried_days_ago=30)
assert "ovr-30d" not in [a.username for a in self.selected()], (
"重检间隔 100 天时,30 天前查过的账号不该重新入选"
)
def test_manual_and_unmanaged_accounts_are_excluded(self):
"""人工状态(停用/归档)与「不参与自动化」的账号不该被自动碰。"""
self.account("ovr-disabled", status="disabled")
self.account("ovr-archived", status="archived")
self.account("ovr-unmanaged", auto_managed=False)
names = [a.username for a in self.selected()]
for bad in ("ovr-disabled", "ovr-archived", "ovr-unmanaged"):
assert bad not in names
def test_batch_size_limits_the_round(self):
for i in range(5):
self.account(f"ovr-batch-{i}")
self.job.batch_size = 2
self.session.commit()
assert len(self.selected()) == 2
def test_remaining_count_works_with_limit_applied(self):
"""回归:计数不能走 `build_select_query`。
那个查询已经 `LIMIT` 了,再 `order_by(None)` 会被 SQLAlchemy 拒绝:
"order_by() being called on a Query which already has LIMIT"。
(真踩过 —— 作业第一版就是这么挂的。)
"""
for i in range(4):
self.account(f"ovr-count-{i}")
self.job.batch_size = 1
self.session.commit()
assert job.count_remaining(self.session, self.job, datetime.now(UTC)) >= 4
class TestHighValueRefresh(_Base):
"""高价值结论有**自己独立的时钟** —— 资料刷新必须跟着它走。
缺口长这样:一键查询里商品接口单独失败(其它项都拿到了),
``overview_queried_at`` 被刷新了、``high_value_checked_at`` 却停在原地。
资料看起来很新,于是**再也不会被选中** —— 高价值状态永久停在旧结论。
"""
def test_stale_high_value_is_selected_even_if_overview_is_fresh(self):
"""资料 1 天前查过(很新),但高价值结论 30 天前 —— 必须重选。"""
self.account(
"ovr-hv-stale",
queried_days_ago=1,
hv_checked_days_ago=30,
hv_status="available",
)
assert "ovr-hv-stale" in [a.username for a in self.selected()]
def test_fresh_high_value_is_not_selected(self):
"""资料和高价值结论都新鲜 —— 不该被重选(避免重复请求)。"""
self.account(
"ovr-hv-fresh",
queried_days_ago=1,
hv_checked_days_ago=1,
hv_status="available",
)
assert "ovr-hv-fresh" not in [a.username for a in self.selected()]
def test_never_checked_high_value_is_not_selected_on_its_own(self):
"""从来没查到过高价值结论,**不**因此被重选。
这种情况要么是「账号根本还没查过资料」(本来就该被选中),
要么是**商品接口对这个账号稳定失败**(没配 manualID、账号没有宝典)。
后者若每轮都重选,同一批账号会反复撞墙,把真正有新欠账的挤掉。
「资料过期」那条已经兜住了它。
"""
self.account("ovr-hv-never", queried_days_ago=1)
assert "ovr-hv-never" not in [a.username for a in self.selected()]
def test_high_value_uses_the_same_ruler_as_the_ui_field(self):
"""高价值过期**用同一把尺子**(界面上那个「同一账号多久重检」)。
之前这里读的是 `job.config["refresh_days"]`,界面上那个字段对本作业
就是死的;同理,若高价值另设一个窗口,那个字段对本作业就变成半死的。
"""
self.job.interval_seconds = 100 * 24 * 3600
self.session.commit()
# 资料 30 天前(按 100 天不该重选),高价值结论 60 天前(同样不该)
self.account(
"ovr-hv-60d",
queried_days_ago=30,
hv_checked_days_ago=60,
hv_status="available",
)
assert "ovr-hv-60d" not in [a.username for a in self.selected()]
# 缩回 7 天 → 高价值 60 天前就过期了,要重选
self.job.interval_seconds = 7 * 24 * 3600
self.session.commit()
assert "ovr-hv-60d" in [a.username for a in self.selected()]
def test_high_value_refresh_still_requires_valid_ck(self):
"""高价值过期**不能**绕过 CK 有效性 —— CK 失效的查了也是白查。"""
self.account(
"ovr-hv-stale-invalid",
ck="invalid",
queried_days_ago=1,
hv_checked_days_ago=30,
hv_status="available",
)
assert "ovr-hv-stale-invalid" not in [a.username for a in self.selected()]
def test_invalidated_conclusion_is_rechecked_within_six_hours(self):
"""结论被**作废**(unknown)的账号,6 小时内要重查 —— 哪怕重检间隔是 7 天。
场景:CK 失效期间账号可能被别人手工登录并兑换(这些号共享给客服),
所以 CK 一判定失效就把结论作废。CK 恢复之后必须**尽快**重新确认 ——
否则运营会一直拿着一份「待确认」的结论,而它其实已经变了。
⚠️ 这里用的是「资料重检间隔」(7 天)之外的**另一个**窗口:
作废的结论等 7 天太久了,运营等不了。但也不能更密 ——
商城接口对某些账号稳定失败,那类账号会一直是 unknown,每轮都重选
就是反复撞墙,挤掉真正有新欠账的账号。
"""
# 资料 1 天前(很新,按 7 天不该重选),高价值结论是 10 小时前的 unknown
self.account(
"ovr-hv-invalidated",
queried_days_ago=1,
hv_checked_days_ago=10 / 24,
hv_status="unknown",
)
assert "ovr-hv-invalidated" in [a.username for a in self.selected()]
def test_invalidated_conclusion_is_not_rechecked_too_often(self):
"""作废的结论 6 小时内**不**重选 —— 不能反复撞同一个失败的接口。"""
self.account(
"ovr-hv-invalidated-fresh",
queried_days_ago=1,
hv_checked_days_ago=1 / 24, # 1 小时前
hv_status="unknown",
)
assert "ovr-hv-invalidated-fresh" not in [a.username for a in self.selected()]
class TestCircuitBreaker(_Base):
"""失败率过高就**自己把自己关掉**(风控保护)。
与「自动巡检」同口径。大面积失败通常意味着被风控或斗鱼接口抖动 ——
这时候继续跑只会把更多账号写坏,停手交给人确认才是对的。
"""
def _run_with(self, total: int, failed: int):
"""打桩执行器,只关心失败率熔断的逻辑。"""
from web.backend.models import DouyuTask
for i in range(total):
self.account(f"ovr-cb-{i}")
def fake_create(db, ids, task_type, scope, created_by, payload=None):
batch_id = "cb-batch"
for account_id in ids:
self.session.add(
DouyuTask(
batch_id=batch_id,
account_id=account_id,
task_type=task_type,
handbook_scope=scope,
status="planned",
created_by=created_by,
)
)
self.session.commit()
return batch_id, len(ids)
session = self.session # 闭包捕获:下面 _FakeRunner 里的 self 是 runner
class _FakeRunner:
def __init__(self, *a, **k):
pass
def run(self):
# 把前 `failed` 个标成失败,其余成功
rows = (
session.query(DouyuTask)
.filter(DouyuTask.batch_id == "cb-batch")
.order_by(DouyuTask.id.asc())
.all()
)
for index, row in enumerate(rows):
row.status = "failed" if index < failed else "success"
session.commit()
import web.backend.services.automation.jobs.account_overview_refresh as mod
original_create = mod.create_douyu_planned_tasks
original_runner = mod.DouyuBatchRunner
mod.create_douyu_planned_tasks = fake_create
mod.DouyuBatchRunner = _FakeRunner
try:
return job.run_account_overview_refresh(self.session, self.job)
finally:
mod.create_douyu_planned_tasks = original_create
mod.DouyuBatchRunner = original_runner
def test_high_failure_ratio_disables_the_job(self):
out = self._run_with(total=20, failed=15) # 75% 失败
self.session.refresh(self.job)
assert out["paused"] is True
assert self.job.enabled is False, "失败率过高应自动关闭作业"
assert "风控" in out["message"]
def test_normal_failure_ratio_keeps_job_running(self):
out = self._run_with(total=20, failed=1) # 5% 失败(真实的 CK 失效率)
self.session.refresh(self.job)
assert out["paused"] is False
assert "风控" not in out["message"]
def test_small_sample_does_not_trigger_breaker(self):
"""样本太少时比例没有意义(1 个失败就是 100%),不该熔断。"""
out = self._run_with(total=3, failed=3)
self.session.refresh(self.job)
assert out["paused"] is False, "样本不足时不该熔断"
+1
View File
@@ -550,6 +550,7 @@ class TestAutomationApi(_Base):
assert payload["scheduler"]["running"] is False
keys = [item["key"] for item in payload["jobs"]]
assert ck_health.JOB_KEY in keys
assert "account_overview_refresh" not in keys
job = next(item for item in payload["jobs"] if item["key"] == ck_health.JOB_KEY)
assert job["enabled"] is False
assert job["interval_seconds"] == 6 * 3600
+38
View File
@@ -22,6 +22,7 @@ from datetime import UTC, datetime, timedelta
from tests.support.db import get_test_engine, make_test_session
from web.backend.models import (
Account,
AccountWorkItem,
AutomationJob,
LoginAttempt,
LoginQuotaSetting,
@@ -735,6 +736,43 @@ class TestSettlementVisibility(_Base):
checker.close()
stale.close()
def test_settlement_releases_claimed_account_without_attempt(self):
"""预算跳过等无尝试记录场景也必须结束 running 工作项。"""
account = self.make_account("acc-no-attempt")
now = datetime.now(UTC)
work = AccountWorkItem(
account_id=account.id,
work_type=ck_relogin.WORK_LOGIN_RELOGIN,
priority=30,
status="running",
due_at=now,
attempt_count=1,
max_attempts=20,
dedupe_key=f"login_relogin:{account.id}",
started_at=now,
)
self.session.add(work)
self.session.commit()
counters = ck_relogin.settle_fresh_results(
"batch-without-attempts",
now=now,
max_attempts=2,
expected_account_ids=[account.id],
)
checker = make_test_session(self.engine)
try:
settled_work = checker.get(AccountWorkItem, work.id)
settled_account = checker.get(Account, account.id)
assert counters["scanned"] == 1
assert counters["retry"] == 1
assert settled_work.status == "pending"
assert settled_work.due_at >= now + timedelta(minutes=59)
assert settled_account.relogin_next_at >= now + timedelta(minutes=59)
finally:
checker.close()
def test_settlement_marks_real_success_as_valid(self):
"""结算看到 success 就必须清零退避,不能写成失败。"""
account = self.make_account(
+1 -1
View File
@@ -42,7 +42,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261122_0116"
HEAD_REVISION = "20261122_0117"
# 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = (
@@ -465,6 +465,10 @@
"20261122_0116_enable_fulfillment_lease_reaper.py": {
"revision": "20261122_0116",
"sha256": "e756ffc9e953fc1b6167376eff830f1441194636626587808e2a2746a9b16fce"
},
"20261122_0117_retire_account_overview_refresh.py": {
"revision": "20261122_0117",
"sha256": "d49c6ddabdee24fc4013e765e2305f7cf9cb4750fae914303501c3d0a1b0723d"
}
}
}
@@ -0,0 +1,39 @@
"""retire obsolete account overview refresh job.
Revision ID: 20261122_0117
Revises: 20261122_0116
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261122_0117"
down_revision: str | None = "20261122_0116"
branch_labels: str | None = None
depends_on: str | None = None
JOB_KEY = "account_overview_refresh"
def upgrade() -> None:
if not sa.inspect(op.get_bind()).has_table("automation_jobs"):
return
op.execute(
sa.text(
"""
UPDATE automation_jobs
SET enabled = 0,
next_run_at = NULL,
last_message = '功能已淘汰:账号资料改由定向同步与履约补货维护',
updated_at = CURRENT_TIMESTAMP
WHERE `key` = :key
"""
).bindparams(key=JOB_KEY)
)
def downgrade() -> None:
"""不自动恢复已淘汰作业。"""
return
+14 -2
View File
@@ -124,10 +124,20 @@ def list_jobs(
):
"""列出作业配置、最近一次结果与调度器状态。"""
ensure_jobs(db)
jobs = db.query(AutomationJob).order_by(AutomationJob.key.asc()).all()
active_job_keys = set(registered_jobs())
jobs = (
db.query(AutomationJob)
.filter(AutomationJob.key.in_(active_job_keys))
.order_by(AutomationJob.key.asc())
.all()
)
latest: dict[str, AutomationRun] = {}
for run in (
db.query(AutomationRun).order_by(AutomationRun.id.desc()).limit(200).all()
db.query(AutomationRun)
.filter(AutomationRun.job_key.in_(active_job_keys))
.order_by(AutomationRun.id.desc())
.limit(200)
.all()
):
latest.setdefault(run.job_key, run)
# 巡检与重登是两个作业,但通过数据库联动:巡检把行标成 invalid,重登作业的
@@ -390,6 +400,8 @@ def list_runs(
query = db.query(AutomationRun)
if job_key:
query = query.filter(AutomationRun.job_key == job_key)
else:
query = query.filter(AutomationRun.job_key.in_(set(registered_jobs())))
runs = query.order_by(AutomationRun.id.desc()).limit(limit).all()
return {"items": [_run_out(run) for run in runs]}
@@ -12,12 +12,6 @@ from .jobs.account_capability_refresh import (
from .jobs.account_capability_refresh import (
run_account_capability_refresh,
)
from .jobs.account_overview_refresh import (
JOB_KEY as ACCOUNT_OVERVIEW_REFRESH_KEY,
)
from .jobs.account_overview_refresh import (
run_account_overview_refresh,
)
from .jobs.account_supply_replenishment import (
JOB_KEY as ACCOUNT_SUPPLY_REPLENISHMENT_KEY,
)
@@ -64,7 +58,6 @@ from .scheduler import (
__all__ = [
"ACCOUNT_CAPABILITY_REFRESH_KEY",
"ACCOUNT_OVERVIEW_REFRESH_KEY",
"ACCOUNT_SUPPLY_REPLENISHMENT_KEY",
"ACCOUNT_SYNC_QUEUE_KEY",
"CK_HEALTH_KEY",
@@ -80,7 +73,6 @@ __all__ = [
"registered_jobs",
"reset_registry",
"run_account_capability_refresh",
"run_account_overview_refresh",
"run_account_supply_replenishment",
"run_account_sync_queue_job",
"run_ck_health",
@@ -172,33 +164,6 @@ def register_default_jobs() -> None:
run=run_ck_relogin,
)
)
register_job(
JobSpec(
key=ACCOUNT_OVERVIEW_REFRESH_KEY,
title="账号资料定时补全",
# 每 10 分钟一轮、每轮 400 个。
#
# 依据是**试水实测**,不是拍脑袋:200 个并发 3 跑了 131 秒
# = **1.53 个/秒 ≈ 5,500 个/小时**。瓶颈不在查询速度,而在调度频率。
# 400 个 × 0.65 秒 ≈ **4.4 分钟**跑完,10 分钟一轮
# = **2,400 个/小时**,7,784 个约 **3.2 小时**补完。
#
# ⚠️ 一轮时长必须**明显小于间隔**:调度器是**串行**的
# (见 scheduler.run_due_jobs 的注释:作业一个一个跑),
# 我的作业跑着的时候 CK 巡检/重登都得等着。所以宁可轮次多一些,
# 也不要一轮跑太久 —— 4.4 分钟 / 10 分钟是刻意留出的余量。
#
# 想更快就调大 `batch_size`(账号列表里那套),但**跑得越快越容易被风控**,
# 而风控是所有账号一起受影响。
# 「同一账号多久重检」= 查过的账号 7 天后再查(界面上的那个字段)。
# ⚠️ 它**不参与调度** —— 调度频率由 run_interval_seconds 决定。
interval_seconds=7 * 24 * 3600,
run_interval_seconds=5 * 60,
batch_size=400,
concurrency=3,
run=run_account_overview_refresh,
)
)
register_job(
JobSpec(
key=CK_HEALTH_KEY,
@@ -1,265 +0,0 @@
"""作业:**定时分批**补全账号基础资料(一键查询的内容)。
## 为什么不能一次性跑全量
生产有约 8,000 个「CK 有效但没查过」的账号。一次性发起等于
**~8 万次斗鱼请求**,很容易触发风控 —— 而风控的代价是所有账号一起受影响。
所以做成**定时分批**:每轮只取 `batch_size` 个(默认 30),
跑完就结束,下一轮再来。慢一点没关系,**稳**最重要。
口径与「自动巡检」(`ck_health`)一致,并复用它的两道保护:
- `cookie_operable_filter()`:账号有 CK 且没有在途尝试
- **错误率熔断**:一轮里失败比例过高(通常是风控或接口抖动)就
**自己把自己关掉**,不再继续污染账号
## 两类账号分开
| 类型 | 条件 | 优先级 |
| --- | --- | --- |
| **没查过的** | `overview_queried_at IS NULL` | **优先**(补历史欠账)|
| 查过但过期了 | `overview_queried_at < now - interval_seconds` | 其次(周期性刷新)|
同一档里面,已分配的排在未分配前面,最近登录成功(`cookie_updated_at`)的再靠前。
价值分要先有查询结果才算得出来,所以新登录、已经分给客服的号先补。
用户明确要求这两类分开 —— 否则"周期性刷新老账号"会永远挤掉"从没查过的新账号"。
## 查询记录
每次都会建真实的 `douyu_tasks` 行(`query_account_overview`),
所以「哪个账号、什么时候、查出来什么」都有据可查;
账号上的 `overview_queried_at` 则是"下次该不该再查"的依据。
## 只查有效 CK
`ck_check_status == 'valid'`。CK 失效的账号查了也是白查(只会多一条失败记录),
交给「CK 检测与重登」去处理。
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from loguru import logger
from sqlalchemy import or_
from sqlalchemy.orm import Session
from ....models import Account, AutomationJob
from ...account_lifecycle import MANUAL_STATES
from ...account_value import high_value_needs_refresh_expr
from ...douyu_runner import DouyuBatchRunner
from ...douyu_service import cookie_operable_filter, create_douyu_planned_tasks
JOB_KEY = "account_overview_refresh"
#: `interval_seconds` 缺失时的兜底(天)—— 正常都会走界面上那个「同一账号多久重检」
DEFAULT_RECHECK_SECONDS = 7 * 24 * 3600
#: 触发熔断的最小样本量:样本太少时比例没有意义(1 个失败就是 100%)
ERROR_PAUSE_MIN_SCANNED = 10
#: 失败比例超过这个值就熔断(与 ck_health 同口径)
ERROR_PAUSE_RATIO = 0.5
def _positive_int(value: Any, default: int) -> int:
try:
parsed = int(value)
except (TypeError, ValueError):
return default
return parsed if parsed > 0 else default
def build_select_query(db: Session, job: AutomationJob, now: datetime):
"""选出本轮要查的账号。
- 只要 **CK 有效**(`ck_check_status == 'valid'`)—— 失效的查了也是白查
- **没查过的优先**:`overview_queried_at IS NULL` 排在最前
(MySQL 没有 `NULLS FIRST`,用 `IS NULL` 的布尔值降序等价实现)
- 查过的只在超过 `interval_seconds` 后才重新入选
(就是界面上那个「同一账号多久重检」—— 与其它作业**同一套口径**)
"""
# ⚠️ 用 `interval_seconds`(界面上那个「同一账号多久重检」),
# 与其它批量作业**同一套口径**。
#
# 之前这里读的是 `job.config["refresh_days"]` —— 于是界面上那个字段
# 对本作业是**死的**(改了没反应)。那正是用户刚批评过的
# 「看起来能配、其实配的是不相干的东西」。
batch = _positive_int(job.batch_size, 30)
return (
_base_query(db, job, now)
# 没查过的排最前。同一档里:已分配的优先,再是最近登录成功的
# (cookie_updated_at 越新越靠前;空的排后面)。
# MySQL 没有 NULLS FIRST,用 IS NULL 的布尔值表达。
.order_by(
Account.overview_queried_at.is_(None).desc(),
Account.assigned_to.isnot(None).desc(),
Account.cookie_updated_at.is_(None).asc(),
Account.cookie_updated_at.desc(),
Account.id.asc(),
)
.limit(batch)
)
def _base_query(db: Session, job: AutomationJob, now: datetime):
"""**不带排序和 LIMIT** 的基础查询。
单独抽出来是因为计数不能再走 `build_select_query`:
SQLAlchemy 不允许在已 `LIMIT` 的查询上再 `order_by(None)`
(抛 "order_by() being called on a Query which already has LIMIT")。
"""
# ⚠️ 用 `interval_seconds`(界面上那个「同一账号多久重检」),
# 与其它批量作业**同一套口径**。
#
# 之前这里读的是 `job.config["refresh_days"]` —— 于是界面上那个字段
# 对本作业是**死的**(改了没反应)。那正是用户刚批评过的
# 「看起来能配、其实配的是不相干的东西」。
recheck_seconds = _positive_int(job.interval_seconds, DEFAULT_RECHECK_SECONDS)
cutoff = now - timedelta(seconds=recheck_seconds)
return db.query(Account).filter(
cookie_operable_filter(),
Account.auto_managed.is_(True),
Account.status.notin_(MANUAL_STATES),
Account.ck_check_status == "valid",
or_(
Account.overview_queried_at.is_(None),
Account.overview_queried_at < cutoff,
# 高价值结论有**自己独立的时钟**(`high_value_checked_at`):
# 一键查询里商品接口单独失败时,资料时间被刷新了、高价值时间却
# 停在原地;CK 失效那段时间查不动也一样。少了这一条,这类账号
# 的高价值状态会**永久**停在旧结论,没人再拉它。
#
# 用的是**同一把尺子**(作业上那个「同一账号多久重检」),
# 不是另设一个窗口 —— 否则界面上那个字段对本作业就又变成半死的。
high_value_needs_refresh_expr(now=now, max_seconds=recheck_seconds),
),
)
def count_remaining(db: Session, job: AutomationJob, now: datetime) -> int:
"""还剩多少待查(只用于提示,不参与调度)。"""
return _base_query(db, job, now).count()
def _system_user_id(db: Session) -> int:
"""任务行的 created_by —— 定时任务不是某个人的操作,记在超管名下。"""
from ....models import User
user = (
db.query(User)
.filter(User.role == "super_admin")
.order_by(User.id.asc())
.first()
)
if user is None:
raise RuntimeError("库中没有超管用户,无法为定时任务提供 created_by")
return int(user.id)
def run_refresh_for_accounts(
db: Session,
job: AutomationJob,
accounts: list[Account],
*,
now: datetime,
remaining_count: int | None = None,
source: str = "scheduled_refresh",
log_label: str = "overview",
empty_message: str = "全库暂无待补全或到期的账号",
) -> dict[str, Any]:
"""对指定账号执行资料刷新;能力刷新作业也复用这条写入链。"""
if not accounts:
return {
"scanned": 0,
"valid": 0,
"invalid": 0,
"succeeded": 0,
"failed": 0,
"message": empty_message,
}
account_ids = [a.id for a in accounts]
batch_id, count = create_douyu_planned_tasks(
db,
account_ids,
"query_account_overview",
"elite",
_system_user_id(db),
# 标上来源,便于在任务列表里区分「定时巡检」与「人工点查询」
{"source": source},
)
if count == 0:
return {
"scanned": 0,
"valid": 0,
"invalid": 0,
"succeeded": 0,
"failed": 0,
"message": "候选账号均不可查(可能正有在途任务)",
}
# 复用现有执行器:并发、代理、进度、失败处理都是现成的
# ⚠️ 必须 close_db=False,避免 runner 结束时把调度线程的 session 关闭
runner = DouyuBatchRunner(
db,
batch_id,
"query_account_overview",
concurrency=max(1, min(int(job.concurrency or 1), 10)),
close_db=False,
)
runner.run()
db.commit()
from ....models import DouyuTask
rows = db.query(DouyuTask).filter(DouyuTask.batch_id == batch_id).all()
succeeded = sum(1 for t in rows if t.status == "success")
failed = sum(1 for t in rows if t.status == "failed")
error_ratio = failed / len(rows) if rows else 0.0
paused = False
if len(rows) >= ERROR_PAUSE_MIN_SCANNED and error_ratio > ERROR_PAUSE_RATIO:
# 大面积失败通常是风控或斗鱼接口抖动。继续跑只会把更多账号写坏,
# 所以**自己把自己关掉**,交给人来确认(与 ck_health 同口径)。
job.enabled = False
db.commit()
paused = True
logger.warning(
f"[{log_label}] 失败率 {error_ratio:.0%} 过高,已自动关闭作业"
f"(batch={batch_id},failed={failed}/{len(rows)})—— 请确认是否被风控"
)
message = f"本轮查 {len(rows)} 个:成功 {succeeded} / 失败 {failed}"
if remaining_count is not None:
message += f"(剩 {remaining_count} 个待查)"
if paused:
# 把**可能的原因**写进消息里:运维看作业历史时才知道该去查什么,
# 否则只看到"失败率高"会一头雾水。
message += ";失败率过高,已自动关闭作业(大面积失败通常是**被风控**或接口抖动,请先确认再重开)"
logger.info(f"[{log_label}] {message}")
return {
"scanned": len(rows),
"valid": succeeded,
"invalid": failed,
"succeeded": succeeded,
"failed": failed,
"batch_id": batch_id,
"paused": paused,
"message": message,
}
def run_account_overview_refresh(db: Session, job: AutomationJob) -> dict[str, Any]:
"""本轮:选一批 → 建任务 → 就地跑完 → 看失败率决定要不要熔断。"""
now = datetime.now(UTC)
accounts = build_select_query(db, job, now).all()
return run_refresh_for_accounts(
db,
job,
accounts,
now=now,
remaining_count=count_remaining(db, job, now),
)
@@ -194,7 +194,11 @@ def _load_batch_state(
def settle_fresh_results(
batch_id: str, *, now: datetime, max_attempts: int = 2
batch_id: str,
*,
now: datetime,
max_attempts: int = 2,
expected_account_ids: list[int] | tuple[int, ...] | None = None,
) -> dict[str, int]:
"""在**新会话(新事务)**里读回批次结果并结算。
@@ -209,10 +213,21 @@ def settle_fresh_results(
session = WorkerSessionLocal()
try:
fresh, order = _load_batch_state(session, batch_id)
# runner 可能因登录预算、账号预检等原因没有生成 LoginTask。此时也必须
# 结算本轮已经 claim 的账号,否则工作项会永久停在 running,并阻塞后续
# 所有重登候选。2026-11-22 生产曾因此卡住 5 条、堵住 261 条积压。
settle_account_ids = list(
dict.fromkeys(
[
*order,
*(int(account_id) for account_id in expected_account_ids or ()),
]
)
)
counters = settle_results(
session,
fresh,
order,
settle_account_ids,
now=now,
max_attempts=max_attempts,
)
@@ -560,7 +575,12 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
runner_db.close()
# runner 在另一个会话里提交了结果:必须用新事务读回(见 settle_fresh_results)。
counters = settle_fresh_results(batch_id, now=now, max_attempts=max_attempts)
counters = settle_fresh_results(
batch_id,
now=now,
max_attempts=max_attempts,
expected_account_ids=account_ids,
)
paused, hard_failures, hard_failure_ratio = evaluate_failure_circuit(
counters,
config,
@@ -649,13 +669,18 @@ def settle_results(
for account_id in account_ids:
entry = fresh.get(account_id)
if entry is None:
account = db.get(Account, int(account_id))
retry_at = now + timedelta(hours=1)
if account is not None:
account.relogin_next_at = retry_at
reschedule_account_work_type(
db,
account_id=int(account_id),
work_type=WORK_LOGIN_RELOGIN,
due_at=now + timedelta(hours=1),
due_at=retry_at,
reason="自动重登未产生账号任务,重新排队",
)
counters["retry"] += 1
continue
account, task = entry
if task.attempt_status == "success":
+1 -1
View File
@@ -78,7 +78,7 @@ export default function MainLayout({ onLogout }: { onLogout?: () => void }) {
douyuItems.push({ key: '/cookie-operations', label: 'CK 检测与重登', icon: <SafetyCertificateOutlined /> });
}
if (can('automation:view')) {
douyuItems.push({ key: '/automation', label: '自动巡检', icon: <RobotOutlined /> });
douyuItems.push({ key: '/automation', label: '斗鱼自动化', icon: <RobotOutlined /> });
}
if (can('douyu:task')) {
douyuItems.push({ key: '/douyu/elite', label: '精英宝典', icon: <BookOutlined /> });
@@ -9,7 +9,6 @@ import { DownloadOutlined, PlayCircleOutlined, ReloadOutlined, StopOutlined } fr
import { accountApi, accountCheckApi, type AccountCheckBatch, type AccountCheckItem } from '../api/modules';
import { formatTime } from '../utils/time';
import { getErrorMessage } from '../utils/error';
import AccountPlatformStatusPanel from './douyu-tasks/AccountPlatformStatusPanel';
const { TextArea } = Input;
const { Text, Title } = Typography;
@@ -341,8 +340,6 @@ export default function AccountCheckPage() {
<Space orientation="vertical" size={16} style={{ width: '100%' }}>
<Title level={3} style={{ margin: 0 }}>账号检测</Title>
<AccountPlatformStatusPanel />
<Card title="选择账号来源">
<Space orientation="vertical" size={12} style={{ width: '100%' }}>
{/* 两种来源并存:手动粘贴是旧逻辑,一条没动;
File diff suppressed because it is too large Load Diff
@@ -4,6 +4,7 @@ import {
Button,
Card,
Col,
Collapse,
Empty,
Form,
Input,
@@ -80,6 +81,76 @@ interface TrendRow {
reloginOk: number;
}
function MiniTrend({
title,
values,
dates,
color,
suffix = '',
}: {
title: string;
values: number[];
dates: string[];
color: string;
suffix?: string;
}) {
const width = 260;
const height = 64;
const padding = 5;
const max = Math.max(...values, 1);
const step = values.length > 1 ? (width - padding * 2) / (values.length - 1) : 0;
const points = values.map((value, index) => {
const x = padding + step * index;
const y = height - padding - (Math.max(0, value) / max) * (height - padding * 2);
return `${x},${y}`;
}).join(' ');
const latest = values.at(-1) || 0;
return (
<div style={{ minWidth: 0 }}>
<Space size={8} align="baseline">
<Text strong>{title}</Text>
<Text type="secondary" style={{ fontSize: 12 }}>
最近 {latest}{suffix}
</Text>
</Space>
<svg
viewBox={`0 0 ${width} ${height}`}
width="100%"
height={74}
role="img"
aria-label={`${title}趋势`}
preserveAspectRatio="none"
style={{ display: 'block', marginTop: 6 }}
>
<line
x1={padding}
y1={height - padding}
x2={width - padding}
y2={height - padding}
stroke="currentColor"
opacity={0.12}
/>
{points ? (
<polyline
points={points}
fill="none"
stroke={color}
strokeWidth={2.5}
strokeLinecap="round"
strokeLinejoin="round"
vectorEffect="non-scaling-stroke"
/>
) : null}
</svg>
<div style={{ display: 'flex', justifyContent: 'space-between' }}>
<Text type="secondary" style={{ fontSize: 11 }}>{dates[0]?.slice(5) || '-'}</Text>
<Text type="secondary" style={{ fontSize: 11 }}>{dates.at(-1)?.slice(5) || '-'}</Text>
</div>
</div>
);
}
export default function AccountPlatformStatusPanel() {
const { can } = usePermissions();
const canManage = can('douyu:config');
@@ -606,16 +677,66 @@ export default function AccountPlatformStatusPanel() {
}}
/>
<Space orientation="vertical" size={8} style={{ width: '100%', marginTop: 20 }}>
<Text strong>{`近 ${trends?.days || 7} 天趋势`}</Text>
<Table
rowKey="date"
size="small"
columns={trendColumns}
dataSource={trendRows}
loading={trendsLoading}
pagination={false}
scroll={{ x: 750 }}
<Space orientation="vertical" size={12} style={{ width: '100%', marginTop: 20 }}>
<div>
<Text strong>{`近 ${trends?.days || 7} 天运行趋势`}</Text>
<Text type="secondary" style={{ marginLeft: 8 }}>
查询、登录、CK 失效和重登结果
</Text>
</div>
<Row gutter={[24, 16]}>
<Col xs={24} md={12} xl={6}>
<MiniTrend
title="查询量"
values={trendRows.map((row) => row.queryTotal)}
dates={trendRows.map((row) => row.date)}
color="#1677ff"
/>
</Col>
<Col xs={24} md={12} xl={6}>
<MiniTrend
title="登录量"
values={trendRows.map((row) => row.loginTotal)}
dates={trendRows.map((row) => row.date)}
color="#13c2c2"
/>
</Col>
<Col xs={24} md={12} xl={6}>
<MiniTrend
title="CK 失效"
values={trendRows.map((row) => row.ckInvalid)}
dates={trendRows.map((row) => row.date)}
color="#ff4d4f"
/>
</Col>
<Col xs={24} md={12} xl={6}>
<MiniTrend
title="重登成功"
values={trendRows.map((row) => row.reloginOk)}
dates={trendRows.map((row) => row.date)}
color="#52c41a"
/>
</Col>
</Row>
<Collapse
ghost
items={[
{
key: 'trend-detail',
label: '查看每日趋势明细',
children: (
<Table
rowKey="date"
size="small"
columns={trendColumns}
dataSource={trendRows}
loading={trendsLoading}
pagination={false}
scroll={{ x: 750 }}
/>
),
},
]}
/>
</Space>
</Card>
@@ -171,6 +171,21 @@ function isLoopbackPaymentUrl(value: string): boolean {
return /^https?:\/\/(localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\])(:|\/|$)/i.test(value);
}
async function waitForBatchCompletion(
handbook: HandbookKind,
batchId: string,
expectedCount: number,
): Promise<DouyuTaskItem[]> {
const deadline = Date.now() + 180_000;
while (true) {
const batchTasks = await douyuApi.listTasks(handbook, batchId);
const allFinished = batchTasks.length >= expectedCount
&& batchTasks.every((task) => !['planned', 'pending', 'running'].includes(task.status));
if (allFinished || Date.now() >= deadline) return batchTasks;
await new Promise((resolve) => window.setTimeout(resolve, 800));
}
}
/** 绑定状态小标签:取值 unknown / unbound / pending / bound。 */
function BindStatusTag({ status, label }: { status?: string; label?: string }) {
if (!status) return null;
@@ -1100,15 +1115,11 @@ export default function DouyuTasksPage({ handbook }: { handbook: HandbookKind })
const created = await startTask('query_elite_status', targetIds);
if (!created) return;
const deadline = Date.now() + 180_000;
let batchTasks: DouyuTaskItem[] = [];
while (true) {
batchTasks = await douyuApi.listTasks(handbook, created.batch_id);
const allFinished = batchTasks.length >= created.count
&& batchTasks.every((task) => !['planned', 'pending', 'running'].includes(task.status));
if (allFinished || Date.now() >= deadline) break;
await new Promise((resolve) => window.setTimeout(resolve, 800));
}
const batchTasks = await waitForBatchCompletion(
handbook,
created.batch_id,
created.count,
);
const unopenedIds = new Set(
batchTasks