feat(account-events): 账号详情页与账号事件历史全链路支持
1. 新增账号事件数据表 AccountEvent 与迁移版本 20261002_0040_account_events,支持审计 Cookie 变更、自动重登、健康检查及任务执行事件; 2. 后端新增 account_events_service 并扩展 /api/accounts 路由,支持事件查询、过滤与上报; 3. 前端新增 AccountDetailPage 账号详情页组件,支持账号元信息展示、事件历史瀑布流与快捷重登操作; 4. 修复 CK 重登与自动化定时任务中事件关联逻辑,补充 test_account_events 单元测试; 5. 新增生产服务器高负载排查分析报告及相关设计文档。
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
"""账号事件流表,以及审计日志的账号关联列。
|
||||
|
||||
## 为什么需要 account_events
|
||||
|
||||
`accounts` 存的是**账号当前状态**(积分、余额、开通标记…),这些字段都是覆盖式更新 ——
|
||||
「什么时候变成这样」没有任何记录。而账号相关的操作散在 `douyu_tasks`(22 万行 / 687MB,
|
||||
掺着大量例行轮询),既不能直接给用户看,也不适合当历史长期保留。
|
||||
|
||||
于是把「发生过什么」独立成一张 append-only 的事件表:`accounts` 是状态,`account_events`
|
||||
是流水。账号详情页 = 状态卡 + 事件时间线。
|
||||
|
||||
**不做历史回填**(2026-10-03 已定):事件从上线起往后累积,既有历史仍去各自的页面查。
|
||||
|
||||
## 为什么审计日志要加 account_id
|
||||
|
||||
实测 `audit_logs.target` 是自由文本(`冰柳192` / `导入103个` / `douyu_batch:xxx` 混着),
|
||||
无法可靠关联到账号,导致「谁什么时候分配/改标签/删除了这个账号」进不了账号时间线。
|
||||
加一列可空的 account_id,账号相关操作写入时带上即可(旧记录保持 NULL)。
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20261002_0040"
|
||||
down_revision: str | None = "20261002_0039"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _tables(bind) -> set[str]:
|
||||
return set(sa.inspect(bind).get_table_names())
|
||||
|
||||
|
||||
def _columns(bind, table: str) -> set[str]:
|
||||
return {item["name"] for item in sa.inspect(bind).get_columns(table)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
|
||||
if "account_events" not in _tables(bind):
|
||||
op.create_table(
|
||||
"account_events",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True),
|
||||
sa.Column(
|
||||
"account_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("accounts.id"),
|
||||
nullable=False,
|
||||
),
|
||||
# import / login_ok / ck_invalid / relogin_ok / relogin_failed /
|
||||
# bind_ok / recharge / exchange / points_credit /
|
||||
# elite_opened / esports_opened / assigned / tag_changed ...
|
||||
sa.Column("event_type", sa.String(length=32), nullable=False),
|
||||
sa.Column("occurred_at", sa.DateTime(), nullable=False),
|
||||
# 直接可展示的一句话,避免前端再各自拼文案
|
||||
sa.Column(
|
||||
"title", sa.String(length=256), nullable=False, server_default=""
|
||||
),
|
||||
# 变动量(充值/消耗/到账,有符号)与变动后余额
|
||||
sa.Column("amount", sa.Integer(), nullable=True),
|
||||
sa.Column("balance_after", sa.Integer(), nullable=True),
|
||||
# 精简细节(不是原始大 JSON),便于排障与以后扩展
|
||||
sa.Column("detail", sa.JSON(), nullable=True),
|
||||
# 来源:login / task / audit / manual
|
||||
sa.Column(
|
||||
"source", sa.String(length=24), nullable=False, server_default=""
|
||||
),
|
||||
# 幂等键:如 douyu_task:123456、login_task:14166、audit:789。
|
||||
# 收集器重跑、事件重复发射都靠它兜住,漏了会造出重复时间线。
|
||||
# **可空且唯一**:MySQL 的唯一索引允许多个 NULL,所以「没有幂等键」的事件
|
||||
# (如手工记录)存 NULL;不能用空串 —— 空串会互相冲突,第二条就插不进去。
|
||||
sa.Column("source_ref", sa.String(length=96), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_account_events_account_occurred",
|
||||
"account_events",
|
||||
["account_id", "occurred_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_account_events_event_type", "account_events", ["event_type"]
|
||||
)
|
||||
op.create_index(
|
||||
"ix_account_events_source_ref",
|
||||
"account_events",
|
||||
["source_ref"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
if "account_id" not in _columns(bind, "audit_logs"):
|
||||
op.add_column(
|
||||
"audit_logs",
|
||||
sa.Column("account_id", sa.Integer(), nullable=True),
|
||||
)
|
||||
op.create_index("ix_audit_logs_account_id", "audit_logs", ["account_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
if "account_id" in _columns(bind, "audit_logs"):
|
||||
op.drop_column("audit_logs", "account_id")
|
||||
if "account_events" in _tables(bind):
|
||||
# 删表会一并删掉索引
|
||||
op.drop_table("account_events")
|
||||
@@ -6,6 +6,7 @@ from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
BigInteger,
|
||||
Boolean,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
@@ -690,11 +691,47 @@ class AuditLog(Base):
|
||||
action: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
target: Mapped[str] = mapped_column(String(256), default="")
|
||||
detail: Mapped[str] = mapped_column(Text, default="")
|
||||
# 账号相关操作用它关联到具体账号(旧记录为 NULL)。
|
||||
# target 是自由文本(用户名/批次号/整句话),不能可靠反查账号,所以单独存 id。
|
||||
account_id: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
|
||||
# 旧记录未标记结果时为 NULL,避免把历史操作误判成失败。
|
||||
success: Mapped[bool | None] = mapped_column(Boolean, nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow)
|
||||
|
||||
|
||||
class AccountEvent(Base):
|
||||
"""账号事件流(append-only)。
|
||||
|
||||
`accounts` 存**当前状态**,这张表存**发生过什么** —— 账号详情页的时间线就读它。
|
||||
状态字段都是覆盖式更新(积分、余额、开通标记),事件表补上「什么时候变成这样」。
|
||||
|
||||
设计约定:
|
||||
- `occurred_at` 是**业务发生时间**(不是入库时间);
|
||||
- `amount` 有符号(充值为正、兑换消耗为负),便于页面直接算累计;
|
||||
- `source_ref` 是幂等键,可空且唯一:收集器重跑、事件重复发射都靠它兜住;
|
||||
- `detail` 只存精简字段,不放原始大 JSON(那是 douyu_tasks 的包袱)。
|
||||
"""
|
||||
|
||||
__tablename__ = "account_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
|
||||
# 不单独建 account_id 索引:迁移建的是 (account_id, occurred_at) 复合索引,
|
||||
# 前缀已覆盖按账号查,再单建一个纯属冗余。
|
||||
account_id: Mapped[int] = mapped_column(ForeignKey("accounts.id"), nullable=False)
|
||||
event_type: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
occurred_at: Mapped[datetime] = mapped_column(UTCDateTime, nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(256), default="", server_default="")
|
||||
amount: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
balance_after: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
detail: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
source: Mapped[str] = mapped_column(String(24), default="", server_default="")
|
||||
# 幂等键:可空且唯一(MySQL 允许多个 NULL,便于「没有幂等键」的事件共存)
|
||||
source_ref: Mapped[str | None] = mapped_column(
|
||||
String(96), nullable=True, index=True, unique=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow)
|
||||
|
||||
|
||||
class AutomationJob(Base):
|
||||
"""自动化作业定义。状态落库,进程重启不丢调度进度。"""
|
||||
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
"""账号管理路由"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy import func, or_, select, text
|
||||
from sqlalchemy.orm import Session, defer, joinedload
|
||||
|
||||
from ..database import get_db
|
||||
from ..deps import get_current_user, require_permission
|
||||
from ..models import (
|
||||
Account,
|
||||
AccountEvent,
|
||||
AuditLog,
|
||||
DouyuTask,
|
||||
DouyuWorkbenchAccount,
|
||||
@@ -24,6 +27,13 @@ from ..schemas import (
|
||||
AccountTag,
|
||||
BatchAssign,
|
||||
)
|
||||
from ..services.account_events_service import (
|
||||
EVENT_CK_INVALID,
|
||||
EVENT_ELITE_OPENED,
|
||||
EVENT_ESPORTS_OPENED,
|
||||
EVENT_EXCHANGE,
|
||||
EVENT_RECHARGE,
|
||||
)
|
||||
from ..services.account_service import (
|
||||
cookie_account_ids_query,
|
||||
parse_and_build_accounts,
|
||||
@@ -565,6 +575,168 @@ def batch_delete_accounts_selection(
|
||||
return {"message": f"已删除 {deleted} 个账号", "deleted": deleted, "success": True}
|
||||
|
||||
|
||||
@router.get("/{account_id}/overview")
|
||||
def account_overview(
|
||||
account_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
current: User = Depends(get_current_user),
|
||||
):
|
||||
"""账号详情页的「状态卡 + 统计」。
|
||||
|
||||
状态取 `accounts` 的当前值(覆盖式),统计由 `account_events` 聚合 —— 这正是
|
||||
「状态与流水分离」的用法:页面上两者拼起来看。
|
||||
"""
|
||||
acc = _visible_accounts_query(db, current).filter(Account.id == account_id).first()
|
||||
if not acc:
|
||||
raise HTTPException(status_code=404, detail="账号不存在")
|
||||
|
||||
stats = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT
|
||||
SUM(CASE WHEN event_type = :recharge THEN amount ELSE 0 END) AS recharge_total,
|
||||
SUM(CASE WHEN event_type = :exchange THEN 1 ELSE 0 END) AS exchange_count,
|
||||
SUM(CASE WHEN event_type = :exchange THEN -amount ELSE 0 END) AS points_spent,
|
||||
SUM(CASE WHEN event_type = :ck_invalid THEN 1 ELSE 0 END) AS ck_invalid_count,
|
||||
MIN(CASE WHEN event_type = :elite THEN occurred_at END) AS elite_opened_at,
|
||||
MIN(CASE WHEN event_type = :esports THEN occurred_at END) AS esports_opened_at,
|
||||
COUNT(*) AS event_count
|
||||
FROM account_events WHERE account_id = :aid
|
||||
"""
|
||||
),
|
||||
{
|
||||
"aid": account_id,
|
||||
"recharge": EVENT_RECHARGE,
|
||||
"exchange": EVENT_EXCHANGE,
|
||||
"ck_invalid": EVENT_CK_INVALID,
|
||||
"elite": EVENT_ELITE_OPENED,
|
||||
"esports": EVENT_ESPORTS_OPENED,
|
||||
},
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
or {}
|
||||
)
|
||||
|
||||
# 登录次数取历史记录(事件表不回填,所以 CK 相关计数仍以 login_tasks 为准)。
|
||||
login_stats = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT COUNT(*) AS login_count,
|
||||
MAX(finished_at) AS last_login_at,
|
||||
MAX(ck_checked_at) AS last_ck_check_at
|
||||
FROM login_tasks
|
||||
WHERE account_id = :aid AND cookie IS NOT NULL AND cookie <> ''
|
||||
"""
|
||||
),
|
||||
{"aid": account_id},
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
or {}
|
||||
)
|
||||
|
||||
return {
|
||||
"id": acc.id,
|
||||
"username": acc.username,
|
||||
"nickname": acc.nickname,
|
||||
"uid": acc.uid,
|
||||
"tag": acc.tag,
|
||||
"remark": acc.remark,
|
||||
"assigned_to": acc.assigned_to,
|
||||
"created_at": acc.created_at, # 导入时间
|
||||
# 当前状态(覆盖式字段)
|
||||
"points": acc.points,
|
||||
"gold_balance": acc.gold_balance,
|
||||
"exchange_balance": acc.exchange_balance,
|
||||
"game_name": acc.game_name,
|
||||
"game_channel": acc.game_channel,
|
||||
"bind_status": acc.bind_status,
|
||||
"elite_opened": acc.elite_opened,
|
||||
"elite_expire_time": acc.elite_expire_time,
|
||||
"esports_points": acc.esports_points,
|
||||
"esports_bind_status": acc.esports_bind_status,
|
||||
"esports_game_name": acc.esports_game_name,
|
||||
"esports_can_change_time": acc.esports_can_change_time,
|
||||
"auto_managed": acc.auto_managed,
|
||||
# 统计
|
||||
"stats": {
|
||||
"recharge_total": int(stats.get("recharge_total") or 0),
|
||||
"exchange_count": int(stats.get("exchange_count") or 0),
|
||||
"points_spent": int(stats.get("points_spent") or 0),
|
||||
"ck_invalid_count": int(stats.get("ck_invalid_count") or 0),
|
||||
"event_count": int(stats.get("event_count") or 0),
|
||||
"login_count": int(login_stats.get("login_count") or 0),
|
||||
"last_login_at": login_stats.get("last_login_at"),
|
||||
"last_ck_check_at": login_stats.get("last_ck_check_at"),
|
||||
},
|
||||
# 「开通时间」由事件表提供;事件表从上线起才有,早于上线的不显示。
|
||||
"opened_at": {
|
||||
"elite": stats.get("elite_opened_at"),
|
||||
"esports": stats.get("esports_opened_at"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{account_id}/events")
|
||||
def account_events(
|
||||
account_id: int,
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
event_type: str = "",
|
||||
db: Session = Depends(get_db),
|
||||
current: User = Depends(get_current_user),
|
||||
):
|
||||
"""账号时间线(倒序分页)。
|
||||
|
||||
权限沿用账号可见性:客服只能看自己名下账号的事件。
|
||||
"""
|
||||
acc = _visible_accounts_query(db, current).filter(Account.id == account_id).first()
|
||||
if not acc:
|
||||
raise HTTPException(status_code=404, detail="账号不存在")
|
||||
|
||||
page = max(1, int(page))
|
||||
page_size = max(1, min(int(page_size), 200))
|
||||
where = "WHERE account_id = :aid"
|
||||
params: dict[str, Any] = {"aid": account_id}
|
||||
if event_type:
|
||||
where += " AND event_type = :etype"
|
||||
params["etype"] = event_type
|
||||
|
||||
total = (
|
||||
db.execute(
|
||||
text(f"SELECT COUNT(*) FROM account_events {where}"), params
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT id, event_type, occurred_at, title, amount, balance_after,
|
||||
detail, source
|
||||
FROM account_events {where}
|
||||
ORDER BY occurred_at DESC, id DESC
|
||||
LIMIT :limit OFFSET :offset
|
||||
"""
|
||||
),
|
||||
{**params, "limit": page_size, "offset": (page - 1) * page_size},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
|
||||
return {
|
||||
"items": [dict(row) for row in rows],
|
||||
"total": int(total),
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{account_id}")
|
||||
def delete_account(
|
||||
account_id: int,
|
||||
@@ -585,6 +757,11 @@ def delete_account(
|
||||
db.query(DouyuWorkbenchAccount).filter(
|
||||
DouyuWorkbenchAccount.account_id == account_id
|
||||
).delete(synchronize_session=False)
|
||||
# 事件表有指向 accounts 的外键,不清会直接删不掉账号。
|
||||
# 账号都没了,它的时间线也没有保留意义(审计仍在 audit_logs 里,那份不带外键)。
|
||||
db.query(AccountEvent).filter(AccountEvent.account_id == account_id).delete(
|
||||
synchronize_session=False
|
||||
)
|
||||
|
||||
db.add(
|
||||
AuditLog(
|
||||
@@ -592,6 +769,7 @@ def delete_account(
|
||||
username=current.username,
|
||||
action="account:delete",
|
||||
target=acc.username,
|
||||
account_id=account_id,
|
||||
)
|
||||
)
|
||||
db.delete(acc)
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
"""账号事件流:写入与查询。
|
||||
|
||||
## 定位
|
||||
|
||||
`accounts` 存**当前状态**(积分、余额、开通标记),这张表存**发生过什么**。
|
||||
账号详情页 = 状态卡(读 `accounts`)+ 时间线(读本表)。
|
||||
|
||||
## 两条写入路径
|
||||
|
||||
1. **直接发射**(`emit`):CK 失效/登录成功/重登、精英宝典与电竞手册开通 —— 这些
|
||||
只在内存里出现过,不记就永远没有;
|
||||
2. **增量收集**(`automation/jobs/account_events_collect.py`):充值/兑换/积分到账/绑定 ——
|
||||
这些已经是 `douyu_tasks` 的行,由收集器按水位线往后提取,不动 runner 代码。
|
||||
本模块只提供 `_task_to_event` 映射给它用。
|
||||
|
||||
## 两条硬约束
|
||||
|
||||
- **发射是旁路**:`emit` 任何失败都不能影响登录/兑换主流程(内部吞异常 + 记日志,
|
||||
并用 SAVEPOINT 隔离,避免污染调用方的事务);
|
||||
- **幂等**:`source_ref` 唯一键兜住「收集器重跑」「事件重复发射」。没有它,重跑一次
|
||||
时间线就多一份重复记录。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# ── 事件类型 ──────────────────────────────────────────────
|
||||
# 命名保持「名词_状态」风格,便于前端归类与筛选。
|
||||
EVENT_IMPORTED = "imported" # 账号导入(读 accounts.created_at,一般不落表)
|
||||
EVENT_LOGIN_OK = "login_ok" # 登录成功(含新 CK)
|
||||
EVENT_CK_INVALID = "ck_invalid" # CK 判定失效
|
||||
EVENT_RELOGIN_OK = "relogin_ok" # 自动重登成功
|
||||
EVENT_RELOGIN_FAILED = "relogin_failed" # 自动重登失败(转人工/退避)
|
||||
EVENT_BIND_OK = "bind_ok" # 绑定/换绑
|
||||
EVENT_RECHARGE = "recharge" # 充值到账
|
||||
EVENT_EXCHANGE = "exchange" # 兑换(含消耗积分)
|
||||
EVENT_POINTS_CREDIT = "points_credit" # 积分到账(含到期时间)
|
||||
EVENT_ELITE_OPENED = "elite_opened" # 精英宝典开通
|
||||
EVENT_ESPORTS_OPENED = "esports_opened" # 电竞手册开通
|
||||
EVENT_ASSIGNED = "assigned" # 分配客服
|
||||
EVENT_TAG_CHANGED = "tag_changed" # 标签变更
|
||||
|
||||
# 有符号金额的事件类型(用于页面「累计充值/累计消耗」):
|
||||
# 充值为正、兑换为负,其余不计入金额统计。
|
||||
INCOME_TYPES = (EVENT_RECHARGE,)
|
||||
OUTCOME_TYPES = (EVENT_EXCHANGE,)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def emit(
|
||||
db: Session,
|
||||
*,
|
||||
account_id: int,
|
||||
event_type: str,
|
||||
title: str,
|
||||
occurred_at: datetime | None = None,
|
||||
amount: int | None = None,
|
||||
balance_after: int | None = None,
|
||||
detail: dict[str, Any] | None = None,
|
||||
source: str = "manual",
|
||||
source_ref: str | None = None,
|
||||
) -> bool:
|
||||
"""记录一条账号事件。返回是否真的写入(幂等命中/异常时为 False)。
|
||||
|
||||
**绝不抛异常**:事件是旁路,不能因为记不上一条时间线就让登录或兑换失败。
|
||||
用 SAVEPOINT 包住,失败只回滚这一条,不影响调用方已经做过的事。
|
||||
"""
|
||||
if not account_id or not event_type:
|
||||
return False
|
||||
|
||||
sql = text(
|
||||
"""
|
||||
INSERT INTO account_events (
|
||||
account_id, event_type, occurred_at, title,
|
||||
amount, balance_after, detail, source, source_ref, created_at
|
||||
) VALUES (
|
||||
:account_id, :event_type, :occurred_at, :title,
|
||||
:amount, :balance_after, :detail, :source, :source_ref, :created_at
|
||||
)
|
||||
"""
|
||||
)
|
||||
params = {
|
||||
"account_id": account_id,
|
||||
"event_type": event_type,
|
||||
"occurred_at": (occurred_at or _now()).astimezone(UTC).replace(tzinfo=None),
|
||||
"title": (title or "")[:256],
|
||||
"amount": amount,
|
||||
"balance_after": balance_after,
|
||||
"detail": json.dumps(detail, ensure_ascii=False, default=str)
|
||||
if detail
|
||||
else None,
|
||||
"source": source[:24],
|
||||
"source_ref": (source_ref or None),
|
||||
"created_at": _now().replace(tzinfo=None),
|
||||
}
|
||||
|
||||
try:
|
||||
# SAVEPOINT:即使这条插入失败(例如 source_ref 撞唯一键),
|
||||
# 也只回滚它自己,调用方事务里已完成的写入不受影响。
|
||||
with db.begin_nested():
|
||||
db.execute(sql, params)
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001 - 旁路,任何失败都不上抛
|
||||
logger.warning(f"写入账号事件失败(account={account_id} {event_type}):{exc}")
|
||||
return False
|
||||
|
||||
|
||||
# ── douyu_tasks 增量收集 ──────────────────────────────────
|
||||
# 只收集「业务事件」,例行轮询(query_*)一律不进时间线 —— 它们在 douyu_tasks 里
|
||||
# 占绝大多数(query_limited_goods 就 6.7 万行),收进去时间线没法看。
|
||||
COLLECTED_TASK_TYPES = (
|
||||
"exchange_goods",
|
||||
"create_gold_qr",
|
||||
"donate_elite_gift",
|
||||
"confirm_bind",
|
||||
"confirm_esports_bind",
|
||||
)
|
||||
|
||||
|
||||
def _task_to_event(task_type: str, result: Any) -> dict[str, Any] | None:
|
||||
"""把一条 douyu_tasks 行转成事件参数;解析不出返回 None(跳过并告警)。
|
||||
|
||||
⚠️ 字段路径是**抽样实测**得到的(见设计文档 §4.2),不是猜的。
|
||||
解析不出时必须跳过而不是抛出去 —— 一行坏数据不能卡住整条水位线。
|
||||
"""
|
||||
if isinstance(result, str):
|
||||
try:
|
||||
result = json.loads(result)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
|
||||
if task_type == "exchange_goods":
|
||||
name = result.get("commodity_name") or "兑换"
|
||||
score = result.get("score")
|
||||
points_after = (result.get("points_query") or {}).get("points")
|
||||
return {
|
||||
"event_type": EVENT_EXCHANGE,
|
||||
"title": f"兑换 {name}",
|
||||
"amount": -int(score) if isinstance(score, (int, float)) else None,
|
||||
"balance_after": points_after if isinstance(points_after, int) else None,
|
||||
"detail": {
|
||||
"commodity_name": name,
|
||||
"commodity_id": result.get("commodity_id"),
|
||||
"order_id": result.get("order_id"),
|
||||
"exchange_id": result.get("exchange_id"),
|
||||
"game_name": result.get("game_name"),
|
||||
"game_channel": result.get("game_channel"),
|
||||
"score": score,
|
||||
},
|
||||
}
|
||||
|
||||
if task_type == "create_gold_qr":
|
||||
amount = result.get("buy_num") or result.get("pay_amount")
|
||||
supplier = result.get("supplier_order") or {}
|
||||
try:
|
||||
amount_int = int(float(amount)) if amount is not None else None
|
||||
except (TypeError, ValueError):
|
||||
amount_int = None
|
||||
return {
|
||||
"event_type": EVENT_RECHARGE,
|
||||
"title": f"充值 {amount_int if amount_int is not None else ''}".strip(),
|
||||
"amount": amount_int,
|
||||
"balance_after": None,
|
||||
"detail": {
|
||||
"order_id": result.get("order_id"),
|
||||
"supplier_order_id": supplier.get("order_id"),
|
||||
"supplier_finished_at": supplier.get("finish_time"),
|
||||
"channel": result.get("recharge_channel"),
|
||||
"douyu_uid": result.get("douyu_uid"),
|
||||
},
|
||||
}
|
||||
|
||||
if task_type == "donate_elite_gift":
|
||||
gain = result.get("gift_points_expected_gain")
|
||||
points = result.get("points")
|
||||
will_expire = ((result.get("points_query") or {}).get("raw") or {}).get("data")
|
||||
expire_at = (
|
||||
will_expire.get("willExpireTime") if isinstance(will_expire, dict) else None
|
||||
)
|
||||
return {
|
||||
"event_type": EVENT_POINTS_CREDIT,
|
||||
"title": f"积分到账 +{gain}" if gain is not None else "积分到账",
|
||||
"amount": int(gain) if isinstance(gain, (int, float)) else None,
|
||||
"balance_after": int(points) if isinstance(points, (int, float)) else None,
|
||||
"detail": {
|
||||
"baseline": result.get("gift_points_baseline"),
|
||||
"target": result.get("gift_points_target"),
|
||||
"points_expire_at": expire_at,
|
||||
"gold_balance": result.get("gold_balance"),
|
||||
},
|
||||
}
|
||||
|
||||
if task_type in ("confirm_bind", "confirm_esports_bind"):
|
||||
return {
|
||||
"event_type": EVENT_BIND_OK,
|
||||
"title": "绑定角色" if task_type == "confirm_bind" else "电竞手册绑定",
|
||||
"amount": None,
|
||||
"balance_after": None,
|
||||
"detail": {"raw_keys": sorted(result.keys())[:20]},
|
||||
}
|
||||
|
||||
return None
|
||||
@@ -6,6 +6,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .jobs.account_events_collect import JOB_KEY as ACCOUNT_EVENTS_KEY
|
||||
from .jobs.account_events_collect import run_account_events_collect
|
||||
from .jobs.ck_health import JOB_KEY as CK_HEALTH_KEY
|
||||
from .jobs.ck_health import run_ck_health
|
||||
from .jobs.ck_relogin import JOB_KEY as CK_RELOGIN_KEY
|
||||
@@ -25,6 +27,7 @@ from .scheduler import (
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ACCOUNT_EVENTS_KEY",
|
||||
"CK_HEALTH_KEY",
|
||||
"CK_RELOGIN_KEY",
|
||||
"JobSpec",
|
||||
@@ -34,6 +37,7 @@ __all__ = [
|
||||
"register_job",
|
||||
"registered_jobs",
|
||||
"reset_registry",
|
||||
"run_account_events_collect",
|
||||
"run_ck_health",
|
||||
"run_ck_relogin",
|
||||
"run_due_jobs",
|
||||
@@ -71,3 +75,15 @@ def register_default_jobs() -> None:
|
||||
run=run_ck_health,
|
||||
)
|
||||
)
|
||||
register_job(
|
||||
JobSpec(
|
||||
key=ACCOUNT_EVENTS_KEY,
|
||||
title="账号事件收集",
|
||||
# interval_seconds 这里无实际意义(收集器每轮只按水位线往后取,
|
||||
# 取不到就空转),保留一个值以复用现有调度与页面。
|
||||
interval_seconds=10 * 60,
|
||||
batch_size=500,
|
||||
concurrency=1,
|
||||
run=run_account_events_collect,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""作业:把 douyu_tasks 里的业务操作增量收集成账号事件。
|
||||
|
||||
为什么用「收集」而不是在 runner 里直接发事件:充值/兑换/积分到账这些**已经是
|
||||
`douyu_tasks` 的行**,再去改 8 个业务文件属于重复劳动;收集器只读不写业务表,
|
||||
零侵入,且新老任务类型统一处理。
|
||||
|
||||
**不回填历史**(2026-10-03 已定):首次运行把水位线定在当前最大 id,
|
||||
只往后收集。既有历史仍去各自页面查。
|
||||
|
||||
水位线存在本作业的 `automation_jobs.config` 里,进程重启不丢。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....models import AutomationJob
|
||||
from ...account_events_service import (
|
||||
COLLECTED_TASK_TYPES,
|
||||
_task_to_event,
|
||||
emit,
|
||||
)
|
||||
|
||||
JOB_KEY = "account_events_collect"
|
||||
WATERMARK_FIELD = "watermark"
|
||||
DEFAULT_BATCH = 500
|
||||
|
||||
|
||||
def _load_config(job: AutomationJob) -> dict[str, Any]:
|
||||
config = job.config
|
||||
if isinstance(config, str):
|
||||
try:
|
||||
config = json.loads(config)
|
||||
except ValueError:
|
||||
config = {}
|
||||
return dict(config) if isinstance(config, dict) else {}
|
||||
|
||||
|
||||
def _save_watermark(db: Session, job: AutomationJob, watermark: int) -> None:
|
||||
config = _load_config(job)
|
||||
config[WATERMARK_FIELD] = int(watermark)
|
||||
job.config = config
|
||||
db.commit()
|
||||
|
||||
|
||||
def run_account_events_collect(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
config = _load_config(job)
|
||||
watermark = config.get(WATERMARK_FIELD)
|
||||
|
||||
if not isinstance(watermark, int):
|
||||
# 首次运行:水位线定在「当前最大 id」= 从此刻起往后收集,**不回溯历史**。
|
||||
max_id = db.execute(
|
||||
text("SELECT COALESCE(MAX(id), 0) FROM douyu_tasks")
|
||||
).scalar()
|
||||
watermark = int(max_id or 0)
|
||||
_save_watermark(db, job, watermark)
|
||||
logger.info(
|
||||
f"账号事件收集:首次运行,水位线定在 douyu_tasks.id={watermark}(不回溯)"
|
||||
)
|
||||
return {
|
||||
"scanned": 0,
|
||||
"skipped": 0,
|
||||
"detail": {"written": 0, "watermark": watermark, "first_run": True},
|
||||
"message": f"首次运行,水位线定在 {watermark}(不回溯历史)",
|
||||
}
|
||||
|
||||
batch_size = max(int(job.batch_size or DEFAULT_BATCH), 1)
|
||||
placeholders = ",".join(f":t{i}" for i in range(len(COLLECTED_TASK_TYPES)))
|
||||
params: dict[str, Any] = {"wm": watermark, "limit": batch_size}
|
||||
params.update({f"t{i}": t for i, t in enumerate(COLLECTED_TASK_TYPES)})
|
||||
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT id, account_id, task_type, status, result, created_at, finished_at
|
||||
FROM douyu_tasks
|
||||
WHERE id > :wm AND task_type IN ({placeholders})
|
||||
ORDER BY id ASC
|
||||
LIMIT :limit
|
||||
"""
|
||||
),
|
||||
params,
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
|
||||
written = skipped = failed = 0
|
||||
max_id = watermark
|
||||
for row in rows:
|
||||
max_id = max(max_id, int(row["id"]))
|
||||
# 失败的任务不产生业务事实:兑换失败不该记「消耗了积分」。
|
||||
if (row["status"] or "") != "success":
|
||||
skipped += 1
|
||||
continue
|
||||
payload = _task_to_event(row["task_type"], row["result"])
|
||||
if payload is None:
|
||||
# 解析不出就跳过并告警,绝不让一行坏数据卡住整条水位线。
|
||||
skipped += 1
|
||||
logger.warning(
|
||||
f"账号事件收集:无法解析 douyu_task {row['id']}({row['task_type']})"
|
||||
)
|
||||
continue
|
||||
ok = emit(
|
||||
db,
|
||||
account_id=int(row["account_id"]),
|
||||
occurred_at=row["finished_at"] or row["created_at"],
|
||||
source="task",
|
||||
source_ref=f"douyu_task:{row['id']}",
|
||||
**payload,
|
||||
)
|
||||
if ok:
|
||||
written += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
if rows:
|
||||
_save_watermark(db, job, max_id)
|
||||
|
||||
return {
|
||||
"scanned": len(rows),
|
||||
"skipped": skipped,
|
||||
"detail": {
|
||||
"written": written,
|
||||
"failed": failed,
|
||||
"watermark": max_id,
|
||||
},
|
||||
"message": f"本轮收集 {written} 条事件,跳过 {skipped} 条,水位线 {max_id}",
|
||||
}
|
||||
@@ -17,6 +17,7 @@ from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....models import Account, AutomationJob, LoginTask
|
||||
from ...account_events_service import EVENT_CK_INVALID, emit
|
||||
from ...cookie_check_service import check_douyu_cookie
|
||||
from ...douyu_service import cookie_operable_filter
|
||||
|
||||
@@ -146,6 +147,27 @@ def run_ck_health(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
"verdict": verdict,
|
||||
}
|
||||
task.ck_checked_at = now
|
||||
|
||||
# 判定失效时记一条账号事件:`ck_check_status` / `ck_checked_at` 都是**覆盖式**
|
||||
# 字段,覆盖之后就再也看不出「什么时候开始失效的」。只在真的判成 invalid 时记,
|
||||
# 避免每轮巡检都给同一个账号刷屏。
|
||||
if verdict == "invalid":
|
||||
emit(
|
||||
db,
|
||||
account_id=task.account_id,
|
||||
event_type=EVENT_CK_INVALID,
|
||||
title="CK 检测失效",
|
||||
occurred_at=now,
|
||||
detail={
|
||||
"message": result.get("message", ""),
|
||||
"nickname": result.get("nickname"),
|
||||
},
|
||||
source="login",
|
||||
# 幂等键带检测时间:同一账号**再次**失效算新事件,但同一次检测
|
||||
# 重跑(例如失败重试)不会重复入库。
|
||||
source_ref=f"ck_invalid:{task.id}:{int(now.timestamp())}",
|
||||
)
|
||||
|
||||
if index % FLUSH_EVERY == 0:
|
||||
db.commit()
|
||||
db.commit()
|
||||
|
||||
@@ -23,6 +23,7 @@ from sqlalchemy.orm import Session
|
||||
from ....database import SessionLocal
|
||||
from ....models import Account, AutomationJob, LoginTask, User
|
||||
from ....models import ProxyConfig as ProxyConfigModel
|
||||
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
|
||||
from ...douyu_service import LOGIN_ATTEMPT_IN_FLIGHT_STATUSES, cookie_holding_filter
|
||||
from ...login_service import LoginBatchRunner, get_relogin_limits
|
||||
|
||||
@@ -131,6 +132,7 @@ def settle_fresh_results(
|
||||
try:
|
||||
rows = session.query(LoginTask).filter(LoginTask.id.in_(task_ids)).all()
|
||||
counters = settle_results(
|
||||
session,
|
||||
{row.id: row for row in rows},
|
||||
task_ids,
|
||||
now=now,
|
||||
@@ -271,6 +273,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
|
||||
|
||||
|
||||
def settle_results(
|
||||
db: Session,
|
||||
fresh: dict[int, LoginTask],
|
||||
task_ids: list[int],
|
||||
*,
|
||||
@@ -281,6 +284,8 @@ def settle_results(
|
||||
|
||||
单独抽出来是因为这段是 P1b 的核心决策逻辑,必须能脱离真实登录做单测。
|
||||
只改 ``relogin_*`` 四个字段,**不碰 CK 本体**(CK 由 runner 在成功时替换)。
|
||||
|
||||
``db`` 用于写账号事件(转人工时记一条);事件是旁路,写失败不影响结算。
|
||||
"""
|
||||
counters = {
|
||||
"scanned": len(task_ids),
|
||||
@@ -311,6 +316,21 @@ def settle_results(
|
||||
# 转人工:极验/凭证问题重试无意义,次数用尽也不该无限撞。
|
||||
counters["manual"] += 1
|
||||
task.relogin_next_at = None
|
||||
# 只在「转人工」时记事件:还要退避重试的不记,否则同一账号每轮都刷一条。
|
||||
emit(
|
||||
db,
|
||||
account_id=task.account_id,
|
||||
event_type=EVENT_RELOGIN_FAILED,
|
||||
title=f"自动重登转人工({kind})",
|
||||
occurred_at=now,
|
||||
detail={
|
||||
"kind": kind,
|
||||
"attempts": task.relogin_attempts,
|
||||
"error": (task.attempt_message or "")[:200],
|
||||
},
|
||||
source="login",
|
||||
source_ref=f"relogin_failed:{task.id}:{int(now.timestamp())}",
|
||||
)
|
||||
continue
|
||||
if kind == "email_quota":
|
||||
counters["email_quota"] += 1
|
||||
|
||||
@@ -11,6 +11,7 @@ from sqlalchemy.orm import Session
|
||||
from core.douyu import DouyuActivityClient, DouyuActivityError
|
||||
|
||||
from ..models import Account, DouyuTask
|
||||
from .account_events_service import EVENT_ELITE_OPENED, EVENT_ESPORTS_OPENED, emit
|
||||
from .douyu_runner_core import DOUYU_PAYMENT_POLL_INTERVAL, DOUYU_PAYMENT_POLL_SECONDS
|
||||
from .douyu_service import account_uid, update_account_profile_from_cookie
|
||||
|
||||
@@ -376,6 +377,23 @@ class ManualMixin:
|
||||
account.bind_status = "elite_opened"
|
||||
account.elite_opened = True
|
||||
account.updated_at = datetime.now(UTC)
|
||||
# elite_opened 是**覆盖式**布尔值:开通之后再操作也看不出「什么时候开的」。
|
||||
# 在这里记一条事件,补上开通时间。
|
||||
emit(
|
||||
db,
|
||||
account_id=account.id,
|
||||
event_type=EVENT_ELITE_OPENED,
|
||||
title="精英宝典开通",
|
||||
occurred_at=account.updated_at,
|
||||
detail={
|
||||
"points": account.points,
|
||||
"expire_time": account.elite_expire_time,
|
||||
},
|
||||
source="manual",
|
||||
source_ref=(
|
||||
f"elite_opened:{account.id}:{int(account.updated_at.timestamp())}"
|
||||
),
|
||||
)
|
||||
self._mark_task(
|
||||
db, task, "success", f"精英宝典已开通,积分: {account.points}", result
|
||||
)
|
||||
@@ -462,6 +480,21 @@ class ManualMixin:
|
||||
if opened:
|
||||
account.esports_bind_status = "esports_opened"
|
||||
account.updated_at = datetime.now(UTC)
|
||||
# 与精英宝典同理:状态字段是覆盖式的,这里补上「什么时候开的」。
|
||||
# 注意只在**真正开通**这一支发事件;上面「基线查到已开通」那一支是检测,
|
||||
# 每次都发会刷屏。
|
||||
emit(
|
||||
db,
|
||||
account_id=account.id,
|
||||
event_type=EVENT_ESPORTS_OPENED,
|
||||
title="电竞手册开通",
|
||||
occurred_at=account.updated_at,
|
||||
detail={"points": account.esports_points},
|
||||
source="manual",
|
||||
source_ref=(
|
||||
f"esports_opened:{account.id}:{int(account.updated_at.timestamp())}"
|
||||
),
|
||||
)
|
||||
self._mark_task(
|
||||
db,
|
||||
task,
|
||||
|
||||
@@ -21,6 +21,7 @@ from core.douyu.proxy_fetcher import ProxyFetcher
|
||||
from ..models import Account as AccountModel
|
||||
from ..models import LoginTask
|
||||
from ..models import ProxyConfig as ProxyConfigModel
|
||||
from .account_events_service import EVENT_LOGIN_OK, EVENT_RELOGIN_OK, emit
|
||||
from .cookie_check_service import check_douyu_cookie
|
||||
from .douyu_service import cookie_holding_filter
|
||||
|
||||
@@ -136,9 +137,19 @@ def _mark_attempt_failed(
|
||||
|
||||
|
||||
def _write_login_success(
|
||||
task: LoginTask, *, cookie: str, message: str, batch_id: str
|
||||
db: Session,
|
||||
task: LoginTask,
|
||||
*,
|
||||
cookie: str,
|
||||
message: str,
|
||||
batch_id: str,
|
||||
event_type: str = EVENT_LOGIN_OK,
|
||||
) -> None:
|
||||
"""登录成功后把新 CK 写回记录:这是唯一替换旧 CK 的入口。"""
|
||||
"""登录成功后把新 CK 写回记录:这是唯一替换旧 CK 的入口。
|
||||
|
||||
顺带记一条账号事件 —— CK 是**覆盖式**字段,换掉之后就看不出「这账号什么时候
|
||||
登录成功过、换过几次 CK」。事件是旁路(emit 内部吞异常),记不上也不影响登录。
|
||||
"""
|
||||
completed_at = datetime.now(UTC)
|
||||
task.status = "success"
|
||||
task.cookie = cookie
|
||||
@@ -150,6 +161,21 @@ def _write_login_success(
|
||||
task.attempt_batch_id = batch_id
|
||||
task.attempt_at = completed_at
|
||||
|
||||
emit(
|
||||
db,
|
||||
account_id=task.account_id,
|
||||
event_type=event_type,
|
||||
title="重新登录成功,Cookie 已更新"
|
||||
if event_type == EVENT_RELOGIN_OK
|
||||
else "登录成功",
|
||||
occurred_at=completed_at,
|
||||
detail={"message": message, "batch_id": batch_id},
|
||||
source="login",
|
||||
# 幂等键用登录记录 + 完成时间:同一账号多次登录是多个事件,
|
||||
# 但同一次登录重复写回不会重复入库。
|
||||
source_ref=f"login_task:{task.id}:{int(completed_at.timestamp())}",
|
||||
)
|
||||
|
||||
|
||||
class LoginBatchRunner:
|
||||
"""批量登录执行器,在线程中运行,通过 ThreadPoolExecutor 并发登录多个账号。"""
|
||||
@@ -408,10 +434,16 @@ class LoginBatchRunner:
|
||||
message = f"重新登录成功,但 Cookie 有效性检测失败: {check_result['message']}"
|
||||
# 唯一替换旧 CK 的入口:登录成功后写回新 CK。
|
||||
_write_login_success(
|
||||
self.db,
|
||||
task,
|
||||
cookie=result.cookie,
|
||||
message=message,
|
||||
batch_id=self.batch_id,
|
||||
event_type=(
|
||||
EVENT_RELOGIN_OK
|
||||
if self.mode == "relogin"
|
||||
else EVENT_LOGIN_OK
|
||||
),
|
||||
)
|
||||
if self.mode == "relogin":
|
||||
if task.ck_check_status == "valid":
|
||||
|
||||
Reference in New Issue
Block a user