feat(douyu): 增加账号平台七天趋势统计

This commit is contained in:
yml
2026-11-21 23:00:32 +08:00
parent 1598fa353f
commit 3e618c1a88
9 changed files with 484 additions and 4 deletions
@@ -11,14 +11,14 @@
更新时间:2026-11-21
整体完成度约 **73%**。
整体完成度约 **75%**。
| 阶段 | 完成度 | 已落地 | 主要缺口 |
| --- | ---: | --- | --- |
| 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 |
| 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 |
| 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 |
| 阶段 3:商品与供给 | 95% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警 | 趋势统计 |
| 阶段 3:商品与供给 | 100% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警、7 天查询/登录/CK 趋势 | 无 |
| 阶段 4:安全恢复自动化 | 50% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效 | 灰度批次自动扩缩和长期趋势告警 |
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
@@ -515,6 +515,14 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。
可通过 `DOUYU_DEFAULT_SUPPLY_TARGETS_ENABLED=false` 关闭初始化。人工新增过任意
供给目标后,启动过程不会再注入或覆盖默认值。
平台状态快照同时聚合最近 7 天趋势:
- 各同步域查询量、成功率和耗时;
- 登录尝试量、成功率和分类分布;
- CK 失效、重登成功/失败事件。
趋势直接读取现有同步运行、登录尝试和账号事件账本,不维护第二份统计事实。
### 阶段 4:安全恢复自动化
- 自动重登小批量开启
@@ -611,3 +619,4 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。
| 2026-11-21 | 建立斗鱼账号平台最终实施蓝图 |
| 2026-11-21 | 增加保守默认供给目标、账号平台供给看板和库存/CK/工作队列阈值告警 |
| 2026-11-21 | 增加供给目标管理 UI,支持新增、编辑、启停和调整安全库存 |
| 2026-11-21 | 完成阶段 3:增加 7 天查询、登录和 CK 生命周期趋势,并补充同步时间索引 |
+78
View File
@@ -7,10 +7,13 @@ from datetime import UTC, datetime, timedelta
from web.backend.models import (
Account,
AccountCommodityState,
AccountEvent,
AccountLease,
AccountSupplyTarget,
AccountSyncRun,
AccountSyncState,
AccountWorkItem,
LoginAttempt,
User,
)
from web.backend.routers.douyu import create_supply_target, list_supply_targets
@@ -266,3 +269,78 @@ def test_account_platform_status_aggregates_queue_sync_and_supply(db):
assert {
alert["code"] for alert in status["alerts"]
} >= {"supply_shortage", "ck_expiring_24h"}
def test_account_platform_status_builds_seven_day_trends(db):
account = _account("platform-trends", points=400)
db.add(account)
db.flush()
db.add_all(
[
AccountSyncRun(
account_id=account.id,
domain="storehome",
started_at=NOW,
finished_at=NOW,
status="success",
duration_ms=120,
),
AccountSyncRun(
account_id=account.id,
domain="role",
started_at=NOW,
finished_at=NOW,
status="failed",
duration_ms=300,
),
LoginAttempt(
account_id=account.id,
category="manual_login",
trigger_source="manual",
started_at=NOW,
finished_at=NOW,
status="success",
quota_day=NOW.date(),
),
LoginAttempt(
account_id=account.id,
category="relogin_ordinary",
trigger_source="system",
started_at=NOW,
finished_at=NOW,
status="failed",
failure_kind="captcha",
quota_day=NOW.date(),
),
AccountEvent(
account_id=account.id,
event_type="ck_invalid",
category="auth",
occurred_at=NOW,
title="CK 失效",
source="pytest",
actor_type="system",
source_ref=f"pytest-trend:{account.id}",
),
]
)
db.commit()
status = account_platform_status(db, now=NOW)
today = NOW.date().isoformat()
query_today = next(row for row in status["trends"]["query"] if row["date"] == today)
login_today = next(row for row in status["trends"]["login"] if row["date"] == today)
lifecycle_today = next(
row for row in status["trends"]["lifecycle"] if row["date"] == today
)
assert status["trends"]["days"] == 7
assert query_today["total"] == 2
assert query_today["success"] == 1
assert query_today["failed"] == 1
assert query_today["success_rate"] == 50.0
assert "storehome" in query_today["domains"]
assert login_today["success"] == 1
assert login_today["failed"] == 1
assert login_today["success_rate"] == 50.0
assert lifecycle_today["ck_invalid"] == 1
+1 -1
View File
@@ -42,7 +42,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261121_0111"
HEAD_REVISION = "20261121_0112"
# 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = (
@@ -445,6 +445,10 @@
"20261121_0111_account_platform_foundation.py": {
"revision": "20261121_0111",
"sha256": "0cf919b766769c84d357271acbf32bd968b2b6756b04cc059b26fd9fb5d41c10"
},
"20261121_0112_account_sync_trend_index.py": {
"revision": "20261121_0112",
"sha256": "9c1ebee5fba13efc413a4dcca258fa66dfbca5b41c1bd9b89ab794c394af3fee"
}
}
}
@@ -0,0 +1,51 @@
"""add started-at index for account sync trend aggregation
The platform status panel aggregates the last seven days across all sync domains.
The existing indexes start with domain/status, so they cannot serve that range
efficiently when no domain filter is present.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261121_0112"
down_revision: str | None = "20261121_0111"
branch_labels: str | None = None
depends_on: str | None = None
INDEX_NAME = "ix_account_sync_runs_started_domain_status"
def _index_exists() -> bool:
return bool(
op.get_bind()
.execute(
sa.text(
"""
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'account_sync_runs'
AND index_name = :index_name
"""
),
{"index_name": INDEX_NAME},
)
.scalar()
)
def upgrade() -> None:
if not _index_exists():
op.create_index(
INDEX_NAME,
"account_sync_runs",
["started_at", "domain", "status"],
)
def downgrade() -> None:
if _index_exists():
op.drop_index(INDEX_NAME, table_name="account_sync_runs")
+6
View File
@@ -2449,6 +2449,12 @@ class AccountSyncRun(Base):
"status",
"started_at",
),
Index(
"ix_account_sync_runs_started_domain_status",
"started_at",
"domain",
"status",
),
)
+176 -1
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from datetime import UTC, date, datetime, timedelta
from typing import Any
from sqlalchemy import case, func
@@ -11,12 +11,186 @@ from sqlalchemy.orm import Session
from ..models import (
Account,
AccountCommodityState,
AccountEvent,
AccountSyncRun,
AccountSyncState,
AccountWorkItem,
LoginAttempt,
)
from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED
from .account_supply_service import evaluate_supply_targets
TREND_DAYS = 7
def _day_key(value: Any) -> str:
if isinstance(value, datetime):
return value.date().isoformat()
if isinstance(value, date):
return value.isoformat()
return str(value or "")[:10]
def _trend_days(moment: datetime) -> list[str]:
today = moment.date()
return [
(today - timedelta(days=offset)).isoformat()
for offset in range(TREND_DAYS - 1, -1, -1)
]
def _rate(success: int, finished: int) -> float:
if finished <= 0:
return 0.0
return round(success * 100 / finished, 1)
def _platform_trends(db: Session, moment: datetime) -> dict[str, Any]:
days = _trend_days(moment)
start_at = datetime.combine(
date.fromisoformat(days[0]),
datetime.min.time(),
tzinfo=UTC,
)
query_days = {
day: {
"date": day,
"total": 0,
"success": 0,
"failed": 0,
"running": 0,
"success_rate": 0.0,
"duration_ms": 0,
"domains": {},
}
for day in days
}
sync_day = func.date(AccountSyncRun.started_at)
sync_rows = (
db.query(
sync_day,
AccountSyncRun.domain,
AccountSyncRun.status,
func.count(AccountSyncRun.id),
func.coalesce(func.sum(AccountSyncRun.duration_ms), 0),
)
.filter(AccountSyncRun.started_at >= start_at)
.group_by(sync_day, AccountSyncRun.domain, AccountSyncRun.status)
.all()
)
for raw_day, domain, status, count, duration_ms in sync_rows:
day = _day_key(raw_day)
bucket = query_days.get(day)
if bucket is None:
continue
count_value = int(count or 0)
domain_bucket = bucket["domains"].setdefault(
str(domain),
{"total": 0, "success": 0, "failed": 0, "running": 0},
)
bucket["total"] += count_value
bucket["duration_ms"] += int(duration_ms or 0)
domain_bucket["total"] += count_value
if status in {"success", "failed", "running"}:
bucket[str(status)] += count_value
domain_bucket[str(status)] += count_value
for bucket in query_days.values():
finished = bucket["success"] + bucket["failed"]
bucket["success_rate"] = _rate(bucket["success"], finished)
for domain_bucket in bucket["domains"].values():
domain_finished = domain_bucket["success"] + domain_bucket["failed"]
domain_bucket["success_rate"] = _rate(
domain_bucket["success"],
domain_finished,
)
login_days = {
day: {
"date": day,
"total": 0,
"success": 0,
"failed": 0,
"pending": 0,
"success_rate": 0.0,
"categories": {},
}
for day in days
}
login_day = LoginAttempt.quota_day
login_rows = (
db.query(
login_day,
LoginAttempt.category,
LoginAttempt.status,
func.count(LoginAttempt.id),
)
.filter(LoginAttempt.quota_day >= date.fromisoformat(days[0]))
.group_by(login_day, LoginAttempt.category, LoginAttempt.status)
.all()
)
for raw_day, category, status, count in login_rows:
day = _day_key(raw_day)
bucket = login_days.get(day)
if bucket is None:
continue
count_value = int(count or 0)
category_bucket = bucket["categories"].setdefault(
str(category),
{"total": 0, "success": 0, "failed": 0, "pending": 0},
)
bucket["total"] += count_value
category_bucket["total"] += count_value
if status in {"success", "failed", "pending"}:
bucket[str(status)] += count_value
category_bucket[str(status)] += count_value
for bucket in login_days.values():
finished = bucket["success"] + bucket["failed"]
bucket["success_rate"] = _rate(bucket["success"], finished)
for category_bucket in bucket["categories"].values():
category_finished = category_bucket["success"] + category_bucket["failed"]
category_bucket["success_rate"] = _rate(
category_bucket["success"],
category_finished,
)
lifecycle_days = {
day: {
"date": day,
"ck_invalid": 0,
"relogin_ok": 0,
"relogin_failed": 0,
}
for day in days
}
lifecycle_events = ("ck_invalid", "relogin_ok", "relogin_failed")
event_day = func.date(AccountEvent.occurred_at)
event_rows = (
db.query(
event_day,
AccountEvent.event_type,
func.count(AccountEvent.id),
)
.filter(
AccountEvent.occurred_at >= start_at,
AccountEvent.event_type.in_(lifecycle_events),
)
.group_by(event_day, AccountEvent.event_type)
.all()
)
for raw_day, event_type, count in event_rows:
bucket = lifecycle_days.get(_day_key(raw_day))
if bucket is not None and event_type in bucket:
bucket[str(event_type)] = int(count or 0)
return {
"days": TREND_DAYS,
"from": days[0],
"to": days[-1],
"query": list(query_days.values()),
"login": list(login_days.values()),
"lifecycle": list(lifecycle_days.values()),
}
def account_platform_status(
db: Session,
@@ -251,6 +425,7 @@ def account_platform_status(
},
"supply_targets": supply_targets,
"alerts": alerts,
"trends": _platform_trends(db, moment),
}
+50
View File
@@ -783,6 +783,55 @@ export interface DouyuPlatformAlert {
count?: number;
}
export interface DouyuQueryTrendDay {
date: string;
total: number;
success: number;
failed: number;
running: number;
success_rate: number;
duration_ms: number;
domains: Record<string, {
total: number;
success: number;
failed: number;
running: number;
success_rate: number;
}>;
}
export interface DouyuLoginTrendDay {
date: string;
total: number;
success: number;
failed: number;
pending: number;
success_rate: number;
categories: Record<string, {
total: number;
success: number;
failed: number;
pending: number;
success_rate: number;
}>;
}
export interface DouyuLifecycleTrendDay {
date: string;
ck_invalid: number;
relogin_ok: number;
relogin_failed: number;
}
export interface DouyuPlatformTrends {
days: number;
from: string;
to: string;
query: DouyuQueryTrendDay[];
login: DouyuLoginTrendDay[];
lifecycle: DouyuLifecycleTrendDay[];
}
export interface DouyuAccountPlatformStatus {
checked_at: string;
accounts: {
@@ -805,6 +854,7 @@ export interface DouyuAccountPlatformStatus {
};
supply_targets: DouyuSupplyTargetStatus[];
alerts: DouyuPlatformAlert[];
trends: DouyuPlatformTrends;
}
export interface DouyuConfig {
@@ -9,6 +9,7 @@ import {
Input,
InputNumber,
Modal,
Progress,
Row,
Select,
Space,
@@ -53,6 +54,16 @@ interface TargetFormValues {
enabled: boolean;
}
interface TrendRow {
date: string;
queryTotal: number;
queryRate: number;
loginTotal: number;
loginRate: number;
ckInvalid: number;
reloginOk: number;
}
export default function AccountPlatformStatusPanel() {
const { can } = usePermissions();
const canManage = can('douyu:config');
@@ -100,6 +111,50 @@ export default function AccountPlatformStatusPanel() {
),
[status?.work_queue],
);
const trendRows = useMemo(() => {
if (!status) return [];
const rows = new Map<string, TrendRow>();
for (const item of status.trends.query) {
rows.set(item.date, {
date: item.date,
queryTotal: item.total,
queryRate: item.success_rate,
loginTotal: 0,
loginRate: 0,
ckInvalid: 0,
reloginOk: 0,
});
}
for (const item of status.trends.login) {
const row = rows.get(item.date) || {
date: item.date,
queryTotal: 0,
queryRate: 0,
loginTotal: 0,
loginRate: 0,
ckInvalid: 0,
reloginOk: 0,
};
row.loginTotal = item.total;
row.loginRate = item.success_rate;
rows.set(item.date, row);
}
for (const item of status.trends.lifecycle) {
const row = rows.get(item.date) || {
date: item.date,
queryTotal: 0,
queryRate: 0,
loginTotal: 0,
loginRate: 0,
ckInvalid: 0,
reloginOk: 0,
};
row.ckInvalid = item.ck_invalid;
row.reloginOk = item.relogin_ok;
rows.set(item.date, row);
}
return [...rows.values()].sort((left, right) => left.date.localeCompare(right.date));
}, [status]);
const openCreate = () => {
setEditing(null);
@@ -232,6 +287,46 @@ export default function AccountPlatformStatusPanel() {
}] : []),
];
const trendColumns: TableProps<TrendRow>['columns'] = [
{
title: '日期',
dataIndex: 'date',
width: 100,
render: (value: string) => value.slice(5),
},
{
title: '查询成功率',
width: 140,
render: (_, row) => (
<Progress
percent={row.queryRate}
size="small"
status={row.queryRate < 70 ? 'exception' : 'normal'}
/>
),
},
{ title: '查询量', dataIndex: 'queryTotal', width: 80 },
{
title: '登录成功率',
width: 140,
render: (_, row) => (
<Progress
percent={row.loginRate}
size="small"
status={row.loginRate < 50 ? 'exception' : 'normal'}
/>
),
},
{ title: '登录量', dataIndex: 'loginTotal', width: 80 },
{
title: 'CK 失效',
dataIndex: 'ckInvalid',
width: 90,
render: (value: number) => value > 0 ? <Text type="danger">{value}</Text> : 0,
},
{ title: '重登成功', dataIndex: 'reloginOk', width: 90 },
];
return (
<>
<Card
@@ -287,6 +382,18 @@ export default function AccountPlatformStatusPanel() {
),
}}
/>
<Space orientation="vertical" size={8} style={{ width: '100%', marginTop: 20 }}>
<Text strong>{`近 ${status?.trends.days || 7} 天趋势`}</Text>
<Table
rowKey="date"
size="small"
columns={trendColumns}
dataSource={trendRows}
pagination={false}
scroll={{ x: 750 }}
/>
</Space>
</Card>
<Modal