feat(douyu): 增加账号平台七天趋势统计

This commit is contained in:
yml
2026-11-21 23:00:32 +08:00
parent 1598fa353f
commit 3e618c1a88
9 changed files with 484 additions and 4 deletions
@@ -11,14 +11,14 @@
更新时间:2026-11-21 更新时间:2026-11-21
整体完成度约 **73%**。 整体完成度约 **75%**。
| 阶段 | 完成度 | 已落地 | 主要缺口 | | 阶段 | 完成度 | 已落地 | 主要缺口 |
| --- | ---: | --- | --- | | --- | ---: | --- | --- |
| 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 | | 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 |
| 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 | | 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 |
| 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 | | 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 |
| 阶段 3:商品与供给 | 95% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警 | 趋势统计 | | 阶段 3:商品与供给 | 100% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警、7 天查询/登录/CK 趋势 | 无 |
| 阶段 4:安全恢复自动化 | 50% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效 | 灰度批次自动扩缩和长期趋势告警 | | 阶段 4:安全恢复自动化 | 50% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效 | 灰度批次自动扩缩和长期趋势告警 |
| 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 | | 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 |
@@ -515,6 +515,14 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。
可通过 `DOUYU_DEFAULT_SUPPLY_TARGETS_ENABLED=false` 关闭初始化。人工新增过任意 可通过 `DOUYU_DEFAULT_SUPPLY_TARGETS_ENABLED=false` 关闭初始化。人工新增过任意
供给目标后,启动过程不会再注入或覆盖默认值。 供给目标后,启动过程不会再注入或覆盖默认值。
平台状态快照同时聚合最近 7 天趋势:
- 各同步域查询量、成功率和耗时;
- 登录尝试量、成功率和分类分布;
- CK 失效、重登成功/失败事件。
趋势直接读取现有同步运行、登录尝试和账号事件账本,不维护第二份统计事实。
### 阶段 4:安全恢复自动化 ### 阶段 4:安全恢复自动化
- 自动重登小批量开启 - 自动重登小批量开启
@@ -611,3 +619,4 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。
| 2026-11-21 | 建立斗鱼账号平台最终实施蓝图 | | 2026-11-21 | 建立斗鱼账号平台最终实施蓝图 |
| 2026-11-21 | 增加保守默认供给目标、账号平台供给看板和库存/CK/工作队列阈值告警 | | 2026-11-21 | 增加保守默认供给目标、账号平台供给看板和库存/CK/工作队列阈值告警 |
| 2026-11-21 | 增加供给目标管理 UI,支持新增、编辑、启停和调整安全库存 | | 2026-11-21 | 增加供给目标管理 UI,支持新增、编辑、启停和调整安全库存 |
| 2026-11-21 | 完成阶段 3:增加 7 天查询、登录和 CK 生命周期趋势,并补充同步时间索引 |
+78
View File
@@ -7,10 +7,13 @@ from datetime import UTC, datetime, timedelta
from web.backend.models import ( from web.backend.models import (
Account, Account,
AccountCommodityState, AccountCommodityState,
AccountEvent,
AccountLease, AccountLease,
AccountSupplyTarget, AccountSupplyTarget,
AccountSyncRun,
AccountSyncState, AccountSyncState,
AccountWorkItem, AccountWorkItem,
LoginAttempt,
User, User,
) )
from web.backend.routers.douyu import create_supply_target, list_supply_targets from web.backend.routers.douyu import create_supply_target, list_supply_targets
@@ -266,3 +269,78 @@ def test_account_platform_status_aggregates_queue_sync_and_supply(db):
assert { assert {
alert["code"] for alert in status["alerts"] alert["code"] for alert in status["alerts"]
} >= {"supply_shortage", "ck_expiring_24h"} } >= {"supply_shortage", "ck_expiring_24h"}
def test_account_platform_status_builds_seven_day_trends(db):
account = _account("platform-trends", points=400)
db.add(account)
db.flush()
db.add_all(
[
AccountSyncRun(
account_id=account.id,
domain="storehome",
started_at=NOW,
finished_at=NOW,
status="success",
duration_ms=120,
),
AccountSyncRun(
account_id=account.id,
domain="role",
started_at=NOW,
finished_at=NOW,
status="failed",
duration_ms=300,
),
LoginAttempt(
account_id=account.id,
category="manual_login",
trigger_source="manual",
started_at=NOW,
finished_at=NOW,
status="success",
quota_day=NOW.date(),
),
LoginAttempt(
account_id=account.id,
category="relogin_ordinary",
trigger_source="system",
started_at=NOW,
finished_at=NOW,
status="failed",
failure_kind="captcha",
quota_day=NOW.date(),
),
AccountEvent(
account_id=account.id,
event_type="ck_invalid",
category="auth",
occurred_at=NOW,
title="CK 失效",
source="pytest",
actor_type="system",
source_ref=f"pytest-trend:{account.id}",
),
]
)
db.commit()
status = account_platform_status(db, now=NOW)
today = NOW.date().isoformat()
query_today = next(row for row in status["trends"]["query"] if row["date"] == today)
login_today = next(row for row in status["trends"]["login"] if row["date"] == today)
lifecycle_today = next(
row for row in status["trends"]["lifecycle"] if row["date"] == today
)
assert status["trends"]["days"] == 7
assert query_today["total"] == 2
assert query_today["success"] == 1
assert query_today["failed"] == 1
assert query_today["success_rate"] == 50.0
assert "storehome" in query_today["domains"]
assert login_today["success"] == 1
assert login_today["failed"] == 1
assert login_today["success_rate"] == 50.0
assert lifecycle_today["ck_invalid"] == 1
+1 -1
View File
@@ -42,7 +42,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261121_0111" HEAD_REVISION = "20261121_0112"
# 迁移新增、但不声明在模型里的复合查询索引。 # 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = ( EXTRA_INDEXES = (
@@ -445,6 +445,10 @@
"20261121_0111_account_platform_foundation.py": { "20261121_0111_account_platform_foundation.py": {
"revision": "20261121_0111", "revision": "20261121_0111",
"sha256": "0cf919b766769c84d357271acbf32bd968b2b6756b04cc059b26fd9fb5d41c10" "sha256": "0cf919b766769c84d357271acbf32bd968b2b6756b04cc059b26fd9fb5d41c10"
},
"20261121_0112_account_sync_trend_index.py": {
"revision": "20261121_0112",
"sha256": "9c1ebee5fba13efc413a4dcca258fa66dfbca5b41c1bd9b89ab794c394af3fee"
} }
} }
} }
@@ -0,0 +1,51 @@
"""add started-at index for account sync trend aggregation
The platform status panel aggregates the last seven days across all sync domains.
The existing indexes start with domain/status, so they cannot serve that range
efficiently when no domain filter is present.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261121_0112"
down_revision: str | None = "20261121_0111"
branch_labels: str | None = None
depends_on: str | None = None
INDEX_NAME = "ix_account_sync_runs_started_domain_status"
def _index_exists() -> bool:
return bool(
op.get_bind()
.execute(
sa.text(
"""
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'account_sync_runs'
AND index_name = :index_name
"""
),
{"index_name": INDEX_NAME},
)
.scalar()
)
def upgrade() -> None:
if not _index_exists():
op.create_index(
INDEX_NAME,
"account_sync_runs",
["started_at", "domain", "status"],
)
def downgrade() -> None:
if _index_exists():
op.drop_index(INDEX_NAME, table_name="account_sync_runs")
+6
View File
@@ -2449,6 +2449,12 @@ class AccountSyncRun(Base):
"status", "status",
"started_at", "started_at",
), ),
Index(
"ix_account_sync_runs_started_domain_status",
"started_at",
"domain",
"status",
),
) )
+176 -1
View File
@@ -2,7 +2,7 @@
from __future__ import annotations from __future__ import annotations
from datetime import UTC, datetime, timedelta from datetime import UTC, date, datetime, timedelta
from typing import Any from typing import Any
from sqlalchemy import case, func from sqlalchemy import case, func
@@ -11,12 +11,186 @@ from sqlalchemy.orm import Session
from ..models import ( from ..models import (
Account, Account,
AccountCommodityState, AccountCommodityState,
AccountEvent,
AccountSyncRun,
AccountSyncState, AccountSyncState,
AccountWorkItem, AccountWorkItem,
LoginAttempt,
) )
from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED
from .account_supply_service import evaluate_supply_targets from .account_supply_service import evaluate_supply_targets
TREND_DAYS = 7
def _day_key(value: Any) -> str:
if isinstance(value, datetime):
return value.date().isoformat()
if isinstance(value, date):
return value.isoformat()
return str(value or "")[:10]
def _trend_days(moment: datetime) -> list[str]:
today = moment.date()
return [
(today - timedelta(days=offset)).isoformat()
for offset in range(TREND_DAYS - 1, -1, -1)
]
def _rate(success: int, finished: int) -> float:
if finished <= 0:
return 0.0
return round(success * 100 / finished, 1)
def _platform_trends(db: Session, moment: datetime) -> dict[str, Any]:
days = _trend_days(moment)
start_at = datetime.combine(
date.fromisoformat(days[0]),
datetime.min.time(),
tzinfo=UTC,
)
query_days = {
day: {
"date": day,
"total": 0,
"success": 0,
"failed": 0,
"running": 0,
"success_rate": 0.0,
"duration_ms": 0,
"domains": {},
}
for day in days
}
sync_day = func.date(AccountSyncRun.started_at)
sync_rows = (
db.query(
sync_day,
AccountSyncRun.domain,
AccountSyncRun.status,
func.count(AccountSyncRun.id),
func.coalesce(func.sum(AccountSyncRun.duration_ms), 0),
)
.filter(AccountSyncRun.started_at >= start_at)
.group_by(sync_day, AccountSyncRun.domain, AccountSyncRun.status)
.all()
)
for raw_day, domain, status, count, duration_ms in sync_rows:
day = _day_key(raw_day)
bucket = query_days.get(day)
if bucket is None:
continue
count_value = int(count or 0)
domain_bucket = bucket["domains"].setdefault(
str(domain),
{"total": 0, "success": 0, "failed": 0, "running": 0},
)
bucket["total"] += count_value
bucket["duration_ms"] += int(duration_ms or 0)
domain_bucket["total"] += count_value
if status in {"success", "failed", "running"}:
bucket[str(status)] += count_value
domain_bucket[str(status)] += count_value
for bucket in query_days.values():
finished = bucket["success"] + bucket["failed"]
bucket["success_rate"] = _rate(bucket["success"], finished)
for domain_bucket in bucket["domains"].values():
domain_finished = domain_bucket["success"] + domain_bucket["failed"]
domain_bucket["success_rate"] = _rate(
domain_bucket["success"],
domain_finished,
)
login_days = {
day: {
"date": day,
"total": 0,
"success": 0,
"failed": 0,
"pending": 0,
"success_rate": 0.0,
"categories": {},
}
for day in days
}
login_day = LoginAttempt.quota_day
login_rows = (
db.query(
login_day,
LoginAttempt.category,
LoginAttempt.status,
func.count(LoginAttempt.id),
)
.filter(LoginAttempt.quota_day >= date.fromisoformat(days[0]))
.group_by(login_day, LoginAttempt.category, LoginAttempt.status)
.all()
)
for raw_day, category, status, count in login_rows:
day = _day_key(raw_day)
bucket = login_days.get(day)
if bucket is None:
continue
count_value = int(count or 0)
category_bucket = bucket["categories"].setdefault(
str(category),
{"total": 0, "success": 0, "failed": 0, "pending": 0},
)
bucket["total"] += count_value
category_bucket["total"] += count_value
if status in {"success", "failed", "pending"}:
bucket[str(status)] += count_value
category_bucket[str(status)] += count_value
for bucket in login_days.values():
finished = bucket["success"] + bucket["failed"]
bucket["success_rate"] = _rate(bucket["success"], finished)
for category_bucket in bucket["categories"].values():
category_finished = category_bucket["success"] + category_bucket["failed"]
category_bucket["success_rate"] = _rate(
category_bucket["success"],
category_finished,
)
lifecycle_days = {
day: {
"date": day,
"ck_invalid": 0,
"relogin_ok": 0,
"relogin_failed": 0,
}
for day in days
}
lifecycle_events = ("ck_invalid", "relogin_ok", "relogin_failed")
event_day = func.date(AccountEvent.occurred_at)
event_rows = (
db.query(
event_day,
AccountEvent.event_type,
func.count(AccountEvent.id),
)
.filter(
AccountEvent.occurred_at >= start_at,
AccountEvent.event_type.in_(lifecycle_events),
)
.group_by(event_day, AccountEvent.event_type)
.all()
)
for raw_day, event_type, count in event_rows:
bucket = lifecycle_days.get(_day_key(raw_day))
if bucket is not None and event_type in bucket:
bucket[str(event_type)] = int(count or 0)
return {
"days": TREND_DAYS,
"from": days[0],
"to": days[-1],
"query": list(query_days.values()),
"login": list(login_days.values()),
"lifecycle": list(lifecycle_days.values()),
}
def account_platform_status( def account_platform_status(
db: Session, db: Session,
@@ -251,6 +425,7 @@ def account_platform_status(
}, },
"supply_targets": supply_targets, "supply_targets": supply_targets,
"alerts": alerts, "alerts": alerts,
"trends": _platform_trends(db, moment),
} }
+50
View File
@@ -783,6 +783,55 @@ export interface DouyuPlatformAlert {
count?: number; count?: number;
} }
export interface DouyuQueryTrendDay {
date: string;
total: number;
success: number;
failed: number;
running: number;
success_rate: number;
duration_ms: number;
domains: Record<string, {
total: number;
success: number;
failed: number;
running: number;
success_rate: number;
}>;
}
export interface DouyuLoginTrendDay {
date: string;
total: number;
success: number;
failed: number;
pending: number;
success_rate: number;
categories: Record<string, {
total: number;
success: number;
failed: number;
pending: number;
success_rate: number;
}>;
}
export interface DouyuLifecycleTrendDay {
date: string;
ck_invalid: number;
relogin_ok: number;
relogin_failed: number;
}
export interface DouyuPlatformTrends {
days: number;
from: string;
to: string;
query: DouyuQueryTrendDay[];
login: DouyuLoginTrendDay[];
lifecycle: DouyuLifecycleTrendDay[];
}
export interface DouyuAccountPlatformStatus { export interface DouyuAccountPlatformStatus {
checked_at: string; checked_at: string;
accounts: { accounts: {
@@ -805,6 +854,7 @@ export interface DouyuAccountPlatformStatus {
}; };
supply_targets: DouyuSupplyTargetStatus[]; supply_targets: DouyuSupplyTargetStatus[];
alerts: DouyuPlatformAlert[]; alerts: DouyuPlatformAlert[];
trends: DouyuPlatformTrends;
} }
export interface DouyuConfig { export interface DouyuConfig {
@@ -9,6 +9,7 @@ import {
Input, Input,
InputNumber, InputNumber,
Modal, Modal,
Progress,
Row, Row,
Select, Select,
Space, Space,
@@ -53,6 +54,16 @@ interface TargetFormValues {
enabled: boolean; enabled: boolean;
} }
interface TrendRow {
date: string;
queryTotal: number;
queryRate: number;
loginTotal: number;
loginRate: number;
ckInvalid: number;
reloginOk: number;
}
export default function AccountPlatformStatusPanel() { export default function AccountPlatformStatusPanel() {
const { can } = usePermissions(); const { can } = usePermissions();
const canManage = can('douyu:config'); const canManage = can('douyu:config');
@@ -100,6 +111,50 @@ export default function AccountPlatformStatusPanel() {
), ),
[status?.work_queue], [status?.work_queue],
); );
const trendRows = useMemo(() => {
if (!status) return [];
const rows = new Map<string, TrendRow>();
for (const item of status.trends.query) {
rows.set(item.date, {
date: item.date,
queryTotal: item.total,
queryRate: item.success_rate,
loginTotal: 0,
loginRate: 0,
ckInvalid: 0,
reloginOk: 0,
});
}
for (const item of status.trends.login) {
const row = rows.get(item.date) || {
date: item.date,
queryTotal: 0,
queryRate: 0,
loginTotal: 0,
loginRate: 0,
ckInvalid: 0,
reloginOk: 0,
};
row.loginTotal = item.total;
row.loginRate = item.success_rate;
rows.set(item.date, row);
}
for (const item of status.trends.lifecycle) {
const row = rows.get(item.date) || {
date: item.date,
queryTotal: 0,
queryRate: 0,
loginTotal: 0,
loginRate: 0,
ckInvalid: 0,
reloginOk: 0,
};
row.ckInvalid = item.ck_invalid;
row.reloginOk = item.relogin_ok;
rows.set(item.date, row);
}
return [...rows.values()].sort((left, right) => left.date.localeCompare(right.date));
}, [status]);
const openCreate = () => { const openCreate = () => {
setEditing(null); setEditing(null);
@@ -232,6 +287,46 @@ export default function AccountPlatformStatusPanel() {
}] : []), }] : []),
]; ];
const trendColumns: TableProps<TrendRow>['columns'] = [
{
title: '日期',
dataIndex: 'date',
width: 100,
render: (value: string) => value.slice(5),
},
{
title: '查询成功率',
width: 140,
render: (_, row) => (
<Progress
percent={row.queryRate}
size="small"
status={row.queryRate < 70 ? 'exception' : 'normal'}
/>
),
},
{ title: '查询量', dataIndex: 'queryTotal', width: 80 },
{
title: '登录成功率',
width: 140,
render: (_, row) => (
<Progress
percent={row.loginRate}
size="small"
status={row.loginRate < 50 ? 'exception' : 'normal'}
/>
),
},
{ title: '登录量', dataIndex: 'loginTotal', width: 80 },
{
title: 'CK 失效',
dataIndex: 'ckInvalid',
width: 90,
render: (value: number) => value > 0 ? <Text type="danger">{value}</Text> : 0,
},
{ title: '重登成功', dataIndex: 'reloginOk', width: 90 },
];
return ( return (
<> <>
<Card <Card
@@ -287,6 +382,18 @@ export default function AccountPlatformStatusPanel() {
), ),
}} }}
/> />
<Space orientation="vertical" size={8} style={{ width: '100%', marginTop: 20 }}>
<Text strong>{`近 ${status?.trends.days || 7} 天趋势`}</Text>
<Table
rowKey="date"
size="small"
columns={trendColumns}
dataSource={trendRows}
pagination={false}
scroll={{ x: 750 }}
/>
</Space>
</Card> </Card>
<Modal <Modal