From 3e618c1a88f29a2dbf81199f331a3d666102f6e0 Mon Sep 17 00:00:00 2001 From: yml Date: Sat, 21 Nov 2026 23:00:32 +0800 Subject: [PATCH] =?UTF-8?q?feat(douyu):=20=E5=A2=9E=E5=8A=A0=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=E5=B9=B3=E5=8F=B0=E4=B8=83=E5=A4=A9=E8=B6=8B=E5=8A=BF?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/斗鱼/斗鱼账号平台最终实施设计.md | 13 +- tests/test_account_supply_service.py | 78 ++++++++ tests/test_migrations.py | 2 +- .../migrations/migration_manifest.json | 4 + .../20261121_0112_account_sync_trend_index.py | 51 +++++ web/backend/models.py | 6 + web/backend/services/account_observability.py | 177 +++++++++++++++++- web/frontend/src/api/types.ts | 50 +++++ .../AccountPlatformStatusPanel.tsx | 107 +++++++++++ 9 files changed, 484 insertions(+), 4 deletions(-) create mode 100644 web/backend/migrations/versions/20261121_0112_account_sync_trend_index.py diff --git a/docs/斗鱼/斗鱼账号平台最终实施设计.md b/docs/斗鱼/斗鱼账号平台最终实施设计.md index 5cc1645..00c6cb5 100644 --- a/docs/斗鱼/斗鱼账号平台最终实施设计.md +++ b/docs/斗鱼/斗鱼账号平台最终实施设计.md @@ -11,14 +11,14 @@ 更新时间:2026-11-21 -整体完成度约 **73%**。 +整体完成度约 **75%**。 | 阶段 | 完成度 | 已落地 | 主要缺口 | | --- | ---: | --- | --- | | 阶段 0:止乱 | 100% | 登录批次上限、任务槽隔离、连接及时释放 | 无 | | 阶段 1:同步基础 | 95% | 新表、迁移、storehome 网关、全量快照、同步运行记录、内部工作队列执行器、session/points/gold/elite/esports/role/exchange 同步状态 | 少量旁路查询仍需继续收口 | | 阶段 2:账号与 CK | 90% | CK 周期字段、invalid 分流、登录流水、日/小时/分类/单账号预算、login_relogin 队列真实消费与退避回队、预算配置 API | 更精细的动态预算计算 | -| 阶段 3:商品与供给 | 95% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警 | 趋势统计 | +| 阶段 3:商品与供给 | 100% | 全量商品能力、限时窗口、原始响应、供给库存和目标缺口、定向补货入队、供给目标配置 API 与管理 UI、默认目标、统一平台状态快照、供给看板、阈值告警、7 天查询/登录/CK 趋势 | 无 | | 阶段 4:安全恢复自动化 | 50% | account_sync_queue 与供给补货作业默认小批量开启,自动重登消费工作队列,硬失败率熔断,登录预算在线配置且熔断阈值实时生效 | 灰度批次自动扩缩和长期趋势告警 | | 阶段 5:清理旧逻辑 | 20% | 履约补货不再直接发官网请求 | 旧资料刷新、旁路查询和旧候选兜底仍需逐步下线 | @@ -515,6 +515,14 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。 可通过 `DOUYU_DEFAULT_SUPPLY_TARGETS_ENABLED=false` 关闭初始化。人工新增过任意 供给目标后,启动过程不会再注入或覆盖默认值。 +平台状态快照同时聚合最近 7 天趋势: + +- 各同步域查询量、成功率和耗时; +- 登录尝试量、成功率和分类分布; +- CK 失效、重登成功/失败事件。 + +趋势直接读取现有同步运行、登录尝试和账号事件账本,不维护第二份统计事实。 + ### 阶段 4:安全恢复自动化 - 自动重登小批量开启 @@ -611,3 +619,4 @@ Runner 启动前必须有预算 token;没有 token 时留在工作队列。 | 2026-11-21 | 建立斗鱼账号平台最终实施蓝图 | | 2026-11-21 | 增加保守默认供给目标、账号平台供给看板和库存/CK/工作队列阈值告警 | | 2026-11-21 | 增加供给目标管理 UI,支持新增、编辑、启停和调整安全库存 | +| 2026-11-21 | 完成阶段 3:增加 7 天查询、登录和 CK 生命周期趋势,并补充同步时间索引 | diff --git a/tests/test_account_supply_service.py b/tests/test_account_supply_service.py index c5a14ca..a5d7ef0 100644 --- a/tests/test_account_supply_service.py +++ b/tests/test_account_supply_service.py @@ -7,10 +7,13 @@ from datetime import UTC, datetime, timedelta from web.backend.models import ( Account, AccountCommodityState, + AccountEvent, AccountLease, AccountSupplyTarget, + AccountSyncRun, AccountSyncState, AccountWorkItem, + LoginAttempt, User, ) from web.backend.routers.douyu import create_supply_target, list_supply_targets @@ -266,3 +269,78 @@ def test_account_platform_status_aggregates_queue_sync_and_supply(db): assert { alert["code"] for alert in status["alerts"] } >= {"supply_shortage", "ck_expiring_24h"} + + +def test_account_platform_status_builds_seven_day_trends(db): + account = _account("platform-trends", points=400) + db.add(account) + db.flush() + db.add_all( + [ + AccountSyncRun( + account_id=account.id, + domain="storehome", + started_at=NOW, + finished_at=NOW, + status="success", + duration_ms=120, + ), + AccountSyncRun( + account_id=account.id, + domain="role", + started_at=NOW, + finished_at=NOW, + status="failed", + duration_ms=300, + ), + LoginAttempt( + account_id=account.id, + category="manual_login", + trigger_source="manual", + started_at=NOW, + finished_at=NOW, + status="success", + quota_day=NOW.date(), + ), + LoginAttempt( + account_id=account.id, + category="relogin_ordinary", + trigger_source="system", + started_at=NOW, + finished_at=NOW, + status="failed", + failure_kind="captcha", + quota_day=NOW.date(), + ), + AccountEvent( + account_id=account.id, + event_type="ck_invalid", + category="auth", + occurred_at=NOW, + title="CK 失效", + source="pytest", + actor_type="system", + source_ref=f"pytest-trend:{account.id}", + ), + ] + ) + db.commit() + + status = account_platform_status(db, now=NOW) + today = NOW.date().isoformat() + query_today = next(row for row in status["trends"]["query"] if row["date"] == today) + login_today = next(row for row in status["trends"]["login"] if row["date"] == today) + lifecycle_today = next( + row for row in status["trends"]["lifecycle"] if row["date"] == today + ) + + assert status["trends"]["days"] == 7 + assert query_today["total"] == 2 + assert query_today["success"] == 1 + assert query_today["failed"] == 1 + assert query_today["success_rate"] == 50.0 + assert "storehome" in query_today["domains"] + assert login_today["success"] == 1 + assert login_today["failed"] == 1 + assert login_today["success_rate"] == 50.0 + assert lifecycle_today["ck_invalid"] == 1 diff --git a/tests/test_migrations.py b/tests/test_migrations.py index f6d3ddf..1edaa6b 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -42,7 +42,7 @@ TESTS_DIR = Path(__file__).resolve().parent PROJECT_ROOT = TESTS_DIR.parent VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" -HEAD_REVISION = "20261121_0111" +HEAD_REVISION = "20261121_0112" # 迁移新增、但不声明在模型里的复合查询索引。 EXTRA_INDEXES = ( diff --git a/web/backend/migrations/migration_manifest.json b/web/backend/migrations/migration_manifest.json index c1bacad..2a9236f 100644 --- a/web/backend/migrations/migration_manifest.json +++ b/web/backend/migrations/migration_manifest.json @@ -445,6 +445,10 @@ "20261121_0111_account_platform_foundation.py": { "revision": "20261121_0111", "sha256": "0cf919b766769c84d357271acbf32bd968b2b6756b04cc059b26fd9fb5d41c10" + }, + "20261121_0112_account_sync_trend_index.py": { + "revision": "20261121_0112", + "sha256": "9c1ebee5fba13efc413a4dcca258fa66dfbca5b41c1bd9b89ab794c394af3fee" } } } diff --git a/web/backend/migrations/versions/20261121_0112_account_sync_trend_index.py b/web/backend/migrations/versions/20261121_0112_account_sync_trend_index.py new file mode 100644 index 0000000..eb803e1 --- /dev/null +++ b/web/backend/migrations/versions/20261121_0112_account_sync_trend_index.py @@ -0,0 +1,51 @@ +"""add started-at index for account sync trend aggregation + +The platform status panel aggregates the last seven days across all sync domains. +The existing indexes start with domain/status, so they cannot serve that range +efficiently when no domain filter is present. +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261121_0112" +down_revision: str | None = "20261121_0111" +branch_labels: str | None = None +depends_on: str | None = None + +INDEX_NAME = "ix_account_sync_runs_started_domain_status" + + +def _index_exists() -> bool: + return bool( + op.get_bind() + .execute( + sa.text( + """ + SELECT COUNT(*) + FROM information_schema.statistics + WHERE table_schema = DATABASE() + AND table_name = 'account_sync_runs' + AND index_name = :index_name + """ + ), + {"index_name": INDEX_NAME}, + ) + .scalar() + ) + + +def upgrade() -> None: + if not _index_exists(): + op.create_index( + INDEX_NAME, + "account_sync_runs", + ["started_at", "domain", "status"], + ) + + +def downgrade() -> None: + if _index_exists(): + op.drop_index(INDEX_NAME, table_name="account_sync_runs") diff --git a/web/backend/models.py b/web/backend/models.py index ab56c47..5382bb3 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -2449,6 +2449,12 @@ class AccountSyncRun(Base): "status", "started_at", ), + Index( + "ix_account_sync_runs_started_domain_status", + "started_at", + "domain", + "status", + ), ) diff --git a/web/backend/services/account_observability.py b/web/backend/services/account_observability.py index f509d61..ccbadc4 100644 --- a/web/backend/services/account_observability.py +++ b/web/backend/services/account_observability.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import UTC, datetime, timedelta +from datetime import UTC, date, datetime, timedelta from typing import Any from sqlalchemy import case, func @@ -11,12 +11,186 @@ from sqlalchemy.orm import Session from ..models import ( Account, AccountCommodityState, + AccountEvent, + AccountSyncRun, AccountSyncState, AccountWorkItem, + LoginAttempt, ) from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED from .account_supply_service import evaluate_supply_targets +TREND_DAYS = 7 + + +def _day_key(value: Any) -> str: + if isinstance(value, datetime): + return value.date().isoformat() + if isinstance(value, date): + return value.isoformat() + return str(value or "")[:10] + + +def _trend_days(moment: datetime) -> list[str]: + today = moment.date() + return [ + (today - timedelta(days=offset)).isoformat() + for offset in range(TREND_DAYS - 1, -1, -1) + ] + + +def _rate(success: int, finished: int) -> float: + if finished <= 0: + return 0.0 + return round(success * 100 / finished, 1) + + +def _platform_trends(db: Session, moment: datetime) -> dict[str, Any]: + days = _trend_days(moment) + start_at = datetime.combine( + date.fromisoformat(days[0]), + datetime.min.time(), + tzinfo=UTC, + ) + query_days = { + day: { + "date": day, + "total": 0, + "success": 0, + "failed": 0, + "running": 0, + "success_rate": 0.0, + "duration_ms": 0, + "domains": {}, + } + for day in days + } + sync_day = func.date(AccountSyncRun.started_at) + sync_rows = ( + db.query( + sync_day, + AccountSyncRun.domain, + AccountSyncRun.status, + func.count(AccountSyncRun.id), + func.coalesce(func.sum(AccountSyncRun.duration_ms), 0), + ) + .filter(AccountSyncRun.started_at >= start_at) + .group_by(sync_day, AccountSyncRun.domain, AccountSyncRun.status) + .all() + ) + for raw_day, domain, status, count, duration_ms in sync_rows: + day = _day_key(raw_day) + bucket = query_days.get(day) + if bucket is None: + continue + count_value = int(count or 0) + domain_bucket = bucket["domains"].setdefault( + str(domain), + {"total": 0, "success": 0, "failed": 0, "running": 0}, + ) + bucket["total"] += count_value + bucket["duration_ms"] += int(duration_ms or 0) + domain_bucket["total"] += count_value + if status in {"success", "failed", "running"}: + bucket[str(status)] += count_value + domain_bucket[str(status)] += count_value + for bucket in query_days.values(): + finished = bucket["success"] + bucket["failed"] + bucket["success_rate"] = _rate(bucket["success"], finished) + for domain_bucket in bucket["domains"].values(): + domain_finished = domain_bucket["success"] + domain_bucket["failed"] + domain_bucket["success_rate"] = _rate( + domain_bucket["success"], + domain_finished, + ) + + login_days = { + day: { + "date": day, + "total": 0, + "success": 0, + "failed": 0, + "pending": 0, + "success_rate": 0.0, + "categories": {}, + } + for day in days + } + login_day = LoginAttempt.quota_day + login_rows = ( + db.query( + login_day, + LoginAttempt.category, + LoginAttempt.status, + func.count(LoginAttempt.id), + ) + .filter(LoginAttempt.quota_day >= date.fromisoformat(days[0])) + .group_by(login_day, LoginAttempt.category, LoginAttempt.status) + .all() + ) + for raw_day, category, status, count in login_rows: + day = _day_key(raw_day) + bucket = login_days.get(day) + if bucket is None: + continue + count_value = int(count or 0) + category_bucket = bucket["categories"].setdefault( + str(category), + {"total": 0, "success": 0, "failed": 0, "pending": 0}, + ) + bucket["total"] += count_value + category_bucket["total"] += count_value + if status in {"success", "failed", "pending"}: + bucket[str(status)] += count_value + category_bucket[str(status)] += count_value + for bucket in login_days.values(): + finished = bucket["success"] + bucket["failed"] + bucket["success_rate"] = _rate(bucket["success"], finished) + for category_bucket in bucket["categories"].values(): + category_finished = category_bucket["success"] + category_bucket["failed"] + category_bucket["success_rate"] = _rate( + category_bucket["success"], + category_finished, + ) + + lifecycle_days = { + day: { + "date": day, + "ck_invalid": 0, + "relogin_ok": 0, + "relogin_failed": 0, + } + for day in days + } + lifecycle_events = ("ck_invalid", "relogin_ok", "relogin_failed") + event_day = func.date(AccountEvent.occurred_at) + event_rows = ( + db.query( + event_day, + AccountEvent.event_type, + func.count(AccountEvent.id), + ) + .filter( + AccountEvent.occurred_at >= start_at, + AccountEvent.event_type.in_(lifecycle_events), + ) + .group_by(event_day, AccountEvent.event_type) + .all() + ) + for raw_day, event_type, count in event_rows: + bucket = lifecycle_days.get(_day_key(raw_day)) + if bucket is not None and event_type in bucket: + bucket[str(event_type)] = int(count or 0) + + return { + "days": TREND_DAYS, + "from": days[0], + "to": days[-1], + "query": list(query_days.values()), + "login": list(login_days.values()), + "lifecycle": list(lifecycle_days.values()), + } + def account_platform_status( db: Session, @@ -251,6 +425,7 @@ def account_platform_status( }, "supply_targets": supply_targets, "alerts": alerts, + "trends": _platform_trends(db, moment), } diff --git a/web/frontend/src/api/types.ts b/web/frontend/src/api/types.ts index 8a98493..9a9585e 100644 --- a/web/frontend/src/api/types.ts +++ b/web/frontend/src/api/types.ts @@ -783,6 +783,55 @@ export interface DouyuPlatformAlert { count?: number; } +export interface DouyuQueryTrendDay { + date: string; + total: number; + success: number; + failed: number; + running: number; + success_rate: number; + duration_ms: number; + domains: Record; +} + +export interface DouyuLoginTrendDay { + date: string; + total: number; + success: number; + failed: number; + pending: number; + success_rate: number; + categories: Record; +} + +export interface DouyuLifecycleTrendDay { + date: string; + ck_invalid: number; + relogin_ok: number; + relogin_failed: number; +} + +export interface DouyuPlatformTrends { + days: number; + from: string; + to: string; + query: DouyuQueryTrendDay[]; + login: DouyuLoginTrendDay[]; + lifecycle: DouyuLifecycleTrendDay[]; +} + export interface DouyuAccountPlatformStatus { checked_at: string; accounts: { @@ -805,6 +854,7 @@ export interface DouyuAccountPlatformStatus { }; supply_targets: DouyuSupplyTargetStatus[]; alerts: DouyuPlatformAlert[]; + trends: DouyuPlatformTrends; } export interface DouyuConfig { diff --git a/web/frontend/src/pages/douyu-tasks/AccountPlatformStatusPanel.tsx b/web/frontend/src/pages/douyu-tasks/AccountPlatformStatusPanel.tsx index 3e5357b..d931561 100644 --- a/web/frontend/src/pages/douyu-tasks/AccountPlatformStatusPanel.tsx +++ b/web/frontend/src/pages/douyu-tasks/AccountPlatformStatusPanel.tsx @@ -9,6 +9,7 @@ import { Input, InputNumber, Modal, + Progress, Row, Select, Space, @@ -53,6 +54,16 @@ interface TargetFormValues { enabled: boolean; } +interface TrendRow { + date: string; + queryTotal: number; + queryRate: number; + loginTotal: number; + loginRate: number; + ckInvalid: number; + reloginOk: number; +} + export default function AccountPlatformStatusPanel() { const { can } = usePermissions(); const canManage = can('douyu:config'); @@ -100,6 +111,50 @@ export default function AccountPlatformStatusPanel() { ), [status?.work_queue], ); + const trendRows = useMemo(() => { + if (!status) return []; + const rows = new Map(); + for (const item of status.trends.query) { + rows.set(item.date, { + date: item.date, + queryTotal: item.total, + queryRate: item.success_rate, + loginTotal: 0, + loginRate: 0, + ckInvalid: 0, + reloginOk: 0, + }); + } + for (const item of status.trends.login) { + const row = rows.get(item.date) || { + date: item.date, + queryTotal: 0, + queryRate: 0, + loginTotal: 0, + loginRate: 0, + ckInvalid: 0, + reloginOk: 0, + }; + row.loginTotal = item.total; + row.loginRate = item.success_rate; + rows.set(item.date, row); + } + for (const item of status.trends.lifecycle) { + const row = rows.get(item.date) || { + date: item.date, + queryTotal: 0, + queryRate: 0, + loginTotal: 0, + loginRate: 0, + ckInvalid: 0, + reloginOk: 0, + }; + row.ckInvalid = item.ck_invalid; + row.reloginOk = item.relogin_ok; + rows.set(item.date, row); + } + return [...rows.values()].sort((left, right) => left.date.localeCompare(right.date)); + }, [status]); const openCreate = () => { setEditing(null); @@ -232,6 +287,46 @@ export default function AccountPlatformStatusPanel() { }] : []), ]; + const trendColumns: TableProps['columns'] = [ + { + title: '日期', + dataIndex: 'date', + width: 100, + render: (value: string) => value.slice(5), + }, + { + title: '查询成功率', + width: 140, + render: (_, row) => ( + + ), + }, + { title: '查询量', dataIndex: 'queryTotal', width: 80 }, + { + title: '登录成功率', + width: 140, + render: (_, row) => ( + + ), + }, + { title: '登录量', dataIndex: 'loginTotal', width: 80 }, + { + title: 'CK 失效', + dataIndex: 'ckInvalid', + width: 90, + render: (value: number) => value > 0 ? {value} : 0, + }, + { title: '重登成功', dataIndex: 'reloginOk', width: 90 }, + ]; + return ( <> + + + {`近 ${status?.trends.days || 7} 天趋势`} + +