Files
live-hub-py/web/backend/services/account_observability.py
T

489 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""斗鱼账号平台运行状态与覆盖率快照。"""
from __future__ import annotations
from datetime import UTC, date, datetime, timedelta
from typing import Any
from sqlalchemy import case, func
from sqlalchemy.orm import Session
from ..models import (
Account,
AccountCommodityState,
AccountEvent,
AccountSyncRun,
AccountSyncState,
AccountWorkItem,
LoginAttempt,
)
from .account_capability_service import AVAILABILITY_AVAILABLE, AVAILABILITY_LIMITED
from .account_supply_service import evaluate_supply_targets
TREND_DAYS = 7
def _day_key(value: Any) -> str:
if isinstance(value, datetime):
return value.date().isoformat()
if isinstance(value, date):
return value.isoformat()
return str(value or "")[:10]
def _trend_days(moment: datetime) -> list[str]:
today = moment.date()
return [
(today - timedelta(days=offset)).isoformat()
for offset in range(TREND_DAYS - 1, -1, -1)
]
def _rate(success: int, finished: int) -> float:
if finished <= 0:
return 0.0
return round(success * 100 / finished, 1)
def _platform_trends(db: Session, moment: datetime) -> dict[str, Any]:
days = _trend_days(moment)
start_at = datetime.combine(
date.fromisoformat(days[0]),
datetime.min.time(),
tzinfo=UTC,
)
query_days = {
day: {
"date": day,
"total": 0,
"success": 0,
"failed": 0,
"running": 0,
"success_rate": 0.0,
"duration_ms": 0,
"domains": {},
}
for day in days
}
sync_day = func.date(AccountSyncRun.started_at)
sync_rows = (
db.query(
sync_day,
AccountSyncRun.domain,
AccountSyncRun.status,
func.count(AccountSyncRun.id),
func.coalesce(func.sum(AccountSyncRun.duration_ms), 0),
)
.filter(AccountSyncRun.started_at >= start_at)
.group_by(sync_day, AccountSyncRun.domain, AccountSyncRun.status)
.all()
)
for raw_day, domain, status, count, duration_ms in sync_rows:
day = _day_key(raw_day)
bucket = query_days.get(day)
if bucket is None:
continue
count_value = int(count or 0)
domain_bucket = bucket["domains"].setdefault(
str(domain),
{"total": 0, "success": 0, "failed": 0, "running": 0},
)
bucket["total"] += count_value
bucket["duration_ms"] += int(duration_ms or 0)
domain_bucket["total"] += count_value
if status in {"success", "failed", "running"}:
bucket[str(status)] += count_value
domain_bucket[str(status)] += count_value
for bucket in query_days.values():
finished = bucket["success"] + bucket["failed"]
bucket["success_rate"] = _rate(bucket["success"], finished)
for domain_bucket in bucket["domains"].values():
domain_finished = domain_bucket["success"] + domain_bucket["failed"]
domain_bucket["success_rate"] = _rate(
domain_bucket["success"],
domain_finished,
)
login_days = {
day: {
"date": day,
"total": 0,
"success": 0,
"failed": 0,
"pending": 0,
"success_rate": 0.0,
"categories": {},
}
for day in days
}
login_day = LoginAttempt.quota_day
login_rows = (
db.query(
login_day,
LoginAttempt.category,
LoginAttempt.status,
func.count(LoginAttempt.id),
)
.filter(LoginAttempt.quota_day >= date.fromisoformat(days[0]))
.group_by(login_day, LoginAttempt.category, LoginAttempt.status)
.all()
)
for raw_day, category, status, count in login_rows:
day = _day_key(raw_day)
bucket = login_days.get(day)
if bucket is None:
continue
count_value = int(count or 0)
category_bucket = bucket["categories"].setdefault(
str(category),
{"total": 0, "success": 0, "failed": 0, "pending": 0},
)
bucket["total"] += count_value
category_bucket["total"] += count_value
if status in {"success", "failed", "pending"}:
bucket[str(status)] += count_value
category_bucket[str(status)] += count_value
for bucket in login_days.values():
finished = bucket["success"] + bucket["failed"]
bucket["success_rate"] = _rate(bucket["success"], finished)
for category_bucket in bucket["categories"].values():
category_finished = category_bucket["success"] + category_bucket["failed"]
category_bucket["success_rate"] = _rate(
category_bucket["success"],
category_finished,
)
lifecycle_days = {
day: {
"date": day,
"ck_invalid": 0,
"relogin_ok": 0,
"relogin_failed": 0,
}
for day in days
}
lifecycle_events = ("ck_invalid", "relogin_ok", "relogin_failed")
event_day = func.date(AccountEvent.occurred_at)
event_rows = (
db.query(
event_day,
AccountEvent.event_type,
func.count(AccountEvent.id),
)
.filter(
AccountEvent.occurred_at >= start_at,
AccountEvent.event_type.in_(lifecycle_events),
)
.group_by(event_day, AccountEvent.event_type)
.all()
)
for raw_day, event_type, count in event_rows:
bucket = lifecycle_days.get(_day_key(raw_day))
if bucket is not None and event_type in bucket:
bucket[str(event_type)] = int(count or 0)
return {
"days": TREND_DAYS,
"from": days[0],
"to": days[-1],
"query": list(query_days.values()),
"login": list(login_days.values()),
"lifecycle": list(lifecycle_days.values()),
}
def account_platform_status(
db: Session,
*,
now: datetime | None = None,
) -> dict[str, Any]:
moment = now or datetime.now(UTC)
in_24h = moment + timedelta(hours=24)
in_72h = moment + timedelta(hours=72)
account_counts = dict(
db.query(Account.ck_check_status, func.count(Account.id))
.group_by(Account.ck_check_status)
.all()
)
total_accounts = int(db.query(func.count(Account.id)).scalar() or 0)
cookie_accounts = int(
db.query(func.count(Account.id))
.filter(Account.cookie.isnot(None), Account.cookie != "")
.scalar()
or 0
)
expiring_24h = int(
db.query(func.count(Account.id))
.filter(
Account.ck_check_status == "valid",
Account.ck_expected_expires_at.isnot(None),
Account.ck_expected_expires_at > moment,
Account.ck_expected_expires_at <= in_24h,
)
.scalar()
or 0
)
expiring_72h = int(
db.query(func.count(Account.id))
.filter(
Account.ck_check_status == "valid",
Account.ck_expected_expires_at.isnot(None),
Account.ck_expected_expires_at > moment,
Account.ck_expected_expires_at <= in_72h,
)
.scalar()
or 0
)
predicted_expired = int(
db.query(func.count(Account.id))
.filter(
Account.ck_check_status == "valid",
Account.ck_expected_expires_at.isnot(None),
Account.ck_expected_expires_at <= moment,
)
.scalar()
or 0
)
queue_rows = (
db.query(
AccountWorkItem.work_type,
AccountWorkItem.status,
func.count(AccountWorkItem.id),
func.min(AccountWorkItem.due_at),
)
.group_by(AccountWorkItem.work_type, AccountWorkItem.status)
.all()
)
work_queue: dict[str, dict[str, Any]] = {}
for work_type, status, count, oldest_due in queue_rows:
bucket = work_queue.setdefault(str(work_type), {})
bucket[str(status)] = int(count or 0)
if status == "pending":
bucket["oldest_pending_due_at"] = oldest_due
sync_rows = (
db.query(
AccountSyncState.domain,
AccountSyncState.status,
func.count(AccountSyncState.id),
)
.group_by(AccountSyncState.domain, AccountSyncState.status)
.all()
)
sync_domains: dict[str, dict[str, Any]] = {}
for domain, status, count in sync_rows:
bucket = sync_domains.setdefault(str(domain), {})
bucket[str(status)] = int(count or 0)
fresh_rows = (
db.query(AccountSyncState.domain, func.count(AccountSyncState.id))
.filter(
AccountSyncState.status == "success",
AccountSyncState.valid_until.isnot(None),
AccountSyncState.valid_until > moment,
)
.group_by(AccountSyncState.domain)
.all()
)
for domain, count in fresh_rows:
sync_domains.setdefault(str(domain), {})["fresh"] = int(count or 0)
commodity_rows = (
db.query(
AccountCommodityState.availability,
func.count(AccountCommodityState.id),
func.sum(
case(
(
AccountCommodityState.valid_until > moment,
1,
),
else_=0,
)
),
)
.filter(AccountCommodityState.platform == "douyu")
.group_by(AccountCommodityState.availability)
.all()
)
commodities = {
str(availability): {
"total": int(total or 0),
"fresh": int(fresh or 0),
}
for availability, total, fresh in commodity_rows
}
fresh_available = int(commodities.get(AVAILABILITY_AVAILABLE, {}).get("fresh", 0))
fresh_limited = int(commodities.get(AVAILABILITY_LIMITED, {}).get("fresh", 0))
inventories = evaluate_supply_targets(db, now=moment)
supply_targets: list[dict[str, Any]] = []
alerts: list[dict[str, Any]] = []
for item in inventories:
shortage = max(
item.covered_shortage,
item.available_shortage,
item.allocatable_shortage,
)
supply_targets.append(
{
"target_id": item.target_id,
"scope": item.scope,
"commodity_id": item.commodity_id,
"value_class": item.value_class,
"target_covered": item.target_covered,
"target_available": item.target_available,
"target_allocatable": item.target_allocatable,
"safety_buffer": item.safety_buffer,
"covered": item.covered,
"available": item.available,
"allocatable": item.allocatable,
"covered_shortage": item.covered_shortage,
"available_shortage": item.available_shortage,
"allocatable_shortage": item.allocatable_shortage,
"needs_replenishment": item.needs_replenishment,
}
)
if not item.needs_replenishment:
continue
critical = (
(item.target_available > 0 and item.available <= 0)
or (item.target_allocatable > 0 and item.allocatable <= 0)
)
alerts.append(
{
"code": "supply_shortage",
"severity": "critical" if critical else "warning",
"scope": item.scope,
"commodity_id": item.commodity_id,
"value_class": item.value_class,
"shortage": shortage,
"message": (
f"{item.scope}/{item.commodity_id or 'all'}/"
f"{item.value_class} 库存短缺 {shortage}"
),
}
)
if not inventories:
alerts.append(
{
"code": "supply_targets_missing",
"severity": "warning",
"message": "尚未配置启用的账号供给目标",
}
)
if expiring_24h:
alerts.append(
{
"code": "ck_expiring_24h",
"severity": "warning",
"count": expiring_24h,
"message": f"未来 24 小时预计有 {expiring_24h} 个 CK 到期",
}
)
overdue_work = int(
db.query(func.count(AccountWorkItem.id))
.filter(
AccountWorkItem.status == "pending",
AccountWorkItem.due_at < moment,
)
.scalar()
or 0
)
if overdue_work:
alerts.append(
{
"code": "account_work_overdue",
"severity": "warning",
"count": overdue_work,
"message": f"账号内部工作队列有 {overdue_work} 项待处理已过期",
}
)
trends = _platform_trends(db, moment)
query_sample = sum(
int(row["success"]) + int(row["failed"])
for row in trends["query"]
)
query_success = sum(int(row["success"]) for row in trends["query"])
query_rate = _rate(query_success, query_sample)
if query_sample >= 20 and query_rate < 80:
alerts.append(
{
"code": "query_success_rate_low",
"severity": "critical" if query_rate < 50 else "warning",
"count": query_sample,
"message": (
f"近 {TREND_DAYS} 天查询成功率 {query_rate:.1f}%"
f"({query_sample} 次已完成查询)"
),
}
)
login_sample = sum(
int(row["success"]) + int(row["failed"])
for row in trends["login"]
)
login_success = sum(int(row["success"]) for row in trends["login"])
login_rate = _rate(login_success, login_sample)
if login_sample >= 10 and login_rate < 60:
alerts.append(
{
"code": "login_success_rate_low",
"severity": "critical" if login_rate < 40 else "warning",
"count": login_sample,
"message": (
f"近 {TREND_DAYS} 天登录成功率 {login_rate:.1f}%"
f"({login_sample} 次已完成尝试)"
),
}
)
relogin_ok = sum(int(row["relogin_ok"]) for row in trends["lifecycle"])
relogin_failed = sum(
int(row["relogin_failed"]) for row in trends["lifecycle"]
)
if relogin_failed >= 5 and relogin_failed > relogin_ok:
alerts.append(
{
"code": "relogin_failure_trend_high",
"severity": "warning",
"count": relogin_failed,
"message": (
f"近 {TREND_DAYS} 天重登失败 {relogin_failed} 次,"
f"成功 {relogin_ok} 次"
),
}
)
return {
"checked_at": moment,
"accounts": {
"total": total_accounts,
"with_cookie": cookie_accounts,
"ck_valid": int(account_counts.get("valid", 0)),
"ck_invalid": int(account_counts.get("invalid", 0)),
"ck_error": int(account_counts.get("error", 0)),
"ck_unchecked": int(account_counts.get("", 0) or 0),
"ck_expiring_24h": expiring_24h,
"ck_expiring_72h": expiring_72h,
"ck_predicted_expired": predicted_expired,
},
"work_queue": work_queue,
"sync_domains": sync_domains,
"commodities": {
"by_availability": commodities,
"fresh_available": fresh_available,
"fresh_limited": fresh_limited,
},
"supply_targets": supply_targets,
"alerts": alerts,
"trends": trends,
}
__all__ = ["account_platform_status"]