perf(account-supply): 合并全局库存统计并增加查询索引

This commit is contained in:
yml
2026-11-22 14:04:24 +08:00
parent f2041f2444
commit 7233bdfa50
7 changed files with 395 additions and 99 deletions
+184 -82
View File
@@ -6,7 +6,7 @@ import os
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from sqlalchemy import and_, or_, select
from sqlalchemy import and_, case, func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -96,8 +96,9 @@ def _active_lease_exists():
return (
select(AccountLease.id)
.where(
AccountLease.account_id == Account.id,
AccountLease.status == "active",
# active_account_id 是生成列,只在 status='active' 时有值。
# 直接命中唯一索引,避免再按 account_id 扫历史租约。
AccountLease.active_account_id == Account.id,
)
.exists()
)
@@ -124,25 +125,29 @@ def _value_filter(value_class: str, *, now: datetime):
raise ValueError(f"不支持的账号价值分类: {value_class}")
def _base_account_filter(*, now: datetime):
ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS)
return and_(
Account.cookie.isnot(None),
Account.cookie != "",
Account.ck_check_status == "valid",
Account.ck_checked_at.isnot(None),
Account.ck_checked_at >= ck_fresh_cutoff,
Account.auto_managed.is_(True),
Account.status.notin_(MANUAL_STATES),
)
def _base_account_query(
db: Session,
*,
now: datetime,
value_class: str,
):
ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS)
return (
db.query(Account)
.filter(
Account.cookie.isnot(None),
Account.cookie != "",
Account.ck_check_status == "valid",
Account.ck_checked_at.isnot(None),
Account.ck_checked_at >= ck_fresh_cutoff,
Account.auto_managed.is_(True),
Account.status.notin_(MANUAL_STATES),
_value_filter(value_class, now=now),
)
.filter(_base_account_filter(now=now))
.filter(_value_filter(value_class, now=now))
)
@@ -179,45 +184,148 @@ def _commodity_query(
now: datetime,
target: AccountSupplyTarget,
):
query = _base_account_query(
db,
now=now,
value_class=target.value_class,
).join(
AccountCommodityState,
AccountCommodityState.account_id == Account.id,
).filter(
AccountCommodityState.platform == target.platform,
AccountCommodityState.scope == target.scope,
AccountCommodityState.commodity_id == target.commodity_id,
AccountCommodityState.observed_at.isnot(None),
query = (
_base_account_query(
db,
now=now,
value_class=target.value_class,
)
.join(
AccountCommodityState,
AccountCommodityState.account_id == Account.id,
)
.filter(
AccountCommodityState.platform == target.platform,
AccountCommodityState.scope == target.scope,
AccountCommodityState.commodity_id == target.commodity_id,
AccountCommodityState.observed_at.isnot(None),
)
)
return query
def _global_state_exists(target: AccountSupplyTarget):
def _global_state_exists_for(*, platform: str, scope: str):
return (
select(AccountCommodityState.id)
.where(
AccountCommodityState.account_id == Account.id,
AccountCommodityState.platform == target.platform,
AccountCommodityState.scope == target.scope,
AccountCommodityState.platform == platform,
AccountCommodityState.scope == scope,
AccountCommodityState.observed_at.isnot(None),
)
.exists()
)
def _global_available_exists(target: AccountSupplyTarget, *, now: datetime):
return (
select(AccountCommodityState.id)
.where(
def _global_inventory_totals(
db: Session,
*,
platform: str,
scope: str,
now: datetime,
) -> dict[str, tuple[int, int, int]]:
"""一次算出某作用域下高价值/普通的覆盖、可兑换和可分配数量。
全局目标不指定 commodity_id,因此不能用
``ix_account_commodity_match`` 的 commodity_id 前缀。固定拆成两条 SQL:
1. accounts + EXISTS:计算 covered;
2. account_commodity_states JOIN accounts:同时计算 available/allocatable。
第二条走 ``ix_account_commodity_global_available``,避免先物化全部商品状态。
"""
value_class = case(
(is_high_value_expr(now=now), VALUE_HIGH),
else_=VALUE_ORDINARY,
)
base_filter = _base_account_filter(now=now)
covered_rows = (
db.query(value_class, func.count(Account.id))
.select_from(Account)
.filter(
base_filter,
_global_state_exists_for(platform=platform, scope=scope),
)
.group_by(value_class)
.all()
)
totals = {
VALUE_HIGH: [0, 0, 0],
VALUE_ORDINARY: [0, 0, 0],
}
for class_name, covered in covered_rows:
totals[str(class_name)][0] = int(covered or 0)
allocatable_account = case(
(_allocatable_filter(now), Account.id),
else_=None,
)
available_rows = (
db.query(
value_class,
func.count(func.distinct(Account.id)),
func.count(func.distinct(allocatable_account)),
)
.select_from(Account)
.join(
AccountCommodityState,
AccountCommodityState.account_id == Account.id,
AccountCommodityState.platform == target.platform,
AccountCommodityState.scope == target.scope,
)
.filter(
base_filter,
AccountCommodityState.platform == platform,
AccountCommodityState.scope == scope,
_fresh_available_filter(now),
)
.exists()
.group_by(value_class)
.all()
)
for class_name, available, allocatable in available_rows:
bucket = totals[str(class_name)]
bucket[1] = int(available or 0)
bucket[2] = int(allocatable or 0)
return {
class_name: (values[0], values[1], values[2])
for class_name, values in totals.items()
}
def _inventory_from_counts(
target: AccountSupplyTarget,
*,
covered: int,
available: int,
allocatable: int,
) -> SupplyInventory:
return SupplyInventory(
target_id=int(target.id),
scope=target.scope,
commodity_id=target.commodity_id,
value_class=target.value_class,
covered=covered,
available=available,
allocatable=allocatable,
target_covered=int(target.target_covered or 0),
target_available=int(target.target_available or 0),
target_allocatable=int(target.target_allocatable or 0),
safety_buffer=int(target.safety_buffer or 0),
covered_shortage=max(
0,
int(target.target_covered or 0) + int(target.safety_buffer or 0) - covered,
),
available_shortage=max(
0,
int(target.target_available or 0)
+ int(target.safety_buffer or 0)
- available,
),
allocatable_shortage=max(
0,
int(target.target_allocatable or 0)
+ int(target.safety_buffer or 0)
- allocatable,
),
)
@@ -237,51 +345,21 @@ def evaluate_supply_target(
covered = base.count()
available_query = base.filter(_fresh_available_filter(moment))
available = available_query.count()
allocatable = available_query.filter(
_allocatable_filter(moment)
).count()
allocatable = available_query.filter(_allocatable_filter(moment)).count()
else:
base = _base_account_query(
totals = _global_inventory_totals(
db,
platform=target.platform,
scope=target.scope,
now=moment,
value_class=target.value_class,
)
covered = base.filter(_global_state_exists(target)).count()
available_query = base.filter(
_global_available_exists(target, now=moment)
)
available = available_query.count()
allocatable = available_query.filter(
_allocatable_filter(moment)
).count()
covered, available, allocatable = totals[target.value_class]
return SupplyInventory(
target_id=int(target.id),
scope=target.scope,
commodity_id=target.commodity_id,
value_class=target.value_class,
return _inventory_from_counts(
target,
covered=covered,
available=available,
allocatable=allocatable,
target_covered=int(target.target_covered or 0),
target_available=int(target.target_available or 0),
target_allocatable=int(target.target_allocatable or 0),
safety_buffer=int(target.safety_buffer or 0),
covered_shortage=max(
0,
int(target.target_covered or 0) + int(target.safety_buffer or 0)
- covered,
),
available_shortage=max(
0,
int(target.target_available or 0) + int(target.safety_buffer or 0)
- available,
),
allocatable_shortage=max(
0,
int(target.target_allocatable or 0) + int(target.safety_buffer or 0)
- allocatable,
),
)
@@ -302,10 +380,30 @@ def evaluate_supply_targets(
)
.all()
)
return [
evaluate_supply_target(db, target, now=moment)
for target in targets
]
global_totals: dict[tuple[str, str], dict[str, tuple[int, int, int]]] = {}
inventories: list[SupplyInventory] = []
for target in targets:
if target.commodity_id:
inventories.append(evaluate_supply_target(db, target, now=moment))
continue
key = (target.platform, target.scope)
if key not in global_totals:
global_totals[key] = _global_inventory_totals(
db,
platform=target.platform,
scope=target.scope,
now=moment,
)
covered, available, allocatable = global_totals[key][target.value_class]
inventories.append(
_inventory_from_counts(
target,
covered=covered,
available=available,
allocatable=allocatable,
)
)
return inventories
def upsert_supply_target(
@@ -502,11 +600,15 @@ def enqueue_supply_replenishment(
inventory.allocatable_shortage,
)
limit = max(1, min(int(max_per_target), missing))
accounts = _sync_candidate_query(
db,
target=target,
now=moment,
).limit(limit).all()
accounts = (
_sync_candidate_query(
db,
target=target,
now=moment,
)
.limit(limit)
.all()
)
for account in accounts:
enqueue_account_work(
db,