perf(account-supply): 合并全局库存统计并增加查询索引

This commit is contained in:
yml
2026-11-22 14:04:24 +08:00
parent f2041f2444
commit 7233bdfa50
7 changed files with 395 additions and 99 deletions
@@ -453,6 +453,10 @@
"20261122_0113_platform_fulfillment_lease_policy.py": {
"revision": "20261122_0113",
"sha256": "195042a0f84e0ba8f0950e305e485ca4a96f69d9c0fd59b7dcc5a9f62b32357e"
},
"20261122_0114_account_commodity_global_available_index.py": {
"revision": "20261122_0114",
"sha256": "3b0da21df3a48784bf5e204d3fc95fc2e40c0edded4601869c36f1d52fe07d4b"
}
}
}
@@ -0,0 +1,51 @@
"""add global commodity availability index
全局账号供给目标不指定 commodity_id。现有 match 索引以 commodity_id 作为第三列,
查询只按 platform/scope/availability/valid_until 时无法形成有效范围,实际扫描
全部商品状态。新增与全局查询同形的组合索引,并用 account_id 和限时窗口字段
覆盖统计所需的列。
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261122_0114"
down_revision: str | None = "20261122_0113"
branch_labels: str | None = None
depends_on: str | None = None
TABLE_NAME = "account_commodity_states"
INDEX_NAME = "ix_account_commodity_global_available"
def _indexes() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(TABLE_NAME):
return set()
return {item["name"] for item in inspector.get_indexes(TABLE_NAME)}
def upgrade() -> None:
if INDEX_NAME in _indexes():
return
op.create_index(
INDEX_NAME,
TABLE_NAME,
[
"platform",
"scope",
"availability",
"valid_until",
"account_id",
"available_from",
"available_until",
],
unique=False,
)
def downgrade() -> None:
if INDEX_NAME in _indexes():
op.drop_index(INDEX_NAME, table_name=TABLE_NAME)
+12
View File
@@ -1971,6 +1971,18 @@ class AccountCommodityState(Base):
"platform",
"scope",
),
# 全局供给目标不指定 commodity_id,不能使用上面的 match 索引:
# commodity_id 位于前缀中间,会让 availability/valid_until 退化为过滤列。
Index(
"ix_account_commodity_global_available",
"platform",
"scope",
"availability",
"valid_until",
"account_id",
"available_from",
"available_until",
),
)
+25 -13
View File
@@ -70,7 +70,9 @@ from ..services.account_protection_service import (
update_protection_group,
)
from ..services.account_supply_service import (
SupplyInventory,
evaluate_supply_target,
evaluate_supply_targets,
update_supply_target,
upsert_supply_target,
)
@@ -701,8 +703,10 @@ def _config_out(config: DouyuConfig) -> DouyuConfigOut:
def _supply_target_out(
db: Session,
target: AccountSupplyTarget,
*,
inventory: SupplyInventory | None = None,
) -> DouyuSupplyTargetOut:
inventory = evaluate_supply_target(db, target)
resolved = inventory or evaluate_supply_target(db, target)
return DouyuSupplyTargetOut(
id=int(target.id),
target_id=int(target.id),
@@ -710,18 +714,18 @@ def _supply_target_out(
scope=target.scope,
commodity_id=target.commodity_id,
value_class=target.value_class,
target_covered=inventory.target_covered,
target_available=inventory.target_available,
target_allocatable=inventory.target_allocatable,
safety_buffer=inventory.safety_buffer,
target_covered=resolved.target_covered,
target_available=resolved.target_available,
target_allocatable=resolved.target_allocatable,
safety_buffer=resolved.safety_buffer,
enabled=bool(target.enabled),
covered=inventory.covered,
available=inventory.available,
allocatable=inventory.allocatable,
covered_shortage=inventory.covered_shortage,
available_shortage=inventory.available_shortage,
allocatable_shortage=inventory.allocatable_shortage,
needs_replenishment=inventory.needs_replenishment,
covered=resolved.covered,
available=resolved.available,
allocatable=resolved.allocatable,
covered_shortage=resolved.covered_shortage,
available_shortage=resolved.available_shortage,
allocatable_shortage=resolved.allocatable_shortage,
needs_replenishment=resolved.needs_replenishment,
updated_by=target.updated_by,
updated_at=target.updated_at,
)
@@ -1056,7 +1060,15 @@ def list_supply_targets(
)
.all()
)
return [_supply_target_out(db, row) for row in rows]
enabled_inventories = {item.target_id: item for item in evaluate_supply_targets(db)}
return [
_supply_target_out(
db,
row,
inventory=enabled_inventories.get(int(row.id)),
)
for row in rows
]
@router.post("/supply-targets", response_model=DouyuSupplyTargetOut)
+184 -82
View File
@@ -6,7 +6,7 @@ import os
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from sqlalchemy import and_, or_, select
from sqlalchemy import and_, case, func, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -96,8 +96,9 @@ def _active_lease_exists():
return (
select(AccountLease.id)
.where(
AccountLease.account_id == Account.id,
AccountLease.status == "active",
# active_account_id 是生成列,只在 status='active' 时有值。
# 直接命中唯一索引,避免再按 account_id 扫历史租约。
AccountLease.active_account_id == Account.id,
)
.exists()
)
@@ -124,25 +125,29 @@ def _value_filter(value_class: str, *, now: datetime):
raise ValueError(f"不支持的账号价值分类: {value_class}")
def _base_account_filter(*, now: datetime):
ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS)
return and_(
Account.cookie.isnot(None),
Account.cookie != "",
Account.ck_check_status == "valid",
Account.ck_checked_at.isnot(None),
Account.ck_checked_at >= ck_fresh_cutoff,
Account.auto_managed.is_(True),
Account.status.notin_(MANUAL_STATES),
)
def _base_account_query(
db: Session,
*,
now: datetime,
value_class: str,
):
ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS)
return (
db.query(Account)
.filter(
Account.cookie.isnot(None),
Account.cookie != "",
Account.ck_check_status == "valid",
Account.ck_checked_at.isnot(None),
Account.ck_checked_at >= ck_fresh_cutoff,
Account.auto_managed.is_(True),
Account.status.notin_(MANUAL_STATES),
_value_filter(value_class, now=now),
)
.filter(_base_account_filter(now=now))
.filter(_value_filter(value_class, now=now))
)
@@ -179,45 +184,148 @@ def _commodity_query(
now: datetime,
target: AccountSupplyTarget,
):
query = _base_account_query(
db,
now=now,
value_class=target.value_class,
).join(
AccountCommodityState,
AccountCommodityState.account_id == Account.id,
).filter(
AccountCommodityState.platform == target.platform,
AccountCommodityState.scope == target.scope,
AccountCommodityState.commodity_id == target.commodity_id,
AccountCommodityState.observed_at.isnot(None),
query = (
_base_account_query(
db,
now=now,
value_class=target.value_class,
)
.join(
AccountCommodityState,
AccountCommodityState.account_id == Account.id,
)
.filter(
AccountCommodityState.platform == target.platform,
AccountCommodityState.scope == target.scope,
AccountCommodityState.commodity_id == target.commodity_id,
AccountCommodityState.observed_at.isnot(None),
)
)
return query
def _global_state_exists(target: AccountSupplyTarget):
def _global_state_exists_for(*, platform: str, scope: str):
return (
select(AccountCommodityState.id)
.where(
AccountCommodityState.account_id == Account.id,
AccountCommodityState.platform == target.platform,
AccountCommodityState.scope == target.scope,
AccountCommodityState.platform == platform,
AccountCommodityState.scope == scope,
AccountCommodityState.observed_at.isnot(None),
)
.exists()
)
def _global_available_exists(target: AccountSupplyTarget, *, now: datetime):
return (
select(AccountCommodityState.id)
.where(
def _global_inventory_totals(
db: Session,
*,
platform: str,
scope: str,
now: datetime,
) -> dict[str, tuple[int, int, int]]:
"""一次算出某作用域下高价值/普通的覆盖、可兑换和可分配数量。
全局目标不指定 commodity_id,因此不能用
``ix_account_commodity_match`` 的 commodity_id 前缀。固定拆成两条 SQL:
1. accounts + EXISTS:计算 covered;
2. account_commodity_states JOIN accounts:同时计算 available/allocatable。
第二条走 ``ix_account_commodity_global_available``,避免先物化全部商品状态。
"""
value_class = case(
(is_high_value_expr(now=now), VALUE_HIGH),
else_=VALUE_ORDINARY,
)
base_filter = _base_account_filter(now=now)
covered_rows = (
db.query(value_class, func.count(Account.id))
.select_from(Account)
.filter(
base_filter,
_global_state_exists_for(platform=platform, scope=scope),
)
.group_by(value_class)
.all()
)
totals = {
VALUE_HIGH: [0, 0, 0],
VALUE_ORDINARY: [0, 0, 0],
}
for class_name, covered in covered_rows:
totals[str(class_name)][0] = int(covered or 0)
allocatable_account = case(
(_allocatable_filter(now), Account.id),
else_=None,
)
available_rows = (
db.query(
value_class,
func.count(func.distinct(Account.id)),
func.count(func.distinct(allocatable_account)),
)
.select_from(Account)
.join(
AccountCommodityState,
AccountCommodityState.account_id == Account.id,
AccountCommodityState.platform == target.platform,
AccountCommodityState.scope == target.scope,
)
.filter(
base_filter,
AccountCommodityState.platform == platform,
AccountCommodityState.scope == scope,
_fresh_available_filter(now),
)
.exists()
.group_by(value_class)
.all()
)
for class_name, available, allocatable in available_rows:
bucket = totals[str(class_name)]
bucket[1] = int(available or 0)
bucket[2] = int(allocatable or 0)
return {
class_name: (values[0], values[1], values[2])
for class_name, values in totals.items()
}
def _inventory_from_counts(
target: AccountSupplyTarget,
*,
covered: int,
available: int,
allocatable: int,
) -> SupplyInventory:
return SupplyInventory(
target_id=int(target.id),
scope=target.scope,
commodity_id=target.commodity_id,
value_class=target.value_class,
covered=covered,
available=available,
allocatable=allocatable,
target_covered=int(target.target_covered or 0),
target_available=int(target.target_available or 0),
target_allocatable=int(target.target_allocatable or 0),
safety_buffer=int(target.safety_buffer or 0),
covered_shortage=max(
0,
int(target.target_covered or 0) + int(target.safety_buffer or 0) - covered,
),
available_shortage=max(
0,
int(target.target_available or 0)
+ int(target.safety_buffer or 0)
- available,
),
allocatable_shortage=max(
0,
int(target.target_allocatable or 0)
+ int(target.safety_buffer or 0)
- allocatable,
),
)
@@ -237,51 +345,21 @@ def evaluate_supply_target(
covered = base.count()
available_query = base.filter(_fresh_available_filter(moment))
available = available_query.count()
allocatable = available_query.filter(
_allocatable_filter(moment)
).count()
allocatable = available_query.filter(_allocatable_filter(moment)).count()
else:
base = _base_account_query(
totals = _global_inventory_totals(
db,
platform=target.platform,
scope=target.scope,
now=moment,
value_class=target.value_class,
)
covered = base.filter(_global_state_exists(target)).count()
available_query = base.filter(
_global_available_exists(target, now=moment)
)
available = available_query.count()
allocatable = available_query.filter(
_allocatable_filter(moment)
).count()
covered, available, allocatable = totals[target.value_class]
return SupplyInventory(
target_id=int(target.id),
scope=target.scope,
commodity_id=target.commodity_id,
value_class=target.value_class,
return _inventory_from_counts(
target,
covered=covered,
available=available,
allocatable=allocatable,
target_covered=int(target.target_covered or 0),
target_available=int(target.target_available or 0),
target_allocatable=int(target.target_allocatable or 0),
safety_buffer=int(target.safety_buffer or 0),
covered_shortage=max(
0,
int(target.target_covered or 0) + int(target.safety_buffer or 0)
- covered,
),
available_shortage=max(
0,
int(target.target_available or 0) + int(target.safety_buffer or 0)
- available,
),
allocatable_shortage=max(
0,
int(target.target_allocatable or 0) + int(target.safety_buffer or 0)
- allocatable,
),
)
@@ -302,10 +380,30 @@ def evaluate_supply_targets(
)
.all()
)
return [
evaluate_supply_target(db, target, now=moment)
for target in targets
]
global_totals: dict[tuple[str, str], dict[str, tuple[int, int, int]]] = {}
inventories: list[SupplyInventory] = []
for target in targets:
if target.commodity_id:
inventories.append(evaluate_supply_target(db, target, now=moment))
continue
key = (target.platform, target.scope)
if key not in global_totals:
global_totals[key] = _global_inventory_totals(
db,
platform=target.platform,
scope=target.scope,
now=moment,
)
covered, available, allocatable = global_totals[key][target.value_class]
inventories.append(
_inventory_from_counts(
target,
covered=covered,
available=available,
allocatable=allocatable,
)
)
return inventories
def upsert_supply_target(
@@ -502,11 +600,15 @@ def enqueue_supply_replenishment(
inventory.allocatable_shortage,
)
limit = max(1, min(int(max_per_target), missing))
accounts = _sync_candidate_query(
db,
target=target,
now=moment,
).limit(limit).all()
accounts = (
_sync_candidate_query(
db,
target=target,
now=moment,
)
.limit(limit)
.all()
)
for account in accounts:
enqueue_account_work(
db,