perf(account-supply): 合并全局库存统计并增加查询索引
This commit is contained in:
@@ -453,6 +453,10 @@
|
||||
"20261122_0113_platform_fulfillment_lease_policy.py": {
|
||||
"revision": "20261122_0113",
|
||||
"sha256": "195042a0f84e0ba8f0950e305e485ca4a96f69d9c0fd59b7dcc5a9f62b32357e"
|
||||
},
|
||||
"20261122_0114_account_commodity_global_available_index.py": {
|
||||
"revision": "20261122_0114",
|
||||
"sha256": "3b0da21df3a48784bf5e204d3fc95fc2e40c0edded4601869c36f1d52fe07d4b"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""add global commodity availability index
|
||||
|
||||
全局账号供给目标不指定 commodity_id。现有 match 索引以 commodity_id 作为第三列,
|
||||
查询只按 platform/scope/availability/valid_until 时无法形成有效范围,实际扫描
|
||||
全部商品状态。新增与全局查询同形的组合索引,并用 account_id 和限时窗口字段
|
||||
覆盖统计所需的列。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20261122_0114"
|
||||
down_revision: str | None = "20261122_0113"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
TABLE_NAME = "account_commodity_states"
|
||||
INDEX_NAME = "ix_account_commodity_global_available"
|
||||
|
||||
|
||||
def _indexes() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(TABLE_NAME):
|
||||
return set()
|
||||
return {item["name"] for item in inspector.get_indexes(TABLE_NAME)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if INDEX_NAME in _indexes():
|
||||
return
|
||||
op.create_index(
|
||||
INDEX_NAME,
|
||||
TABLE_NAME,
|
||||
[
|
||||
"platform",
|
||||
"scope",
|
||||
"availability",
|
||||
"valid_until",
|
||||
"account_id",
|
||||
"available_from",
|
||||
"available_until",
|
||||
],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if INDEX_NAME in _indexes():
|
||||
op.drop_index(INDEX_NAME, table_name=TABLE_NAME)
|
||||
@@ -1971,6 +1971,18 @@ class AccountCommodityState(Base):
|
||||
"platform",
|
||||
"scope",
|
||||
),
|
||||
# 全局供给目标不指定 commodity_id,不能使用上面的 match 索引:
|
||||
# commodity_id 位于前缀中间,会让 availability/valid_until 退化为过滤列。
|
||||
Index(
|
||||
"ix_account_commodity_global_available",
|
||||
"platform",
|
||||
"scope",
|
||||
"availability",
|
||||
"valid_until",
|
||||
"account_id",
|
||||
"available_from",
|
||||
"available_until",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -70,7 +70,9 @@ from ..services.account_protection_service import (
|
||||
update_protection_group,
|
||||
)
|
||||
from ..services.account_supply_service import (
|
||||
SupplyInventory,
|
||||
evaluate_supply_target,
|
||||
evaluate_supply_targets,
|
||||
update_supply_target,
|
||||
upsert_supply_target,
|
||||
)
|
||||
@@ -701,8 +703,10 @@ def _config_out(config: DouyuConfig) -> DouyuConfigOut:
|
||||
def _supply_target_out(
|
||||
db: Session,
|
||||
target: AccountSupplyTarget,
|
||||
*,
|
||||
inventory: SupplyInventory | None = None,
|
||||
) -> DouyuSupplyTargetOut:
|
||||
inventory = evaluate_supply_target(db, target)
|
||||
resolved = inventory or evaluate_supply_target(db, target)
|
||||
return DouyuSupplyTargetOut(
|
||||
id=int(target.id),
|
||||
target_id=int(target.id),
|
||||
@@ -710,18 +714,18 @@ def _supply_target_out(
|
||||
scope=target.scope,
|
||||
commodity_id=target.commodity_id,
|
||||
value_class=target.value_class,
|
||||
target_covered=inventory.target_covered,
|
||||
target_available=inventory.target_available,
|
||||
target_allocatable=inventory.target_allocatable,
|
||||
safety_buffer=inventory.safety_buffer,
|
||||
target_covered=resolved.target_covered,
|
||||
target_available=resolved.target_available,
|
||||
target_allocatable=resolved.target_allocatable,
|
||||
safety_buffer=resolved.safety_buffer,
|
||||
enabled=bool(target.enabled),
|
||||
covered=inventory.covered,
|
||||
available=inventory.available,
|
||||
allocatable=inventory.allocatable,
|
||||
covered_shortage=inventory.covered_shortage,
|
||||
available_shortage=inventory.available_shortage,
|
||||
allocatable_shortage=inventory.allocatable_shortage,
|
||||
needs_replenishment=inventory.needs_replenishment,
|
||||
covered=resolved.covered,
|
||||
available=resolved.available,
|
||||
allocatable=resolved.allocatable,
|
||||
covered_shortage=resolved.covered_shortage,
|
||||
available_shortage=resolved.available_shortage,
|
||||
allocatable_shortage=resolved.allocatable_shortage,
|
||||
needs_replenishment=resolved.needs_replenishment,
|
||||
updated_by=target.updated_by,
|
||||
updated_at=target.updated_at,
|
||||
)
|
||||
@@ -1056,7 +1060,15 @@ def list_supply_targets(
|
||||
)
|
||||
.all()
|
||||
)
|
||||
return [_supply_target_out(db, row) for row in rows]
|
||||
enabled_inventories = {item.target_id: item for item in evaluate_supply_targets(db)}
|
||||
return [
|
||||
_supply_target_out(
|
||||
db,
|
||||
row,
|
||||
inventory=enabled_inventories.get(int(row.id)),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
@router.post("/supply-targets", response_model=DouyuSupplyTargetOut)
|
||||
|
||||
@@ -6,7 +6,7 @@ import os
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import and_, or_, select
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -96,8 +96,9 @@ def _active_lease_exists():
|
||||
return (
|
||||
select(AccountLease.id)
|
||||
.where(
|
||||
AccountLease.account_id == Account.id,
|
||||
AccountLease.status == "active",
|
||||
# active_account_id 是生成列,只在 status='active' 时有值。
|
||||
# 直接命中唯一索引,避免再按 account_id 扫历史租约。
|
||||
AccountLease.active_account_id == Account.id,
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
@@ -124,25 +125,29 @@ def _value_filter(value_class: str, *, now: datetime):
|
||||
raise ValueError(f"不支持的账号价值分类: {value_class}")
|
||||
|
||||
|
||||
def _base_account_filter(*, now: datetime):
|
||||
ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS)
|
||||
return and_(
|
||||
Account.cookie.isnot(None),
|
||||
Account.cookie != "",
|
||||
Account.ck_check_status == "valid",
|
||||
Account.ck_checked_at.isnot(None),
|
||||
Account.ck_checked_at >= ck_fresh_cutoff,
|
||||
Account.auto_managed.is_(True),
|
||||
Account.status.notin_(MANUAL_STATES),
|
||||
)
|
||||
|
||||
|
||||
def _base_account_query(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime,
|
||||
value_class: str,
|
||||
):
|
||||
ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS)
|
||||
return (
|
||||
db.query(Account)
|
||||
.filter(
|
||||
Account.cookie.isnot(None),
|
||||
Account.cookie != "",
|
||||
Account.ck_check_status == "valid",
|
||||
Account.ck_checked_at.isnot(None),
|
||||
Account.ck_checked_at >= ck_fresh_cutoff,
|
||||
Account.auto_managed.is_(True),
|
||||
Account.status.notin_(MANUAL_STATES),
|
||||
_value_filter(value_class, now=now),
|
||||
)
|
||||
.filter(_base_account_filter(now=now))
|
||||
.filter(_value_filter(value_class, now=now))
|
||||
)
|
||||
|
||||
|
||||
@@ -179,45 +184,148 @@ def _commodity_query(
|
||||
now: datetime,
|
||||
target: AccountSupplyTarget,
|
||||
):
|
||||
query = _base_account_query(
|
||||
db,
|
||||
now=now,
|
||||
value_class=target.value_class,
|
||||
).join(
|
||||
AccountCommodityState,
|
||||
AccountCommodityState.account_id == Account.id,
|
||||
).filter(
|
||||
AccountCommodityState.platform == target.platform,
|
||||
AccountCommodityState.scope == target.scope,
|
||||
AccountCommodityState.commodity_id == target.commodity_id,
|
||||
AccountCommodityState.observed_at.isnot(None),
|
||||
query = (
|
||||
_base_account_query(
|
||||
db,
|
||||
now=now,
|
||||
value_class=target.value_class,
|
||||
)
|
||||
.join(
|
||||
AccountCommodityState,
|
||||
AccountCommodityState.account_id == Account.id,
|
||||
)
|
||||
.filter(
|
||||
AccountCommodityState.platform == target.platform,
|
||||
AccountCommodityState.scope == target.scope,
|
||||
AccountCommodityState.commodity_id == target.commodity_id,
|
||||
AccountCommodityState.observed_at.isnot(None),
|
||||
)
|
||||
)
|
||||
return query
|
||||
|
||||
|
||||
def _global_state_exists(target: AccountSupplyTarget):
|
||||
def _global_state_exists_for(*, platform: str, scope: str):
|
||||
return (
|
||||
select(AccountCommodityState.id)
|
||||
.where(
|
||||
AccountCommodityState.account_id == Account.id,
|
||||
AccountCommodityState.platform == target.platform,
|
||||
AccountCommodityState.scope == target.scope,
|
||||
AccountCommodityState.platform == platform,
|
||||
AccountCommodityState.scope == scope,
|
||||
AccountCommodityState.observed_at.isnot(None),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def _global_available_exists(target: AccountSupplyTarget, *, now: datetime):
|
||||
return (
|
||||
select(AccountCommodityState.id)
|
||||
.where(
|
||||
def _global_inventory_totals(
|
||||
db: Session,
|
||||
*,
|
||||
platform: str,
|
||||
scope: str,
|
||||
now: datetime,
|
||||
) -> dict[str, tuple[int, int, int]]:
|
||||
"""一次算出某作用域下高价值/普通的覆盖、可兑换和可分配数量。
|
||||
|
||||
全局目标不指定 commodity_id,因此不能用
|
||||
``ix_account_commodity_match`` 的 commodity_id 前缀。固定拆成两条 SQL:
|
||||
|
||||
1. accounts + EXISTS:计算 covered;
|
||||
2. account_commodity_states JOIN accounts:同时计算 available/allocatable。
|
||||
|
||||
第二条走 ``ix_account_commodity_global_available``,避免先物化全部商品状态。
|
||||
"""
|
||||
value_class = case(
|
||||
(is_high_value_expr(now=now), VALUE_HIGH),
|
||||
else_=VALUE_ORDINARY,
|
||||
)
|
||||
base_filter = _base_account_filter(now=now)
|
||||
|
||||
covered_rows = (
|
||||
db.query(value_class, func.count(Account.id))
|
||||
.select_from(Account)
|
||||
.filter(
|
||||
base_filter,
|
||||
_global_state_exists_for(platform=platform, scope=scope),
|
||||
)
|
||||
.group_by(value_class)
|
||||
.all()
|
||||
)
|
||||
totals = {
|
||||
VALUE_HIGH: [0, 0, 0],
|
||||
VALUE_ORDINARY: [0, 0, 0],
|
||||
}
|
||||
for class_name, covered in covered_rows:
|
||||
totals[str(class_name)][0] = int(covered or 0)
|
||||
|
||||
allocatable_account = case(
|
||||
(_allocatable_filter(now), Account.id),
|
||||
else_=None,
|
||||
)
|
||||
available_rows = (
|
||||
db.query(
|
||||
value_class,
|
||||
func.count(func.distinct(Account.id)),
|
||||
func.count(func.distinct(allocatable_account)),
|
||||
)
|
||||
.select_from(Account)
|
||||
.join(
|
||||
AccountCommodityState,
|
||||
AccountCommodityState.account_id == Account.id,
|
||||
AccountCommodityState.platform == target.platform,
|
||||
AccountCommodityState.scope == target.scope,
|
||||
)
|
||||
.filter(
|
||||
base_filter,
|
||||
AccountCommodityState.platform == platform,
|
||||
AccountCommodityState.scope == scope,
|
||||
_fresh_available_filter(now),
|
||||
)
|
||||
.exists()
|
||||
.group_by(value_class)
|
||||
.all()
|
||||
)
|
||||
for class_name, available, allocatable in available_rows:
|
||||
bucket = totals[str(class_name)]
|
||||
bucket[1] = int(available or 0)
|
||||
bucket[2] = int(allocatable or 0)
|
||||
return {
|
||||
class_name: (values[0], values[1], values[2])
|
||||
for class_name, values in totals.items()
|
||||
}
|
||||
|
||||
|
||||
def _inventory_from_counts(
|
||||
target: AccountSupplyTarget,
|
||||
*,
|
||||
covered: int,
|
||||
available: int,
|
||||
allocatable: int,
|
||||
) -> SupplyInventory:
|
||||
return SupplyInventory(
|
||||
target_id=int(target.id),
|
||||
scope=target.scope,
|
||||
commodity_id=target.commodity_id,
|
||||
value_class=target.value_class,
|
||||
covered=covered,
|
||||
available=available,
|
||||
allocatable=allocatable,
|
||||
target_covered=int(target.target_covered or 0),
|
||||
target_available=int(target.target_available or 0),
|
||||
target_allocatable=int(target.target_allocatable or 0),
|
||||
safety_buffer=int(target.safety_buffer or 0),
|
||||
covered_shortage=max(
|
||||
0,
|
||||
int(target.target_covered or 0) + int(target.safety_buffer or 0) - covered,
|
||||
),
|
||||
available_shortage=max(
|
||||
0,
|
||||
int(target.target_available or 0)
|
||||
+ int(target.safety_buffer or 0)
|
||||
- available,
|
||||
),
|
||||
allocatable_shortage=max(
|
||||
0,
|
||||
int(target.target_allocatable or 0)
|
||||
+ int(target.safety_buffer or 0)
|
||||
- allocatable,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -237,51 +345,21 @@ def evaluate_supply_target(
|
||||
covered = base.count()
|
||||
available_query = base.filter(_fresh_available_filter(moment))
|
||||
available = available_query.count()
|
||||
allocatable = available_query.filter(
|
||||
_allocatable_filter(moment)
|
||||
).count()
|
||||
allocatable = available_query.filter(_allocatable_filter(moment)).count()
|
||||
else:
|
||||
base = _base_account_query(
|
||||
totals = _global_inventory_totals(
|
||||
db,
|
||||
platform=target.platform,
|
||||
scope=target.scope,
|
||||
now=moment,
|
||||
value_class=target.value_class,
|
||||
)
|
||||
covered = base.filter(_global_state_exists(target)).count()
|
||||
available_query = base.filter(
|
||||
_global_available_exists(target, now=moment)
|
||||
)
|
||||
available = available_query.count()
|
||||
allocatable = available_query.filter(
|
||||
_allocatable_filter(moment)
|
||||
).count()
|
||||
covered, available, allocatable = totals[target.value_class]
|
||||
|
||||
return SupplyInventory(
|
||||
target_id=int(target.id),
|
||||
scope=target.scope,
|
||||
commodity_id=target.commodity_id,
|
||||
value_class=target.value_class,
|
||||
return _inventory_from_counts(
|
||||
target,
|
||||
covered=covered,
|
||||
available=available,
|
||||
allocatable=allocatable,
|
||||
target_covered=int(target.target_covered or 0),
|
||||
target_available=int(target.target_available or 0),
|
||||
target_allocatable=int(target.target_allocatable or 0),
|
||||
safety_buffer=int(target.safety_buffer or 0),
|
||||
covered_shortage=max(
|
||||
0,
|
||||
int(target.target_covered or 0) + int(target.safety_buffer or 0)
|
||||
- covered,
|
||||
),
|
||||
available_shortage=max(
|
||||
0,
|
||||
int(target.target_available or 0) + int(target.safety_buffer or 0)
|
||||
- available,
|
||||
),
|
||||
allocatable_shortage=max(
|
||||
0,
|
||||
int(target.target_allocatable or 0) + int(target.safety_buffer or 0)
|
||||
- allocatable,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -302,10 +380,30 @@ def evaluate_supply_targets(
|
||||
)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
evaluate_supply_target(db, target, now=moment)
|
||||
for target in targets
|
||||
]
|
||||
global_totals: dict[tuple[str, str], dict[str, tuple[int, int, int]]] = {}
|
||||
inventories: list[SupplyInventory] = []
|
||||
for target in targets:
|
||||
if target.commodity_id:
|
||||
inventories.append(evaluate_supply_target(db, target, now=moment))
|
||||
continue
|
||||
key = (target.platform, target.scope)
|
||||
if key not in global_totals:
|
||||
global_totals[key] = _global_inventory_totals(
|
||||
db,
|
||||
platform=target.platform,
|
||||
scope=target.scope,
|
||||
now=moment,
|
||||
)
|
||||
covered, available, allocatable = global_totals[key][target.value_class]
|
||||
inventories.append(
|
||||
_inventory_from_counts(
|
||||
target,
|
||||
covered=covered,
|
||||
available=available,
|
||||
allocatable=allocatable,
|
||||
)
|
||||
)
|
||||
return inventories
|
||||
|
||||
|
||||
def upsert_supply_target(
|
||||
@@ -502,11 +600,15 @@ def enqueue_supply_replenishment(
|
||||
inventory.allocatable_shortage,
|
||||
)
|
||||
limit = max(1, min(int(max_per_target), missing))
|
||||
accounts = _sync_candidate_query(
|
||||
db,
|
||||
target=target,
|
||||
now=moment,
|
||||
).limit(limit).all()
|
||||
accounts = (
|
||||
_sync_candidate_query(
|
||||
db,
|
||||
target=target,
|
||||
now=moment,
|
||||
)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
for account in accounts:
|
||||
enqueue_account_work(
|
||||
db,
|
||||
|
||||
Reference in New Issue
Block a user