diff --git a/tests/test_account_supply_service.py b/tests/test_account_supply_service.py index c607992..52974a1 100644 --- a/tests/test_account_supply_service.py +++ b/tests/test_account_supply_service.py @@ -4,6 +4,8 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta +from sqlalchemy import event + from web.backend.models import ( Account, AccountCommodityState, @@ -13,6 +15,7 @@ from web.backend.models import ( AccountSyncRun, AccountSyncState, AccountWorkItem, + DouyuExternalOrder, LoginAttempt, User, ) @@ -21,9 +24,11 @@ from web.backend.schemas import DouyuSupplyTargetUpsert from web.backend.services.account_observability import account_platform_status from web.backend.services.account_supply_service import ( VALUE_HIGH, + _global_inventory_totals, enqueue_supply_replenishment, ensure_default_supply_targets, evaluate_supply_target, + evaluate_supply_targets, upsert_supply_target, ) from web.backend.services.account_work_queue import enqueue_account_work @@ -106,6 +111,115 @@ def test_supply_inventory_separates_available_and_allocatable(db): assert inventory.needs_replenishment is True +def test_global_supply_inventory_batches_value_classes_in_two_queries(db): + user = User( + username="global-supply-owner", + password_hash="h", + role="super_admin", + is_active=True, + ) + high_free = _account("global-high-free", points=400) + high_leased = _account("global-high-leased", points=400) + high_limited = _account("global-high-limited", points=400) + ordinary_free = _account("global-ordinary-free") + ordinary_collab = _account("global-ordinary-collab") + db.add_all( + [user, high_free, high_leased, high_limited, ordinary_free, ordinary_collab] + ) + db.flush() + limited_state = _state(high_limited) + limited_state.availability = "limited" + db.add_all( + [ + _state(high_free), + _state(high_leased), + limited_state, + _state(ordinary_free), + _state(ordinary_collab), + AccountLease( + account_id=high_leased.id, + holder_user_id=user.id, + scope="elite", + purpose="platform_fulfillment", + status="active", + ), + DouyuExternalOrder( + order_no="global-supply-collab", + account_id=ordinary_collab.id, + account_username=ordinary_collab.username, + status="incomplete", + ), + AccountSupplyTarget( + platform="douyu", + scope="elite", + commodity_id="", + value_class=VALUE_HIGH, + target_covered=3, + target_available=2, + target_allocatable=1, + ), + AccountSupplyTarget( + platform="douyu", + scope="elite", + commodity_id="", + value_class="ordinary", + target_covered=2, + target_available=2, + target_allocatable=1, + ), + ] + ) + db.commit() + + statements: list[str] = [] + + def capture(_conn, _cursor, statement, _parameters, _context, _executemany): + if statement.lstrip().upper().startswith("SELECT"): + statements.append(statement) + + engine = db.get_bind() + event.listen(engine, "before_cursor_execute", capture) + try: + totals = _global_inventory_totals( + db, + platform="douyu", + scope="elite", + now=NOW, + ) + finally: + event.remove(engine, "before_cursor_execute", capture) + + assert totals == { + "high_value": (3, 2, 1), + "ordinary": (2, 2, 1), + } + assert len(statements) == 2 + + inventories = { + item.value_class: item for item in evaluate_supply_targets(db, now=NOW) + } + assert ( + inventories["high_value"].covered, + inventories["high_value"].available, + inventories["high_value"].allocatable, + ) == (3, 2, 1) + assert ( + inventories["ordinary"].covered, + inventories["ordinary"].available, + inventories["ordinary"].allocatable, + ) == (2, 2, 1) + + high_target = next( + item for item in inventories.values() if item.value_class == VALUE_HIGH + ) + single = evaluate_supply_target( + db, + db.get(AccountSupplyTarget, high_target.target_id), + now=NOW, + ) + assert (single.covered, single.available, single.allocatable) == (3, 2, 1) + + def test_supply_shortage_enqueues_missing_coverage(db): account = _account("supply-missing", points=400) db.add(account) @@ -280,9 +394,10 @@ def test_account_platform_status_aggregates_queue_sync_and_supply(db): assert status["commodities"]["fresh_available"] == 1 assert status["supply_targets"][0]["covered_shortage"] == 1 assert status["supply_targets"][0]["target_available"] == 2 - assert { - alert["code"] for alert in status["alerts"] - } >= {"supply_shortage", "ck_expiring_24h"} + assert {alert["code"] for alert in status["alerts"]} >= { + "supply_shortage", + "ck_expiring_24h", + } def test_account_platform_status_builds_seven_day_trends(db): diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 6c7f3a1..e8bb125 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -42,7 +42,7 @@ TESTS_DIR = Path(__file__).resolve().parent PROJECT_ROOT = TESTS_DIR.parent VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" -HEAD_REVISION = "20261122_0113" +HEAD_REVISION = "20261122_0114" # 迁移新增、但不声明在模型里的复合查询索引。 EXTRA_INDEXES = ( diff --git a/web/backend/migrations/migration_manifest.json b/web/backend/migrations/migration_manifest.json index e44d965..970b607 100644 --- a/web/backend/migrations/migration_manifest.json +++ b/web/backend/migrations/migration_manifest.json @@ -453,6 +453,10 @@ "20261122_0113_platform_fulfillment_lease_policy.py": { "revision": "20261122_0113", "sha256": "195042a0f84e0ba8f0950e305e485ca4a96f69d9c0fd59b7dcc5a9f62b32357e" + }, + "20261122_0114_account_commodity_global_available_index.py": { + "revision": "20261122_0114", + "sha256": "3b0da21df3a48784bf5e204d3fc95fc2e40c0edded4601869c36f1d52fe07d4b" } } } diff --git a/web/backend/migrations/versions/20261122_0114_account_commodity_global_available_index.py b/web/backend/migrations/versions/20261122_0114_account_commodity_global_available_index.py new file mode 100644 index 0000000..8f705b7 --- /dev/null +++ b/web/backend/migrations/versions/20261122_0114_account_commodity_global_available_index.py @@ -0,0 +1,51 @@ +"""add global commodity availability index + +全局账号供给目标不指定 commodity_id。现有 match 索引以 commodity_id 作为第三列, +查询只按 platform/scope/availability/valid_until 时无法形成有效范围,实际扫描 +全部商品状态。新增与全局查询同形的组合索引,并用 account_id 和限时窗口字段 +覆盖统计所需的列。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261122_0114" +down_revision: str | None = "20261122_0113" +branch_labels: str | None = None +depends_on: str | None = None + +TABLE_NAME = "account_commodity_states" +INDEX_NAME = "ix_account_commodity_global_available" + + +def _indexes() -> set[str]: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table(TABLE_NAME): + return set() + return {item["name"] for item in inspector.get_indexes(TABLE_NAME)} + + +def upgrade() -> None: + if INDEX_NAME in _indexes(): + return + op.create_index( + INDEX_NAME, + TABLE_NAME, + [ + "platform", + "scope", + "availability", + "valid_until", + "account_id", + "available_from", + "available_until", + ], + unique=False, + ) + + +def downgrade() -> None: + if INDEX_NAME in _indexes(): + op.drop_index(INDEX_NAME, table_name=TABLE_NAME) diff --git a/web/backend/models.py b/web/backend/models.py index 36e5d31..312527e 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -1971,6 +1971,18 @@ class AccountCommodityState(Base): "platform", "scope", ), + # 全局供给目标不指定 commodity_id,不能使用上面的 match 索引: + # commodity_id 位于前缀中间,会让 availability/valid_until 退化为过滤列。 + Index( + "ix_account_commodity_global_available", + "platform", + "scope", + "availability", + "valid_until", + "account_id", + "available_from", + "available_until", + ), ) diff --git a/web/backend/routers/douyu.py b/web/backend/routers/douyu.py index 33e2282..be78f45 100644 --- a/web/backend/routers/douyu.py +++ b/web/backend/routers/douyu.py @@ -70,7 +70,9 @@ from ..services.account_protection_service import ( update_protection_group, ) from ..services.account_supply_service import ( + SupplyInventory, evaluate_supply_target, + evaluate_supply_targets, update_supply_target, upsert_supply_target, ) @@ -701,8 +703,10 @@ def _config_out(config: DouyuConfig) -> DouyuConfigOut: def _supply_target_out( db: Session, target: AccountSupplyTarget, + *, + inventory: SupplyInventory | None = None, ) -> DouyuSupplyTargetOut: - inventory = evaluate_supply_target(db, target) + resolved = inventory or evaluate_supply_target(db, target) return DouyuSupplyTargetOut( id=int(target.id), target_id=int(target.id), @@ -710,18 +714,18 @@ def _supply_target_out( scope=target.scope, commodity_id=target.commodity_id, value_class=target.value_class, - target_covered=inventory.target_covered, - target_available=inventory.target_available, - target_allocatable=inventory.target_allocatable, - safety_buffer=inventory.safety_buffer, + target_covered=resolved.target_covered, + target_available=resolved.target_available, + target_allocatable=resolved.target_allocatable, + safety_buffer=resolved.safety_buffer, enabled=bool(target.enabled), - covered=inventory.covered, - available=inventory.available, - allocatable=inventory.allocatable, - covered_shortage=inventory.covered_shortage, - available_shortage=inventory.available_shortage, - allocatable_shortage=inventory.allocatable_shortage, - needs_replenishment=inventory.needs_replenishment, + covered=resolved.covered, + available=resolved.available, + allocatable=resolved.allocatable, + covered_shortage=resolved.covered_shortage, + available_shortage=resolved.available_shortage, + allocatable_shortage=resolved.allocatable_shortage, + needs_replenishment=resolved.needs_replenishment, updated_by=target.updated_by, updated_at=target.updated_at, ) @@ -1056,7 +1060,15 @@ def list_supply_targets( ) .all() ) - return [_supply_target_out(db, row) for row in rows] + enabled_inventories = {item.target_id: item for item in evaluate_supply_targets(db)} + return [ + _supply_target_out( + db, + row, + inventory=enabled_inventories.get(int(row.id)), + ) + for row in rows + ] @router.post("/supply-targets", response_model=DouyuSupplyTargetOut) diff --git a/web/backend/services/account_supply_service.py b/web/backend/services/account_supply_service.py index e3c9707..09362e9 100644 --- a/web/backend/services/account_supply_service.py +++ b/web/backend/services/account_supply_service.py @@ -6,7 +6,7 @@ import os from dataclasses import dataclass from datetime import UTC, datetime, timedelta -from sqlalchemy import and_, or_, select +from sqlalchemy import and_, case, func, or_, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -96,8 +96,9 @@ def _active_lease_exists(): return ( select(AccountLease.id) .where( - AccountLease.account_id == Account.id, - AccountLease.status == "active", + # active_account_id 是生成列,只在 status='active' 时有值。 + # 直接命中唯一索引,避免再按 account_id 扫历史租约。 + AccountLease.active_account_id == Account.id, ) .exists() ) @@ -124,25 +125,29 @@ def _value_filter(value_class: str, *, now: datetime): raise ValueError(f"不支持的账号价值分类: {value_class}") +def _base_account_filter(*, now: datetime): + ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS) + return and_( + Account.cookie.isnot(None), + Account.cookie != "", + Account.ck_check_status == "valid", + Account.ck_checked_at.isnot(None), + Account.ck_checked_at >= ck_fresh_cutoff, + Account.auto_managed.is_(True), + Account.status.notin_(MANUAL_STATES), + ) + + def _base_account_query( db: Session, *, now: datetime, value_class: str, ): - ck_fresh_cutoff = now - timedelta(seconds=CK_FRESH_SECONDS) return ( db.query(Account) - .filter( - Account.cookie.isnot(None), - Account.cookie != "", - Account.ck_check_status == "valid", - Account.ck_checked_at.isnot(None), - Account.ck_checked_at >= ck_fresh_cutoff, - Account.auto_managed.is_(True), - Account.status.notin_(MANUAL_STATES), - _value_filter(value_class, now=now), - ) + .filter(_base_account_filter(now=now)) + .filter(_value_filter(value_class, now=now)) ) @@ -179,45 +184,148 @@ def _commodity_query( now: datetime, target: AccountSupplyTarget, ): - query = _base_account_query( - db, - now=now, - value_class=target.value_class, - ).join( - AccountCommodityState, - AccountCommodityState.account_id == Account.id, - ).filter( - AccountCommodityState.platform == target.platform, - AccountCommodityState.scope == target.scope, - AccountCommodityState.commodity_id == target.commodity_id, - AccountCommodityState.observed_at.isnot(None), + query = ( + _base_account_query( + db, + now=now, + value_class=target.value_class, + ) + .join( + AccountCommodityState, + AccountCommodityState.account_id == Account.id, + ) + .filter( + AccountCommodityState.platform == target.platform, + AccountCommodityState.scope == target.scope, + AccountCommodityState.commodity_id == target.commodity_id, + AccountCommodityState.observed_at.isnot(None), + ) ) return query -def _global_state_exists(target: AccountSupplyTarget): +def _global_state_exists_for(*, platform: str, scope: str): return ( select(AccountCommodityState.id) .where( AccountCommodityState.account_id == Account.id, - AccountCommodityState.platform == target.platform, - AccountCommodityState.scope == target.scope, + AccountCommodityState.platform == platform, + AccountCommodityState.scope == scope, AccountCommodityState.observed_at.isnot(None), ) .exists() ) -def _global_available_exists(target: AccountSupplyTarget, *, now: datetime): - return ( - select(AccountCommodityState.id) - .where( +def _global_inventory_totals( + db: Session, + *, + platform: str, + scope: str, + now: datetime, +) -> dict[str, tuple[int, int, int]]: + """一次算出某作用域下高价值/普通的覆盖、可兑换和可分配数量。 + + 全局目标不指定 commodity_id,因此不能用 + ``ix_account_commodity_match`` 的 commodity_id 前缀。固定拆成两条 SQL: + + 1. accounts + EXISTS:计算 covered; + 2. account_commodity_states JOIN accounts:同时计算 available/allocatable。 + + 第二条走 ``ix_account_commodity_global_available``,避免先物化全部商品状态。 + """ + value_class = case( + (is_high_value_expr(now=now), VALUE_HIGH), + else_=VALUE_ORDINARY, + ) + base_filter = _base_account_filter(now=now) + + covered_rows = ( + db.query(value_class, func.count(Account.id)) + .select_from(Account) + .filter( + base_filter, + _global_state_exists_for(platform=platform, scope=scope), + ) + .group_by(value_class) + .all() + ) + totals = { + VALUE_HIGH: [0, 0, 0], + VALUE_ORDINARY: [0, 0, 0], + } + for class_name, covered in covered_rows: + totals[str(class_name)][0] = int(covered or 0) + + allocatable_account = case( + (_allocatable_filter(now), Account.id), + else_=None, + ) + available_rows = ( + db.query( + value_class, + func.count(func.distinct(Account.id)), + func.count(func.distinct(allocatable_account)), + ) + .select_from(Account) + .join( + AccountCommodityState, AccountCommodityState.account_id == Account.id, - AccountCommodityState.platform == target.platform, - AccountCommodityState.scope == target.scope, + ) + .filter( + base_filter, + AccountCommodityState.platform == platform, + AccountCommodityState.scope == scope, _fresh_available_filter(now), ) - .exists() + .group_by(value_class) + .all() + ) + for class_name, available, allocatable in available_rows: + bucket = totals[str(class_name)] + bucket[1] = int(available or 0) + bucket[2] = int(allocatable or 0) + return { + class_name: (values[0], values[1], values[2]) + for class_name, values in totals.items() + } + + +def _inventory_from_counts( + target: AccountSupplyTarget, + *, + covered: int, + available: int, + allocatable: int, +) -> SupplyInventory: + return SupplyInventory( + target_id=int(target.id), + scope=target.scope, + commodity_id=target.commodity_id, + value_class=target.value_class, + covered=covered, + available=available, + allocatable=allocatable, + target_covered=int(target.target_covered or 0), + target_available=int(target.target_available or 0), + target_allocatable=int(target.target_allocatable or 0), + safety_buffer=int(target.safety_buffer or 0), + covered_shortage=max( + 0, + int(target.target_covered or 0) + int(target.safety_buffer or 0) - covered, + ), + available_shortage=max( + 0, + int(target.target_available or 0) + + int(target.safety_buffer or 0) + - available, + ), + allocatable_shortage=max( + 0, + int(target.target_allocatable or 0) + + int(target.safety_buffer or 0) + - allocatable, + ), ) @@ -237,51 +345,21 @@ def evaluate_supply_target( covered = base.count() available_query = base.filter(_fresh_available_filter(moment)) available = available_query.count() - allocatable = available_query.filter( - _allocatable_filter(moment) - ).count() + allocatable = available_query.filter(_allocatable_filter(moment)).count() else: - base = _base_account_query( + totals = _global_inventory_totals( db, + platform=target.platform, + scope=target.scope, now=moment, - value_class=target.value_class, ) - covered = base.filter(_global_state_exists(target)).count() - available_query = base.filter( - _global_available_exists(target, now=moment) - ) - available = available_query.count() - allocatable = available_query.filter( - _allocatable_filter(moment) - ).count() + covered, available, allocatable = totals[target.value_class] - return SupplyInventory( - target_id=int(target.id), - scope=target.scope, - commodity_id=target.commodity_id, - value_class=target.value_class, + return _inventory_from_counts( + target, covered=covered, available=available, allocatable=allocatable, - target_covered=int(target.target_covered or 0), - target_available=int(target.target_available or 0), - target_allocatable=int(target.target_allocatable or 0), - safety_buffer=int(target.safety_buffer or 0), - covered_shortage=max( - 0, - int(target.target_covered or 0) + int(target.safety_buffer or 0) - - covered, - ), - available_shortage=max( - 0, - int(target.target_available or 0) + int(target.safety_buffer or 0) - - available, - ), - allocatable_shortage=max( - 0, - int(target.target_allocatable or 0) + int(target.safety_buffer or 0) - - allocatable, - ), ) @@ -302,10 +380,30 @@ def evaluate_supply_targets( ) .all() ) - return [ - evaluate_supply_target(db, target, now=moment) - for target in targets - ] + global_totals: dict[tuple[str, str], dict[str, tuple[int, int, int]]] = {} + inventories: list[SupplyInventory] = [] + for target in targets: + if target.commodity_id: + inventories.append(evaluate_supply_target(db, target, now=moment)) + continue + key = (target.platform, target.scope) + if key not in global_totals: + global_totals[key] = _global_inventory_totals( + db, + platform=target.platform, + scope=target.scope, + now=moment, + ) + covered, available, allocatable = global_totals[key][target.value_class] + inventories.append( + _inventory_from_counts( + target, + covered=covered, + available=available, + allocatable=allocatable, + ) + ) + return inventories def upsert_supply_target( @@ -502,11 +600,15 @@ def enqueue_supply_replenishment( inventory.allocatable_shortage, ) limit = max(1, min(int(max_per_target), missing)) - accounts = _sync_candidate_query( - db, - target=target, - now=moment, - ).limit(limit).all() + accounts = ( + _sync_candidate_query( + db, + target=target, + now=moment, + ) + .limit(limit) + .all() + ) for account in accounts: enqueue_account_work( db,