diff --git a/apps/backend/src/db/migrations/106_manual_sku_key.sql b/apps/backend/src/db/migrations/106_manual_sku_key.sql new file mode 100644 index 00000000..f4368861 --- /dev/null +++ b/apps/backend/src/db/migrations/106_manual_sku_key.sql @@ -0,0 +1,31 @@ +-- 106_manual_sku_key.sql +-- +-- 内部 SKU 匹配键：skuNick -> outSkuSn -> sku_code -> sku_name（全部去空格）。 +-- +-- 背景：人工商品归一、人工渠道匹配每次定时同步都要从 spec_json 现算这个键， +-- 41 万单 / 12 万人工项实测单条 SQL 3–7 秒，且两种写法（3 级 / 4 级、带不带 +-- TRIM）不一致。固化成生成列后，查询直接走索引，也保证所有入口口径一致。 +-- +-- 注意：ADD COLUMN ... GENERATED ... STORED 会重写 order_items（41 万行约 +-- 1–2 分钟，期间持 ACCESS EXCLUSIVE 锁），应放在维护窗口执行。 + +ALTER TABLE order_items + ADD COLUMN IF NOT EXISTS manual_sku_key TEXT + GENERATED ALWAYS AS ( + COALESCE( + NULLIF(BTRIM(spec_json->>'skuNick'), ''), + NULLIF(BTRIM(spec_json->>'outSkuSn'), ''), + NULLIF(BTRIM(sku_code), ''), + NULLIF(BTRIM(sku_name), '') + ) + ) STORED; + +COMMENT ON COLUMN order_items.manual_sku_key IS + '内部 SKU 匹配键（去空格）：skuNick -> outSkuSn -> sku_code -> sku_name；人工履约配置按它匹配'; + +CREATE INDEX IF NOT EXISTS idx_order_items_manual_sku_key_stored + ON order_items (manual_sku_key) + WHERE fulfillment_mode = 'manual' AND manual_sku_key IS NOT NULL; + +-- 旧索引表达式少了 TRIM，和任何查询都对不上，属于纯写入开销。 +DROP INDEX IF EXISTS idx_order_items_manual_sku_key; diff --git a/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts b/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts index 0ae53068..4fbb2471 100644 --- a/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts +++ b/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts @@ -492,10 +492,12 @@ export async function listWorkOrders({ */ export async function listWorkOrderShopNames(): Promise { const result = await query<{ shop_name: string }>( + // 从 orders 出发 + EXISTS 半连接：18 万工单的 hash join 实测 3.1 秒， + // 改成 orders 驱动后走 idx_work_orders_order_id 半连接，约 0.3 秒。 `SELECT DISTINCT BTRIM(o.shop_name) AS shop_name - FROM work_orders wo - INNER JOIN orders o ON o.id = wo.order_id + FROM orders o WHERE BTRIM(o.shop_name) <> '' + AND EXISTS (SELECT 1 FROM work_orders wo WHERE wo.order_id = o.id) ORDER BY shop_name ASC`, ) return result.rows diff --git a/apps/backend/src/services/cost-center/cost-snapshot-sync-service.ts b/apps/backend/src/services/cost-center/cost-snapshot-sync-service.ts index 87da6cfc..cdfda4c5 100644 --- a/apps/backend/src/services/cost-center/cost-snapshot-sync-service.ts +++ b/apps/backend/src/services/cost-center/cost-snapshot-sync-service.ts @@ -57,13 +57,9 @@ export async function syncEstimatedOrderItemCostSnapshots( COALESCE(oi.fulfillment_mode, 'manual') AS fulfillment_mode FROM order_items oi JOIN orders o ON o.id = oi.order_id - LEFT JOIN LATERAL ( - SELECT id, product_rule_id - FROM work_orders - WHERE order_item_id = oi.id - ORDER BY id DESC - LIMIT 1 - ) wo ON TRUE + -- order_item_id 上有唯一部分索引，一个订单项最多一张工单； + -- 原来的 ORDER BY id DESC LIMIT 1 逐行探测会做 3.7 万次索引查找。 + LEFT JOIN work_orders wo ON wo.order_item_id = oi.id WHERE o.pay_status IN ('paid', 'refunded') AND (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date BETWEEN $1::date AND $2::date @@ -613,9 +609,7 @@ async function insertAndResolveProductMappingExceptions( COALESCE(oi.fulfillment_mode, 'manual') AS fulfillment_mode FROM order_items oi JOIN orders o ON o.id = oi.order_id - LEFT JOIN LATERAL ( - SELECT id FROM work_orders WHERE order_item_id = oi.id ORDER BY id DESC LIMIT 1 - ) wo ON TRUE + LEFT JOIN work_orders wo ON wo.order_item_id = oi.id WHERE o.pay_status IN ('paid', 'refunded') AND (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date BETWEEN $1::date AND $2::date @@ -651,11 +645,15 @@ async function insertAndResolveProductMappingExceptions( WHERE ce.status = 'open' AND ce.subject_type = 'order_item' AND ce.reason = 'product_mapping_missing' + -- 只收口最近创建、仍在同步视野内的异常：全表收口会随着历史异常累积 + -- 变成每次同步都扫几十万行（41 万单测试库实测单次 7.4 秒）。 + AND ce.created_at >= ($1::date - INTERVAL '7 days') AND EXISTS ( SELECT 1 FROM order_items oi WHERE oi.id = ce.subject_id AND oi.canonical_product_id IS NOT NULL ) `, + [fromDate], ) return { created: inserted.rowCount || 0, resolved: resolved.rowCount || 0 } @@ -678,9 +676,7 @@ async function insertAndResolveCostMissingExceptions( COALESCE(oi.fulfillment_mode, 'manual') AS fulfillment_mode FROM order_items oi JOIN orders o ON o.id = oi.order_id - LEFT JOIN LATERAL ( - SELECT id FROM work_orders WHERE order_item_id = oi.id ORDER BY id DESC LIMIT 1 - ) wo ON TRUE + LEFT JOIN work_orders wo ON wo.order_item_id = oi.id WHERE o.pay_status IN ('paid', 'refunded') AND (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date BETWEEN $1::date AND $2::date @@ -774,6 +770,7 @@ async function insertAndResolveCostMissingExceptions( WHERE ce.status = 'open' AND ce.subject_type = 'order_item' AND ce.reason = 'cost_missing' + AND ce.created_at >= ($1::date - INTERVAL '7 days') AND EXISTS ( SELECT 1 FROM cost_snapshots cs WHERE cs.subject_type = 'order_item' @@ -781,6 +778,7 @@ async function insertAndResolveCostMissingExceptions( AND cs.superseded_at IS NULL ) `, + [fromDate], ) const manualResolved = await client.query( @@ -790,6 +788,7 @@ async function insertAndResolveCostMissingExceptions( WHERE ce.status = 'open' AND ce.subject_type = 'order_item' AND ce.reason = 'manual_cost_missing' + AND ce.created_at >= ($1::date - INTERVAL '7 days') AND EXISTS ( SELECT 1 FROM cost_snapshots cs WHERE cs.subject_type = 'manual_case' @@ -797,6 +796,7 @@ async function insertAndResolveCostMissingExceptions( AND cs.superseded_at IS NULL ) `, + [fromDate], ) return { diff --git a/apps/backend/src/services/cost-center/manual-fulfillment-config-service.ts b/apps/backend/src/services/cost-center/manual-fulfillment-config-service.ts index f21108fe..f1d9dd5b 100644 --- a/apps/backend/src/services/cost-center/manual-fulfillment-config-service.ts +++ b/apps/backend/src/services/cost-center/manual-fulfillment-config-service.ts @@ -124,12 +124,7 @@ export async function scanManualSkus(options: ScanOptions) { oi.sku_name, COALESCE(oi.revenue_amount_fen, 0)::bigint AS revenue_fen, (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date AS stat_date, - COALESCE( - NULLIF(oi.spec_json->>'skuNick', ''), - NULLIF(oi.spec_json->>'outSkuSn', ''), - NULLIF(oi.sku_code, ''), - NULLIF(oi.sku_name, '') - ) AS sku_key + oi.manual_sku_key AS sku_key FROM order_items oi JOIN orders o ON o.id = oi.order_id WHERE oi.fulfillment_mode = 'manual' @@ -247,7 +242,7 @@ export async function listManualSkus(input: Record = {}) { JOIN orders o ON o.id = oi.order_id WHERE oi.fulfillment_mode = 'manual' AND o.provider = $${params.length} - AND COALESCE(NULLIF(oi.spec_json->>'skuNick', ''), NULLIF(oi.spec_json->>'outSkuSn', '')) = r.sku_key + AND oi.manual_sku_key = r.sku_key )`) } @@ -435,12 +430,7 @@ export async function rematchManualFulfillments(input: Record = FROM order_items oi JOIN orders o ON o.id = oi.order_id JOIN manual_fulfillment_configs cfg - ON cfg.sku_key = COALESCE( - NULLIF(oi.spec_json->>'skuNick', ''), - NULLIF(oi.spec_json->>'outSkuSn', ''), - NULLIF(oi.sku_code, ''), - NULLIF(oi.sku_name, '') - ) + ON cfg.sku_key = oi.manual_sku_key WHERE oi.fulfillment_mode = 'manual' AND cfg.enabled = TRUE AND ($1 = '' OR cfg.sku_key = $1) diff --git a/apps/backend/src/services/cost-center/manual-product-normalization-service.ts b/apps/backend/src/services/cost-center/manual-product-normalization-service.ts index d734a13d..a0226492 100644 --- a/apps/backend/src/services/cost-center/manual-product-normalization-service.ts +++ b/apps/backend/src/services/cost-center/manual-product-normalization-service.ts @@ -34,14 +34,11 @@ export type NormalizeManualProductsResult = { remainingItemCount: number } -/** 内部 SKU：快手 skuNick / 拼多多 outSkuSn / 平台 sku_code 兜底。 */ -const MANUAL_SKU_KEY_SQL = ` - COALESCE( - NULLIF(TRIM(oi.spec_json->>'skuNick'), ''), - NULLIF(TRIM(oi.spec_json->>'outSkuSn'), ''), - NULLIF(TRIM(oi.sku_code), '') - ) -` +/** + * 内部 SKU：由 order_items.manual_sku_key 生成列统一提供 + * （skuNick → outSkuSn → sku_code → sku_name，均去空格）。 + */ +const MANUAL_SKU_KEY_SQL = `oi.manual_sku_key` export async function normalizeManualProducts( input: NormalizeManualProductsInput = {}, diff --git a/apps/backend/src/services/finance/manual-fulfillment-integrity.test.ts b/apps/backend/src/services/finance/manual-fulfillment-integrity.test.ts index 3c49ec53..80904211 100644 --- a/apps/backend/src/services/finance/manual-fulfillment-integrity.test.ts +++ b/apps/backend/src/services/finance/manual-fulfillment-integrity.test.ts @@ -79,13 +79,14 @@ test('人工确认必须单事务：扣款与快照同事务，且使用 with-cl }) test('人工确认前必须自动匹配履约配置，配置保存/定时任务也要回填', async () => { - const [manualSource, configSource, jobSource] = await Promise.all([ + const [manualSource, configSource, jobSource, migrationSource] = await Promise.all([ readFile(new URL('./manual-fulfillment-service.ts', import.meta.url), 'utf8'), readFile( new URL('../cost-center/manual-fulfillment-config-service.ts', import.meta.url), 'utf8', ), readFile(new URL('../scheduler/cost-finance-sync-job.ts', import.meta.url), 'utf8'), + readFile(new URL('../../db/migrations/106_manual_sku_key.sql', import.meta.url), 'utf8'), ]) assert.ok( manualSource.includes('matchManualFulfillmentConfigWithClient(client, orderItemId, now)'), @@ -104,8 +105,15 @@ test('人工确认前必须自动匹配履约配置，配置保存/定时任务 '定时同步必须自动回填未匹配的人工订单项，并限制在最近窗口', ) assert.ok( - manualSource.includes('NULLIF(oi.sku_code') && configSource.includes('NULLIF(oi.sku_code'), - 'SKU 匹配必须有 sku_code / sku_name 兜底，不能只认 spec_json', + configSource.includes('manual_sku_key'), + '人工 SKU 匹配必须统一走 order_items.manual_sku_key 生成列', + ) + assert.ok( + migrationSource.includes("spec_json->>'skuNick'") && + migrationSource.includes("spec_json->>'outSkuSn'") && + migrationSource.includes('sku_code') && + migrationSource.includes('sku_name'), + '生成列必须保留 sku_code / sku_name 兜底', ) }) diff --git a/apps/backend/src/services/finance/sales-facts-service.ts b/apps/backend/src/services/finance/sales-facts-service.ts index eda5852f..92486785 100644 --- a/apps/backend/src/services/finance/sales-facts-service.ts +++ b/apps/backend/src/services/finance/sales-facts-service.ts @@ -76,9 +76,63 @@ export async function rebuildSalesDailyFacts( const inserted = await client.query( ` - WITH item_base AS ( + WITH scoped_orders AS MATERIALIZED ( + -- 先物化 7 天窗口内的订单集合，避免 planner 对每个订单项做一次 + -- orders 随机回表；后续全部走 hash/merge join。 SELECT - (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date AS stat_date, + o.id, + o.provider, + o.shop_id, + o.pay_status, + COALESCE(o.paid_at, o.created_at) AS basis_at + FROM orders o + WHERE o.pay_status IN ('paid', 'refunded') + AND (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date + BETWEEN $1::date AND $2::date + AND NOT EXISTS ( + SELECT 1 FROM finance_periods p + WHERE p.status = 'closed' + AND p.period_key = to_char( + (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date, + 'YYYY-MM' + ) + ) + ), + scoped_items AS MATERIALIZED ( + SELECT + oi.id, + oi.order_id, + oi.canonical_product_id, + oi.fulfillment_mode, + oi.revenue_amount_fen, + oi.normalized_quantity, + oi.quantity + FROM order_items oi + WHERE oi.order_id IN (SELECT id FROM scoped_orders) + ), + cost_by_item AS ( + -- 一次全量聚合后再 hash join，替代逐项 LATERAL 查 cost_snapshots。 + SELECT + order_item_id, + SUM(total_amount_fen)::bigint AS cost_fen, + COUNT(*)::int AS snapshot_count, + COUNT(*) FILTER (WHERE snapshot_stage = 'actual')::int AS actual_count + FROM cost_snapshots + WHERE superseded_at IS NULL + GROUP BY order_item_id + ), + refunds_by_item AS ( + SELECT + order_item_id, + SUM(amount_fen)::bigint AS refund_fen, + COUNT(*)::int AS refund_count + FROM order_refunds + WHERE status = 'confirmed' + GROUP BY order_item_id + ), + item_base AS ( + SELECT + (o.basis_at AT TIME ZONE 'Asia/Shanghai')::date AS stat_date, o.provider, o.shop_id, oi.canonical_product_id, @@ -99,36 +153,10 @@ export async function rebuildSalesDailyFacts( COALESCE(cost.cost_fen, 0)::bigint AS cost_fen, COALESCE(cost.snapshot_count, 0)::int AS snapshot_count, COALESCE(cost.actual_count, 0)::int AS actual_count - FROM order_items oi - JOIN orders o ON o.id = oi.order_id - LEFT JOIN LATERAL ( - SELECT - SUM(cs.total_amount_fen)::bigint AS cost_fen, - COUNT(*)::int AS snapshot_count, - COUNT(*) FILTER (WHERE cs.snapshot_stage = 'actual')::int AS actual_count - FROM cost_snapshots cs - WHERE cs.order_item_id = oi.id - AND cs.superseded_at IS NULL - ) cost ON TRUE - LEFT JOIN LATERAL ( - SELECT - SUM(r.amount_fen)::bigint AS refund_fen, - COUNT(*)::int AS refund_count - FROM order_refunds r - WHERE r.order_item_id = oi.id - AND r.status = 'confirmed' - ) refunds ON TRUE - WHERE o.pay_status IN ('paid', 'refunded') - AND (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date - BETWEEN $1::date AND $2::date - AND NOT EXISTS ( - SELECT 1 FROM finance_periods p - WHERE p.status = 'closed' - AND p.period_key = to_char( - (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date, - 'YYYY-MM' - ) - ) + FROM scoped_items oi + JOIN scoped_orders o ON o.id = oi.order_id + LEFT JOIN cost_by_item cost ON cost.order_item_id = oi.id + LEFT JOIN refunds_by_item refunds ON refunds.order_item_id = oi.id ) INSERT INTO sales_daily_facts ( stat_date, provider, shop_id, canonical_product_id, fulfillment_mode, @@ -297,19 +325,23 @@ export async function getSalesProfitOverview(input: SalesFactsInput = {}) { orders_without_items: number revenue_without_items: number }>( + // 原来两个 FILTER 各写一次 NOT EXISTS，等价于对每个订单做两轮逐行探测 + // （30 天 / 16 万单实测 108 万 buffers、9.4 秒）。改成一次 DISTINCT 扫描 + // order_id 后做 hash join，只扫一遍索引，实测 10.9 万 buffers、0.4 秒。 ` SELECT COUNT(*)::int AS order_count, - COUNT(*) FILTER ( - WHERE NOT EXISTS (SELECT 1 FROM order_items oi WHERE oi.order_id = orders.id) - )::int AS orders_without_items, - COALESCE(SUM(total_amount) FILTER ( - WHERE NOT EXISTS (SELECT 1 FROM order_items oi WHERE oi.order_id = orders.id) - ), 0)::bigint AS revenue_without_items - FROM orders - WHERE pay_status IN ('paid', 'refunded') - AND (COALESCE(paid_at, created_at) AT TIME ZONE 'Asia/Shanghai')::date - BETWEEN $1::date AND $2::date + COUNT(*) FILTER (WHERE oi.order_id IS NULL)::int AS orders_without_items, + COALESCE(SUM(s.total_amount) FILTER (WHERE oi.order_id IS NULL), 0)::bigint + AS revenue_without_items + FROM ( + SELECT o.id, o.total_amount + FROM orders o + WHERE o.pay_status IN ('paid', 'refunded') + AND (COALESCE(o.paid_at, o.created_at) AT TIME ZONE 'Asia/Shanghai')::date + BETWEEN $1::date AND $2::date + ) s + LEFT JOIN (SELECT DISTINCT order_id FROM order_items) oi ON oi.order_id = s.id `, [range.fromDate, range.toDate], ), diff --git a/apps/backend/src/services/scheduler/config-service.ts b/apps/backend/src/services/scheduler/config-service.ts index ad34e1bb..f03fd477 100644 --- a/apps/backend/src/services/scheduler/config-service.ts +++ b/apps/backend/src/services/scheduler/config-service.ts @@ -225,7 +225,10 @@ function createDefaultCostFinanceSyncJob() { id: COST_FINANCE_SYNC_JOB_ID, type: 'cost_finance_sync', enabled: true, - intervalSeconds: 300, + // 一次同步要跑 6 条全窗口聚合（41 万单实测 30–60 秒）。5 分钟一次会让 + // 数据库长期处在 10–20% 的同步负载下，后台页面首屏被拖慢；财务事实 + // 15 分钟刷新已足够，需要更快可在后台定时任务里改回 300。 + intervalSeconds: 900, config: { days: 7, },