perf(pipeline): 订单项管道改用水位游标并只扫缺失

- 新增 watermark-store:统一读写游标/日水位/失败原因
- 渠道判定、商品归一、成本快照从上次游标继续,到表尾后回绕重试
- 成本快照只处理没有有效快照的订单项,避免每 5 分钟全量重扫
This commit is contained in:
yml2213
2026-10-31 20:17:33 +08:00
parent ce775ca102
commit d979ef50d6
6 changed files with 134 additions and 52 deletions
@@ -2,6 +2,7 @@ import { query } from '../../db/client.js'
import { createHttpError } from '../../utils/http.js'
import { nowIso } from '../../utils/time.js'
import { upsertAggregatedException } from '../cost-center/cost-exception-service.js'
import { readWatermarkItemId } from '../watermark-store.js'
import {
FULFILLMENT_CHANNELS,
MATCH_KEY_TYPES,
@@ -138,6 +139,8 @@ export type AssignChannelsResult = {
conflicted: number
stillPending: number
lastId: number
/** 下一轮起点;到达表尾时回绕到 0,保证漏判的订单还能被重新评估。 */
nextCursor: number
byChannel: Record<string, number>
}
@@ -147,7 +150,11 @@ export async function assignPendingOrderItemChannels(
): Promise<AssignChannelsResult> {
const batchSize = Math.min(5000, Math.max(1, Math.trunc(options.batchSize || 500)))
const maxBatches = Math.min(200, Math.max(1, Math.trunc(options.maxBatches || 20)))
let cursor = Math.max(0, Math.trunc(options.afterId || 0))
let cursor =
options.afterId != null
? Math.max(0, Math.trunc(options.afterId))
: await readWatermarkItemId('channel_assign')
let reachedEnd = false
const rules = await loadEnabledRules()
const result: AssignChannelsResult = {
@@ -156,6 +163,7 @@ export async function assignPendingOrderItemChannels(
conflicted: 0,
stillPending: 0,
lastId: cursor,
nextCursor: cursor,
byChannel: {},
}
@@ -188,7 +196,11 @@ export async function assignPendingOrderItemChannels(
[cursor, batchSize],
)
if (rows.rows.length === 0) break
if (rows.rows.length === 0) {
reachedEnd = true
break
}
if (rows.rows.length < batchSize) reachedEnd = true
result.scanned += rows.rows.length
cursor = Number(rows.rows[rows.rows.length - 1]!.id)
result.lastId = cursor
@@ -245,6 +257,7 @@ export async function assignPendingOrderItemChannels(
}
}
result.nextCursor = reachedEnd ? 0 : cursor
return result
}