perf(pipeline): 订单项管道改用水位游标并只扫缺失

- 新增 watermark-store:统一读写游标/日水位/失败原因
- 渠道判定、商品归一、成本快照从上次游标继续,到表尾后回绕重试
- 成本快照只处理没有有效快照的订单项,避免每 5 分钟全量重扫
This commit is contained in:
yml2213
2026-10-31 20:17:33 +08:00
parent ce775ca102
commit d979ef50d6
6 changed files with 134 additions and 52 deletions
@@ -39,10 +39,12 @@ export async function runChannelAssignJob(job: JsonObject = {}) {
const result = await assignPendingOrderItemChannels({
batchSize: clampInt(config.batchSize, 500, 1, 5000),
maxBatches: clampInt(config.maxBatches, 20, 1, 200),
afterId: clampInt(config.afterId, 0, 0, Number.MAX_SAFE_INTEGER),
...(config.afterId != null
? { afterId: clampInt(config.afterId, 0, 0, Number.MAX_SAFE_INTEGER) }
: {}),
})
await touchWatermark('channel_assign', { lastItemId: result.lastId })
await touchWatermark('channel_assign', { lastItemId: result.nextCursor })
return {
status: 'ok',
message:
@@ -188,7 +190,7 @@ export async function runProductNormalizeJob(job: JsonObject = {}) {
maxBatches: clampInt(config.maxBatches, 20, 1, 200),
})
await touchWatermark('product_normalize', { lastItemId: result.lastId })
await touchWatermark('product_normalize', { lastItemId: result.nextCursor })
return {
status: 'ok',
message: `商品归一:扫描 ${result.scanned},命中 ${result.matched},未命中 ${result.unmatched}`,
@@ -203,7 +205,7 @@ export async function runCostSnapshotJob(job: JsonObject = {}) {
maxBatches: clampInt(config.maxBatches, 20, 1, 200),
})
await touchWatermark('cost_snapshot', { lastItemId: result.lastId })
await touchWatermark('cost_snapshot', { lastItemId: result.nextCursor })
return {
status: 'ok',
message: `成本快照:扫描 ${result.scanned},生成 ${result.created},缺失 ${result.missing}`,
@@ -1,6 +1,7 @@
import { query } from '../../db/client.js'
import { nowIso } from '../../utils/time.js'
import { getScheduledJobRuntimeStates } from './scheduler-service.js'
import { recordPipelineError, touchWatermark } from '../watermark-store.js'
import type { WatermarkPipeline } from '../watermark-store.js'
/**
* 财务管道可观测性:水位 + 定时任务运行状态。
@@ -22,49 +23,12 @@ export const FINANCE_PIPELINES = [
export type FinancePipelineName = (typeof FINANCE_PIPELINES)[number]['name']
export { recordPipelineError, touchWatermark }
export function isFinancePipelineName(value: unknown): value is FinancePipelineName {
return FINANCE_PIPELINES.some((item) => item.name === value)
}
/** 记录一次成功运行的水位。 */
export async function touchWatermark(
pipeline: FinancePipelineName,
patch: { lastItemId?: number; lastStatDate?: string | null } = {},
): Promise<void> {
const now = nowIso()
await query(
`
INSERT INTO finance_watermarks (pipeline, last_item_id, last_stat_date, last_success_at, updated_at)
VALUES ($1, $2, $3::date, $4, $4)
ON CONFLICT (pipeline) DO UPDATE SET
last_item_id = GREATEST(finance_watermarks.last_item_id, EXCLUDED.last_item_id),
last_stat_date = COALESCE(EXCLUDED.last_stat_date, finance_watermarks.last_stat_date),
last_success_at = EXCLUDED.last_success_at,
last_error = '',
updated_at = EXCLUDED.updated_at
`,
[pipeline, Math.max(0, Math.trunc(patch.lastItemId || 0)), patch.lastStatDate ?? null, now],
)
}
/** 记录一次失败原因,不改水位。 */
export async function recordPipelineError(
pipeline: FinancePipelineName,
error: unknown,
): Promise<void> {
const message = error instanceof Error ? error.message : String(error)
await query(
`
INSERT INTO finance_watermarks (pipeline, last_error, updated_at)
VALUES ($1, $2, $3)
ON CONFLICT (pipeline) DO UPDATE SET
last_error = EXCLUDED.last_error,
updated_at = EXCLUDED.updated_at
`,
[pipeline, message.slice(0, 500), nowIso()],
)
}
export type FinancePipelineStatusRow = {
pipeline: string
label: string