perf(pipeline): 订单项管道改用水位游标并只扫缺失
- 新增 watermark-store:统一读写游标/日水位/失败原因 - 渠道判定、商品归一、成本快照从上次游标继续,到表尾后回绕重试 - 成本快照只处理没有有效快照的订单项,避免每 5 分钟全量重扫
This commit is contained in:
@@ -39,10 +39,12 @@ export async function runChannelAssignJob(job: JsonObject = {}) {
|
||||
const result = await assignPendingOrderItemChannels({
|
||||
batchSize: clampInt(config.batchSize, 500, 1, 5000),
|
||||
maxBatches: clampInt(config.maxBatches, 20, 1, 200),
|
||||
afterId: clampInt(config.afterId, 0, 0, Number.MAX_SAFE_INTEGER),
|
||||
...(config.afterId != null
|
||||
? { afterId: clampInt(config.afterId, 0, 0, Number.MAX_SAFE_INTEGER) }
|
||||
: {}),
|
||||
})
|
||||
|
||||
await touchWatermark('channel_assign', { lastItemId: result.lastId })
|
||||
await touchWatermark('channel_assign', { lastItemId: result.nextCursor })
|
||||
return {
|
||||
status: 'ok',
|
||||
message:
|
||||
@@ -188,7 +190,7 @@ export async function runProductNormalizeJob(job: JsonObject = {}) {
|
||||
maxBatches: clampInt(config.maxBatches, 20, 1, 200),
|
||||
})
|
||||
|
||||
await touchWatermark('product_normalize', { lastItemId: result.lastId })
|
||||
await touchWatermark('product_normalize', { lastItemId: result.nextCursor })
|
||||
return {
|
||||
status: 'ok',
|
||||
message: `商品归一:扫描 ${result.scanned},命中 ${result.matched},未命中 ${result.unmatched}`,
|
||||
@@ -203,7 +205,7 @@ export async function runCostSnapshotJob(job: JsonObject = {}) {
|
||||
maxBatches: clampInt(config.maxBatches, 20, 1, 200),
|
||||
})
|
||||
|
||||
await touchWatermark('cost_snapshot', { lastItemId: result.lastId })
|
||||
await touchWatermark('cost_snapshot', { lastItemId: result.nextCursor })
|
||||
return {
|
||||
status: 'ok',
|
||||
message: `成本快照:扫描 ${result.scanned},生成 ${result.created},缺失 ${result.missing}`,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { query } from '../../db/client.js'
|
||||
import { nowIso } from '../../utils/time.js'
|
||||
import { getScheduledJobRuntimeStates } from './scheduler-service.js'
|
||||
import { recordPipelineError, touchWatermark } from '../watermark-store.js'
|
||||
import type { WatermarkPipeline } from '../watermark-store.js'
|
||||
|
||||
/**
|
||||
* 财务管道可观测性:水位 + 定时任务运行状态。
|
||||
@@ -22,49 +23,12 @@ export const FINANCE_PIPELINES = [
|
||||
|
||||
export type FinancePipelineName = (typeof FINANCE_PIPELINES)[number]['name']
|
||||
|
||||
export { recordPipelineError, touchWatermark }
|
||||
|
||||
export function isFinancePipelineName(value: unknown): value is FinancePipelineName {
|
||||
return FINANCE_PIPELINES.some((item) => item.name === value)
|
||||
}
|
||||
|
||||
/** 记录一次成功运行的水位。 */
|
||||
export async function touchWatermark(
|
||||
pipeline: FinancePipelineName,
|
||||
patch: { lastItemId?: number; lastStatDate?: string | null } = {},
|
||||
): Promise<void> {
|
||||
const now = nowIso()
|
||||
await query(
|
||||
`
|
||||
INSERT INTO finance_watermarks (pipeline, last_item_id, last_stat_date, last_success_at, updated_at)
|
||||
VALUES ($1, $2, $3::date, $4, $4)
|
||||
ON CONFLICT (pipeline) DO UPDATE SET
|
||||
last_item_id = GREATEST(finance_watermarks.last_item_id, EXCLUDED.last_item_id),
|
||||
last_stat_date = COALESCE(EXCLUDED.last_stat_date, finance_watermarks.last_stat_date),
|
||||
last_success_at = EXCLUDED.last_success_at,
|
||||
last_error = '',
|
||||
updated_at = EXCLUDED.updated_at
|
||||
`,
|
||||
[pipeline, Math.max(0, Math.trunc(patch.lastItemId || 0)), patch.lastStatDate ?? null, now],
|
||||
)
|
||||
}
|
||||
|
||||
/** 记录一次失败原因,不改水位。 */
|
||||
export async function recordPipelineError(
|
||||
pipeline: FinancePipelineName,
|
||||
error: unknown,
|
||||
): Promise<void> {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
await query(
|
||||
`
|
||||
INSERT INTO finance_watermarks (pipeline, last_error, updated_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (pipeline) DO UPDATE SET
|
||||
last_error = EXCLUDED.last_error,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
`,
|
||||
[pipeline, message.slice(0, 500), nowIso()],
|
||||
)
|
||||
}
|
||||
|
||||
export type FinancePipelineStatusRow = {
|
||||
pipeline: string
|
||||
label: string
|
||||
|
||||
Reference in New Issue
Block a user