perf(db): 收入口径回填改为分批脚本
迁移 104 里对 order_items 的全表 UPDATE 实测 34.4 秒（375,447 行）， 远超默认语句超时 15s，单事务迁移会拖住启动甚至直接失败。 - 104 只保留 DDL 与索引/约束，删除全表 UPDATE； - 新增 scripts/backfill-revenue-basis.ts：按 id 分批、幂等、支持 --dry-run / --batch-size / --max-batches（kebab 与 camel 两种写法）； - package.json 增加 revenue:backfill； - migrate.ts 启动时做一次带索引计数，仍有 unknown 时打印告警并给出命令， 不阻塞启动； - 文档补充回填步骤与「不执行的后果」。 验证： - 实测 37.5 万行全表更新 34.4s → 脚本按批执行，单批 5000 行； - 功能验证：单条订单项置 unknown → 脚本回填 → 金额与 basis 恢复正确； - 启动告警验证：保留 1 条 unknown 重启 → 日志出现告警；回填后计数归零； - 全新库从 102 迁移到 107 仍成功（2s）。
This commit is contained in:
yml2213
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* 历史订单项收入口径回填（迁移 104 只建列，不再做全表更新）。
|
||||
*
|
||||
* 背景：375,447 行的全表 UPDATE 实测约 34 秒，超过默认语句超时，放在单事务迁移里
|
||||
* 会拖住启动；因此迁到本脚本，按 id 分批执行。
|
||||
*
|
||||
* 口径（与 order-item-repo 写入规则一致）：
|
||||
* item_count = 1 → revenue = 订单实付，revenue_basis = 'order_total'
|
||||
* item_count > 1 → 等额拆分，余数给最后一项，revenue_basis = 'equal_split'
|
||||
*
|
||||
* 幂等：只处理 revenue_basis = 'unknown' 的行，重复执行不会二次拆分。
|
||||
*
|
||||
* 用法：
|
||||
* npm run revenue:backfill # 默认每批 5000 行
|
||||
* npm run revenue:backfill -- --batch-size=2000 --max-batches=100
|
||||
* npm run revenue:backfill -- --dry-run # 只统计并抽样预览，不写库
|
||||
*/
|
||||
import process from 'node:process'
|
||||
|
||||
import { closeDb, query } from '../src/db/client.js'
|
||||
|
||||
const options = parseOptions(process.argv.slice(2))
|
||||
const batchSize = normalizeInteger(options.batchSize, 5000, 1, 50_000)
|
||||
const maxBatches = normalizeInteger(options.maxBatches, 1000, 1, 10_000)
|
||||
const dryRun = options.dryRun === 'true'
|
||||
|
||||
const BATCH_SQL = `
|
||||
WITH target AS (
|
||||
SELECT id, order_id
|
||||
FROM order_items
|
||||
WHERE revenue_basis = 'unknown'
|
||||
ORDER BY id
|
||||
LIMIT $1
|
||||
),
|
||||
item_counts AS (
|
||||
SELECT oi.order_id, COUNT(*)::int AS item_count
|
||||
FROM order_items oi
|
||||
WHERE oi.order_id IN (SELECT order_id FROM target)
|
||||
GROUP BY oi.order_id
|
||||
),
|
||||
ranked AS (
|
||||
SELECT
|
||||
oi.id,
|
||||
GREATEST(COALESCE(o.total_amount, 0), 0)::bigint AS order_revenue_fen,
|
||||
ic.item_count,
|
||||
ROW_NUMBER() OVER (PARTITION BY oi.order_id ORDER BY oi.id ASC) AS item_rank
|
||||
FROM order_items oi
|
||||
JOIN orders o ON o.id = oi.order_id
|
||||
JOIN item_counts ic ON ic.order_id = oi.order_id
|
||||
WHERE oi.id IN (SELECT id FROM target)
|
||||
)
|
||||
UPDATE order_items oi
|
||||
SET
|
||||
revenue_amount_fen = CASE
|
||||
WHEN r.item_count = 1 THEN r.order_revenue_fen
|
||||
ELSE (r.order_revenue_fen / r.item_count)
|
||||
+ CASE WHEN r.item_rank = r.item_count THEN (r.order_revenue_fen % r.item_count) ELSE 0 END
|
||||
END,
|
||||
revenue_basis = CASE WHEN r.item_count = 1 THEN 'order_total' ELSE 'equal_split' END,
|
||||
updated_at = NOW()
|
||||
FROM ranked r
|
||||
WHERE oi.id = r.id
|
||||
AND oi.revenue_basis = 'unknown'
|
||||
RETURNING oi.id
|
||||
`
|
||||
|
||||
async function main() {
|
||||
await assertColumnExists()
|
||||
|
||||
const before = await countUnknown()
|
||||
const startedAt = Date.now()
|
||||
|
||||
if (dryRun) {
|
||||
const samples = await previewSamples()
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
dryRun: true,
|
||||
pendingCount: before,
|
||||
batchSize,
|
||||
estimatedBatches: Math.ceil(before / batchSize),
|
||||
samples,
|
||||
durationMs: Date.now() - startedAt,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
let batches = 0
|
||||
let updated = 0
|
||||
while (batches < maxBatches) {
|
||||
const result = await query(BATCH_SQL, [batchSize])
|
||||
const affected = result.rowCount ?? 0
|
||||
if (affected === 0) break
|
||||
batches += 1
|
||||
updated += affected
|
||||
if (affected < batchSize) break
|
||||
}
|
||||
|
||||
const remaining = await countUnknown()
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
dryRun: false,
|
||||
batchSize,
|
||||
maxBatches,
|
||||
batches,
|
||||
updatedCount: updated,
|
||||
pendingBefore: before,
|
||||
remainingCount: remaining,
|
||||
durationMs: Date.now() - startedAt,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
)
|
||||
|
||||
if (remaining > 0) {
|
||||
console.error(`仍有 ${remaining} 个订单项未回填；请增大 --max-batches 后重跑（幂等）。`)
|
||||
process.exitCode = 2
|
||||
}
|
||||
}
|
||||
|
||||
async function assertColumnExists() {
|
||||
const { rows } = await query<{ exists: boolean }>(
|
||||
`
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'order_items'
|
||||
AND column_name = 'revenue_basis'
|
||||
) AS exists
|
||||
`,
|
||||
)
|
||||
if (!rows[0]?.exists) {
|
||||
throw new Error('order_items.revenue_basis 不存在，请先执行 npm run db:migrate（迁移 104）')
|
||||
}
|
||||
}
|
||||
|
||||
async function countUnknown() {
|
||||
const { rows } = await query<{ count: number }>(
|
||||
`SELECT COUNT(*)::int AS count FROM order_items WHERE revenue_basis = 'unknown'`,
|
||||
)
|
||||
return rows[0]?.count ?? 0
|
||||
}
|
||||
|
||||
async function previewSamples() {
|
||||
const { rows } = await query<{
|
||||
order_id: number
|
||||
item_count: number
|
||||
order_total: number
|
||||
item_ids: number[]
|
||||
}>(
|
||||
`
|
||||
SELECT
|
||||
oi.order_id,
|
||||
COUNT(*)::int AS item_count,
|
||||
MAX(GREATEST(COALESCE(o.total_amount, 0), 0))::bigint AS order_total,
|
||||
array_agg(oi.id ORDER BY oi.id) AS item_ids
|
||||
FROM order_items oi
|
||||
JOIN orders o ON o.id = oi.order_id
|
||||
WHERE oi.revenue_basis = 'unknown'
|
||||
GROUP BY oi.order_id
|
||||
ORDER BY oi.order_id
|
||||
LIMIT 3
|
||||
`,
|
||||
)
|
||||
return rows
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
.finally(async () => {
|
||||
await closeDb()
|
||||
})
|
||||
|
||||
function parseOptions(args: string[]) {
|
||||
const result: Record<string, string> = {}
|
||||
for (const arg of args) {
|
||||
if (!arg.startsWith('--')) continue
|
||||
const body = arg.slice(2)
|
||||
const index = body.indexOf('=')
|
||||
const key = index === -1 ? body : body.slice(0, index)
|
||||
const value = index === -1 ? 'true' : body.slice(index + 1)
|
||||
// 同时接受 --batch-size 与 --batchSize 两种写法。
|
||||
result[key] = value
|
||||
result[toCamelCase(key)] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
function toCamelCase(value: string) {
|
||||
return value.replace(/-([a-z])/g, (_, char: string) => char.toUpperCase())
|
||||
}
|
||||
|
||||
function normalizeInteger(value: unknown, fallback: number, min: number, max: number) {
|
||||
const parsed = Number(value)
|
||||
if (!Number.isInteger(parsed)) return fallback
|
||||
return Math.min(max, Math.max(min, parsed))
|
||||
}
|
||||
Reference in New Issue
Block a user