feat(cost-center): 成本异常 60 天归档
cost_exceptions 只增不减:本地快照 260,253 行(resolved 151,939 / open 108,314), 已处理的行只是审计痕迹,留在热表会拖慢「状态 + 时间」分页,并让唯一索引持续膨胀。 - 迁移 108:新增 cost_exceptions_archive(保留原 id 与时间字段,加 archived_at), 热表补部分索引 idx_cost_exceptions_archivable (COALESCE(resolved_at, created_at)) WHERE status <> 'open',让每日任务不扫全表; - cost-exception-archive-service:只归档 status <> 'open' 且超期的行,每批 1000 行, DELETE ... RETURNING + INSERT 在同一语句内完成,单批原子,不存在「删了没归档」; - data-retention 每日任务接入;保留期走 COST_EXCEPTION_RETENTION_DAYS(默认 60); - 新增 npm run cost-exceptions:archive(--dry-run / --days / --batch-size)。 为什么 open 行不归档:open 是待办队列,超期不等于已解决,归档等于把问题藏起来。 验证: - 3 条 resolved 回拨到 90 天前 → 归档 3 条,热表 resolved 151,939 → 151,936; - 1 条 open 回拨到 90 天前 → 仍在热表,未被归档; - 归档表保留原 id 与时间,archived_at 记录归档时刻; - 恢复测试数据后计数回到 151,939 / 108,314。
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* 成本异常归档:把超过保留期的「已处理」异常搬进 cost_exceptions_archive。
|
||||
*
|
||||
* 口径:
|
||||
* - 只归档 status <> 'open' 的行;open 是待办队列,超期也必须留着;
|
||||
* - 时钟取 COALESCE(resolved_at, created_at):ignored 行没有 resolved_at 时按创建时间;
|
||||
* - 每批 1000 行,DELETE ... RETURNING + INSERT 在同一语句内完成,单批原子,
|
||||
* 不存在「删了但没归档」的窗口;
|
||||
* - 归档保留 id 与原时间字段,可按原口径追溯。
|
||||
*
|
||||
* 调用方:data-retention 定时任务(每日)、scripts/archive-cost-exceptions.ts(手动)。
|
||||
*/
|
||||
import { query } from '../../db/client.js'
|
||||
|
||||
export type ArchiveCostExceptionsResult = {
|
||||
archivedRows: number
|
||||
remainingArchivableRows: number
|
||||
retentionDays: number
|
||||
}
|
||||
|
||||
export async function countArchivableCostExceptions(retentionDays: number) {
|
||||
const { rows } = await query<{ count: number }>(
|
||||
`
|
||||
SELECT COUNT(*)::int AS count
|
||||
FROM cost_exceptions
|
||||
WHERE status <> 'open'
|
||||
AND COALESCE(resolved_at, created_at) < NOW() - make_interval(days => $1::int)
|
||||
`,
|
||||
[retentionDays],
|
||||
)
|
||||
return rows[0]?.count ?? 0
|
||||
}
|
||||
|
||||
export async function archiveResolvedCostExceptions({
|
||||
retentionDays,
|
||||
batchSize = 1000,
|
||||
maxBatches = 10_000,
|
||||
}: {
|
||||
retentionDays: number
|
||||
batchSize?: number
|
||||
maxBatches?: number
|
||||
}): Promise<ArchiveCostExceptionsResult> {
|
||||
const archiveExists = await query<{ exists: boolean }>(
|
||||
`SELECT to_regclass('public.cost_exceptions_archive') IS NOT NULL AS exists`,
|
||||
)
|
||||
if (!archiveExists.rows[0]?.exists) {
|
||||
return { archivedRows: 0, remainingArchivableRows: 0, retentionDays }
|
||||
}
|
||||
|
||||
let archivedRows = 0
|
||||
let batches = 0
|
||||
while (batches < maxBatches) {
|
||||
const result = await query(
|
||||
`
|
||||
WITH moved AS (
|
||||
DELETE FROM cost_exceptions
|
||||
WHERE id IN (
|
||||
SELECT id
|
||||
FROM cost_exceptions
|
||||
WHERE status <> 'open'
|
||||
AND COALESCE(resolved_at, created_at) < NOW() - make_interval(days => $1::int)
|
||||
ORDER BY id
|
||||
LIMIT $2
|
||||
)
|
||||
RETURNING id, subject_type, subject_id, order_item_id, work_order_id,
|
||||
reason, status, detail_json, created_at, resolved_at,
|
||||
resolved_by_admin_user_id, resolution_note
|
||||
)
|
||||
INSERT INTO cost_exceptions_archive (
|
||||
id, subject_type, subject_id, order_item_id, work_order_id,
|
||||
reason, status, detail_json, created_at, resolved_at,
|
||||
resolved_by_admin_user_id, resolution_note, archived_at
|
||||
)
|
||||
SELECT id, subject_type, subject_id, order_item_id, work_order_id,
|
||||
reason, status, detail_json, created_at, resolved_at,
|
||||
resolved_by_admin_user_id, resolution_note, NOW()
|
||||
FROM moved
|
||||
`,
|
||||
[retentionDays, batchSize],
|
||||
)
|
||||
const movedRows = result.rowCount || 0
|
||||
archivedRows += movedRows
|
||||
batches += 1
|
||||
if (movedRows < batchSize) break
|
||||
}
|
||||
|
||||
const remainingArchivableRows = await countArchivableCostExceptions(retentionDays)
|
||||
return { archivedRows, remainingArchivableRows, retentionDays }
|
||||
}
|
||||
@@ -23,6 +23,30 @@ test('保留清理必须覆盖 work_order_events,否则这张只增不减的
|
||||
)
|
||||
})
|
||||
|
||||
test('成本异常只归档已处理的行,open 队列不能被藏起来', async () => {
|
||||
const job = await readJobSource()
|
||||
assert.ok(
|
||||
job.includes('archiveResolvedCostExceptions'),
|
||||
'成本异常必须走归档,不能直接 DELETE 掉审计痕迹',
|
||||
)
|
||||
assert.ok(job.includes('retention.costExceptionDays'), '保留天数必须走配置项')
|
||||
|
||||
// 归档 SQL 在独立服务里,检查服务源码而不是任务源码。
|
||||
const service = await readFile(
|
||||
new URL('../cost-center/cost-exception-archive-service.ts', import.meta.url),
|
||||
'utf8',
|
||||
)
|
||||
assert.ok(service.includes("status <> 'open'"), 'open 行是待办队列,不能因超期被归档')
|
||||
assert.ok(
|
||||
service.includes('COALESCE(resolved_at, created_at)'),
|
||||
'归档时钟优先用 resolved_at,ignored 行没有 resolved_at 时回退 created_at',
|
||||
)
|
||||
assert.ok(
|
||||
service.includes('INSERT INTO cost_exceptions_archive'),
|
||||
'归档必须落到 cost_exceptions_archive,保持可追溯',
|
||||
)
|
||||
})
|
||||
|
||||
test('保留任务要回填漏写的付款派生列,并在有命中时打日志', async () => {
|
||||
const src = await readJobSource()
|
||||
assert.ok(
|
||||
@@ -48,6 +72,7 @@ test('保留清理默认窗口:敏感报文/事件 180 天,审计 365 天,
|
||||
'webhookEventDays: 180',
|
||||
'auditLogDays: 365',
|
||||
'workOrderEventDays: 180',
|
||||
'costExceptionDays: 60',
|
||||
]) {
|
||||
assert.ok(defaults.includes(line), `默认保留策略缺少 ${line}`)
|
||||
}
|
||||
@@ -62,4 +87,10 @@ test('保留清理默认窗口:敏感报文/事件 180 天,审计 365 天,
|
||||
),
|
||||
'WORK_ORDER_EVENT_RETENTION_DAYS 需要环境变量出口,否则生产无法调整',
|
||||
)
|
||||
assert.ok(
|
||||
overrides.includes(
|
||||
"integerEnv('COST_EXCEPTION_RETENTION_DAYS', ['retention', 'costExceptionDays'])",
|
||||
),
|
||||
'COST_EXCEPTION_RETENTION_DAYS 需要环境变量出口,否则生产无法调整归档窗口',
|
||||
)
|
||||
})
|
||||
|
||||
@@ -2,6 +2,7 @@ import { query } from '../../db/client.js'
|
||||
import { runtimeConfig } from '../../config/runtime.js'
|
||||
import { logWarn } from '../../utils/logger.js'
|
||||
import { getWorkerProductMatchConfig } from '../worker-platform/worker-product-match-config-service.js'
|
||||
import { archiveResolvedCostExceptions } from '../cost-center/cost-exception-archive-service.js'
|
||||
import { pruneWorkProductMatchLogs } from '../../repositories/worker-platform/product-match-repo.js'
|
||||
|
||||
export type DataRetentionResult = {
|
||||
@@ -15,6 +16,7 @@ export type DataRetentionResult = {
|
||||
matchLogRows: number
|
||||
matchLogDeletedRows: number
|
||||
sourceOrderAmountBackfillRows: number
|
||||
costExceptionArchivedRows: number
|
||||
}
|
||||
|
||||
/** 清理敏感原始报文并删除已过保留期的事件/审计记录。 */
|
||||
@@ -26,6 +28,8 @@ export async function runDataRetentionJob(): Promise<DataRetentionResult> {
|
||||
const auditLogDays = positiveDays(retention.auditLogDays, 365)
|
||||
// 工单事件表以前完全没纳入保留:717k 行 / 269MB 且只增不减,会一直拖慢历史单检索。
|
||||
const workOrderEventDays = positiveDays(retention.workOrderEventDays, 180)
|
||||
// 成本异常只归档「已处理」的行;open 行是待办队列,不能因为超期就藏起来。
|
||||
const costExceptionDays = positiveDays(retention.costExceptionDays, 60)
|
||||
|
||||
// 1. 按配置清理超出上限或过期的打手商品匹配日志
|
||||
const matchConfig = getWorkerProductMatchConfig()
|
||||
@@ -49,6 +53,10 @@ export async function runDataRetentionJob(): Promise<DataRetentionResult> {
|
||||
const workOrderEventRows = await deleteOldRowsInBatches('work_order_events', workOrderEventDays)
|
||||
// 统计热路径已不再读 material_json。若将来有第三个写入点忘了维护派生列,
|
||||
// 金额会静默回退 orders.total_amount。这里把漏写补上并打日志,让口径变化可见。
|
||||
const costExceptionArchive = await archiveResolvedCostExceptions({
|
||||
retentionDays: costExceptionDays,
|
||||
})
|
||||
const costExceptionArchivedRows = costExceptionArchive.archivedRows
|
||||
const sourceOrderAmountBackfillRows = await backfillMissingSourceOrderAmount()
|
||||
if (sourceOrderAmountBackfillRows > 0) {
|
||||
logWarn('[data-retention]', 'source_order_amount_fen 出现漏写,已按 JSON 回填', {
|
||||
@@ -67,6 +75,7 @@ export async function runDataRetentionJob(): Promise<DataRetentionResult> {
|
||||
auditLogRows,
|
||||
workOrderEventRows,
|
||||
sourceOrderAmountBackfillRows,
|
||||
costExceptionArchivedRows,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user