- 新增每日数据保留清理任务:按保留期清空原始报文、删除过期 事件/审计记录,批量幂等可重跑;067 迁移补齐清理索引并清理 冗余索引,check-sql-guard 增加防线 - 生产 CORS 改为显式来源白名单(CORS_ALLOWED_ORIGINS),禁用 通配符,非通配响应补充 Vary: Origin - 文件上传任一步失败时回滚已上传对象,清理失败记录 warn 日志 - 新增 HTTP/CORS 与 PostgreSQL 集成测试;若干文件仅 prettier 重排
102 lines
3.0 KiB
TypeScript
102 lines
3.0 KiB
TypeScript
import { query } from '../../db/client.js'
|
|
import { runtimeConfig } from '../../config/runtime.js'
|
|
|
|
export type DataRetentionResult = {
|
|
status: 'ok'
|
|
message: string
|
|
rawPayloadRows: number
|
|
taskEventRows: number
|
|
webhookEventRows: number
|
|
auditLogRows: number
|
|
matchLogRows: number
|
|
}
|
|
|
|
/** 清理敏感原始报文并删除已过保留期的事件/审计记录。 */
|
|
export async function runDataRetentionJob(): Promise<DataRetentionResult> {
|
|
const retention = runtimeConfig.retention || {}
|
|
const rawPayloadDays = positiveDays(retention.rawPayloadDays, 180)
|
|
const taskEventDays = positiveDays(retention.taskEventDays, 180)
|
|
const webhookEventDays = positiveDays(retention.webhookEventDays, 180)
|
|
const auditLogDays = positiveDays(retention.auditLogDays, 365)
|
|
|
|
const rawPayloadRows =
|
|
(await scrubJsonColumnInBatches('orders', 'updated_at', rawPayloadDays)) +
|
|
(await scrubJsonColumnInBatches('kuaishou_industry_vouchers', 'updated_at', rawPayloadDays))
|
|
const matchLogRows = await scrubJsonColumnInBatches(
|
|
'work_product_match_logs',
|
|
'created_at',
|
|
rawPayloadDays,
|
|
)
|
|
const taskEventRows = await deleteOldRowsInBatches('task_events', taskEventDays)
|
|
const webhookEventRows = await deleteOldRowsInBatches('webhook_events', webhookEventDays)
|
|
const auditLogRows = await deleteOldRowsInBatches('admin_audit_logs', auditLogDays)
|
|
|
|
return {
|
|
status: 'ok',
|
|
message: '数据保留清理完成',
|
|
rawPayloadRows,
|
|
matchLogRows,
|
|
taskEventRows,
|
|
webhookEventRows,
|
|
auditLogRows,
|
|
}
|
|
}
|
|
|
|
async function scrubJsonColumnInBatches(
|
|
table: 'orders' | 'kuaishou_industry_vouchers' | 'work_product_match_logs',
|
|
dateColumn: 'created_at' | 'updated_at',
|
|
retentionDays: number,
|
|
) {
|
|
let total = 0
|
|
while (true) {
|
|
const result = await query(
|
|
`
|
|
WITH candidates AS (
|
|
SELECT ctid
|
|
FROM ${table}
|
|
WHERE ${dateColumn} < NOW() - make_interval(days => $1::int)
|
|
AND raw_payload_json <> '{}'::jsonb
|
|
LIMIT 1000
|
|
)
|
|
UPDATE ${table} AS target
|
|
SET raw_payload_json = '{}'::jsonb
|
|
FROM candidates
|
|
WHERE target.ctid = candidates.ctid
|
|
`,
|
|
[retentionDays],
|
|
)
|
|
total += result.rowCount || 0
|
|
if ((result.rowCount || 0) < 1000) return total
|
|
}
|
|
}
|
|
|
|
async function deleteOldRowsInBatches(
|
|
table: 'task_events' | 'webhook_events' | 'admin_audit_logs',
|
|
retentionDays: number,
|
|
) {
|
|
let total = 0
|
|
while (true) {
|
|
const result = await query(
|
|
`
|
|
WITH candidates AS (
|
|
SELECT ctid
|
|
FROM ${table}
|
|
WHERE created_at < NOW() - make_interval(days => $1::int)
|
|
LIMIT 1000
|
|
)
|
|
DELETE FROM ${table} AS target
|
|
USING candidates
|
|
WHERE target.ctid = candidates.ctid
|
|
`,
|
|
[retentionDays],
|
|
)
|
|
total += result.rowCount || 0
|
|
if ((result.rowCount || 0) < 1000) return total
|
|
}
|
|
}
|
|
|
|
function positiveDays(value: unknown, fallback: number) {
|
|
const days = Number(value)
|
|
return Number.isInteger(days) && days > 0 ? days : fallback
|
|
}
|