feat(ops): 数据保留清理、CORS 白名单与上传失败回滚
- 新增每日数据保留清理任务:按保留期清空原始报文、删除过期 事件/审计记录,批量幂等可重跑;067 迁移补齐清理索引并清理 冗余索引,check-sql-guard 增加防线 - 生产 CORS 改为显式来源白名单(CORS_ALLOWED_ORIGINS),禁用 通配符,非通配响应补充 Vary: Origin - 文件上传任一步失败时回滚已上传对象,清理失败记录 warn 日志 - 新增 HTTP/CORS 与 PostgreSQL 集成测试;若干文件仅 prettier 重排
This commit is contained in:
@@ -92,3 +92,16 @@ test('normalizeScheduledJobsConfig 支持多账号去重并保留 0 阈值', ()
|
||||
assert.equal(config.jobs[0].intervalSeconds, 60)
|
||||
assert.equal(config.jobs[0].cooldownSeconds, 60)
|
||||
})
|
||||
|
||||
test('normalizeScheduledJobsConfig 自动补齐数据保留清理任务', () => {
|
||||
const config = normalizeScheduledJobsConfig({ enabled: true, jobs: [] })
|
||||
const retentionJob = config.jobs.find((job) => job.type === 'data_retention')
|
||||
|
||||
assert.deepEqual(retentionJob, {
|
||||
id: 'data-retention',
|
||||
type: 'data_retention',
|
||||
enabled: true,
|
||||
intervalSeconds: 86400,
|
||||
config: {},
|
||||
})
|
||||
})
|
||||
|
||||
@@ -9,6 +9,7 @@ const SCHEDULED_JOBS_FILE_PATH = path.join(PROJECT_ROOT, 'data', 'scheduled-jobs
|
||||
const CLOUDTENTACLES_HEALTH_JOB_ID = 'cloudtentacles-health'
|
||||
const WORK_ORDER_TIMEOUT_JOB_ID = 'work-order-timeout'
|
||||
const DEPOSIT_UNFREEZE_JOB_ID = 'deposit-unfreeze'
|
||||
const DATA_RETENTION_JOB_ID = 'data-retention'
|
||||
|
||||
export function getScheduledJobsFilePath() {
|
||||
return SCHEDULED_JOBS_FILE_PATH
|
||||
@@ -73,6 +74,7 @@ export function normalizeScheduledJobsConfig(rawValue: unknown) {
|
||||
const hasCloudtentaclesHealth = jobs.some((item) => item.id === CLOUDTENTACLES_HEALTH_JOB_ID)
|
||||
const hasWorkOrderTimeout = jobs.some((item) => item.id === WORK_ORDER_TIMEOUT_JOB_ID)
|
||||
const hasDepositUnfreeze = jobs.some((item) => item.id === DEPOSIT_UNFREEZE_JOB_ID)
|
||||
const hasDataRetention = jobs.some((item) => item.id === DATA_RETENTION_JOB_ID)
|
||||
|
||||
if (!hasCloudtentaclesHealth) {
|
||||
jobs.push(createDefaultCloudtentaclesHealthJob())
|
||||
@@ -83,6 +85,9 @@ export function normalizeScheduledJobsConfig(rawValue: unknown) {
|
||||
if (!hasDepositUnfreeze) {
|
||||
jobs.push(createDefaultDepositUnfreezeJob())
|
||||
}
|
||||
if (!hasDataRetention) {
|
||||
jobs.push(createDefaultDataRetentionJob())
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: typeof source.enabled === 'boolean' ? source.enabled : true,
|
||||
@@ -105,6 +110,9 @@ function normalizeScheduledJob(rawValue: unknown) {
|
||||
if (type === 'deposit_unfreeze') {
|
||||
return normalizeDepositUnfreezeJob(rawValue)
|
||||
}
|
||||
if (type === 'data_retention') {
|
||||
return normalizeDataRetentionJob(rawValue)
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
@@ -123,6 +131,16 @@ function normalizeDepositUnfreezeJob(rawValue: JsonObject) {
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeDataRetentionJob(rawValue: JsonObject) {
|
||||
return {
|
||||
id: DATA_RETENTION_JOB_ID,
|
||||
type: 'data_retention',
|
||||
enabled: rawValue.enabled !== false,
|
||||
intervalSeconds: normalizeRangeInteger(rawValue.intervalSeconds, 86400, 3600, 604800),
|
||||
config: {},
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeWorkOrderTimeoutJob(rawValue: JsonObject) {
|
||||
const config = isPlainObject(rawValue.config) ? rawValue.config : {}
|
||||
|
||||
@@ -211,10 +229,21 @@ function createDefaultScheduledJobsConfig() {
|
||||
createDefaultCloudtentaclesHealthJob(),
|
||||
createDefaultWorkOrderTimeoutJob(),
|
||||
createDefaultDepositUnfreezeJob(),
|
||||
createDefaultDataRetentionJob(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
function createDefaultDataRetentionJob() {
|
||||
return {
|
||||
id: DATA_RETENTION_JOB_ID,
|
||||
type: 'data_retention',
|
||||
enabled: true,
|
||||
intervalSeconds: 86400,
|
||||
config: {},
|
||||
}
|
||||
}
|
||||
|
||||
function createDefaultDepositUnfreezeJob() {
|
||||
return {
|
||||
id: DEPOSIT_UNFREEZE_JOB_ID,
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { query } from '../../db/client.js'
|
||||
import { runtimeConfig } from '../../config/runtime.js'
|
||||
|
||||
export type DataRetentionResult = {
|
||||
status: 'ok'
|
||||
message: string
|
||||
rawPayloadRows: number
|
||||
taskEventRows: number
|
||||
webhookEventRows: number
|
||||
auditLogRows: number
|
||||
matchLogRows: number
|
||||
}
|
||||
|
||||
/** 清理敏感原始报文并删除已过保留期的事件/审计记录。 */
|
||||
export async function runDataRetentionJob(): Promise<DataRetentionResult> {
|
||||
const retention = runtimeConfig.retention || {}
|
||||
const rawPayloadDays = positiveDays(retention.rawPayloadDays, 180)
|
||||
const taskEventDays = positiveDays(retention.taskEventDays, 180)
|
||||
const webhookEventDays = positiveDays(retention.webhookEventDays, 180)
|
||||
const auditLogDays = positiveDays(retention.auditLogDays, 365)
|
||||
|
||||
const rawPayloadRows =
|
||||
(await scrubJsonColumnInBatches('orders', 'updated_at', rawPayloadDays)) +
|
||||
(await scrubJsonColumnInBatches('kuaishou_industry_vouchers', 'updated_at', rawPayloadDays))
|
||||
const matchLogRows = await scrubJsonColumnInBatches(
|
||||
'work_product_match_logs',
|
||||
'created_at',
|
||||
rawPayloadDays,
|
||||
)
|
||||
const taskEventRows = await deleteOldRowsInBatches('task_events', taskEventDays)
|
||||
const webhookEventRows = await deleteOldRowsInBatches('webhook_events', webhookEventDays)
|
||||
const auditLogRows = await deleteOldRowsInBatches('admin_audit_logs', auditLogDays)
|
||||
|
||||
return {
|
||||
status: 'ok',
|
||||
message: '数据保留清理完成',
|
||||
rawPayloadRows,
|
||||
matchLogRows,
|
||||
taskEventRows,
|
||||
webhookEventRows,
|
||||
auditLogRows,
|
||||
}
|
||||
}
|
||||
|
||||
async function scrubJsonColumnInBatches(
|
||||
table: 'orders' | 'kuaishou_industry_vouchers' | 'work_product_match_logs',
|
||||
dateColumn: 'created_at' | 'updated_at',
|
||||
retentionDays: number,
|
||||
) {
|
||||
let total = 0
|
||||
while (true) {
|
||||
const result = await query(
|
||||
`
|
||||
WITH candidates AS (
|
||||
SELECT ctid
|
||||
FROM ${table}
|
||||
WHERE ${dateColumn} < NOW() - make_interval(days => $1::int)
|
||||
AND raw_payload_json <> '{}'::jsonb
|
||||
LIMIT 1000
|
||||
)
|
||||
UPDATE ${table} AS target
|
||||
SET raw_payload_json = '{}'::jsonb
|
||||
FROM candidates
|
||||
WHERE target.ctid = candidates.ctid
|
||||
`,
|
||||
[retentionDays],
|
||||
)
|
||||
total += result.rowCount || 0
|
||||
if ((result.rowCount || 0) < 1000) return total
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteOldRowsInBatches(
|
||||
table: 'task_events' | 'webhook_events' | 'admin_audit_logs',
|
||||
retentionDays: number,
|
||||
) {
|
||||
let total = 0
|
||||
while (true) {
|
||||
const result = await query(
|
||||
`
|
||||
WITH candidates AS (
|
||||
SELECT ctid
|
||||
FROM ${table}
|
||||
WHERE created_at < NOW() - make_interval(days => $1::int)
|
||||
LIMIT 1000
|
||||
)
|
||||
DELETE FROM ${table} AS target
|
||||
USING candidates
|
||||
WHERE target.ctid = candidates.ctid
|
||||
`,
|
||||
[retentionDays],
|
||||
)
|
||||
total += result.rowCount || 0
|
||||
if ((result.rowCount || 0) < 1000) return total
|
||||
}
|
||||
}
|
||||
|
||||
function positiveDays(value: unknown, fallback: number) {
|
||||
const days = Number(value)
|
||||
return Number.isInteger(days) && days > 0 ? days : fallback
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { getScheduledJobById, getScheduledJobsConfig } from './config-service.js
|
||||
import { runCloudtentaclesHealthJob } from './cloudtentacles-health-job.js'
|
||||
import { runDepositUnfreezeJob } from './deposit-unfreeze-job.js'
|
||||
import { runWorkOrderTimeoutJob } from './work-order-timeout-job.js'
|
||||
import { runDataRetentionJob } from './data-retention-job.js'
|
||||
|
||||
const timers = new Map<string, NodeJS.Timeout>()
|
||||
const jobStates = new Map<string, JsonObject>()
|
||||
@@ -150,6 +151,15 @@ async function runJob(job: JsonObject, { manual = false }: { manual?: boolean }
|
||||
jobId,
|
||||
type: job.type,
|
||||
status: String(summary.status || 'ok'),
|
||||
...(job.type === 'data_retention'
|
||||
? {
|
||||
rawPayloadRows: Number(summary.rawPayloadRows || 0),
|
||||
matchLogRows: Number(summary.matchLogRows || 0),
|
||||
taskEventRows: Number(summary.taskEventRows || 0),
|
||||
webhookEventRows: Number(summary.webhookEventRows || 0),
|
||||
auditLogRows: Number(summary.auditLogRows || 0),
|
||||
}
|
||||
: {}),
|
||||
})
|
||||
return result
|
||||
} catch (error) {
|
||||
@@ -184,6 +194,9 @@ function dispatchJob(job: JsonObject) {
|
||||
if (job.type === 'deposit_unfreeze') {
|
||||
return runDepositUnfreezeJob(job)
|
||||
}
|
||||
if (job.type === 'data_retention') {
|
||||
return runDataRetentionJob()
|
||||
}
|
||||
|
||||
throw createHttpError(`不支持的定时任务类型:${job.type}`, {
|
||||
statusCode: 400,
|
||||
|
||||
Reference in New Issue
Block a user