dachui/order_site
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(worker-platform): 支持后台配置匹配日志保留上限与天数并提供一键清理历史多余记录

Browse Source 
1. 扩展商品匹配配置支持 maxMatchLogs (条数上限) 与 matchLogRetentionDays (保留天数)；
2. 新增分批物理清理逻辑 pruneWorkProductMatchLogs，通过索引高效分批删除超出限制的记录；
3. 集成定时任务数据保留清理 data-retention-job，并在保存配置与写入时自动滚动清理；
4. 匹配诊断日志面板新增可视化保留策略设置栏，支持管理员自定义上限、一键保存并提供一键清理超出记录按钮。
This commit is contained in:
yml2213
2026-09-12 16:01:26 +08:00
parent e7b51f09a6
commit 1217da7393
8 changed files with 321 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/src/repositories/worker-platform/product-match-repo.ts
+94
View File
@@ -293,3 +293,97 @@ export async function listWorkProductMatchLogs({
)
return { items: result.rows, total: Number(totalResult.rows[0]?.total || 0) }
}
export type PruneWorkProductMatchLogsOptions = {
maxRows?: number
retentionDays?: number
batchSize?: number
}
export type PruneWorkProductMatchLogsResult = {
deletedByAge: number
deletedByCount: number
totalDeleted: number
remainingTotal: number
}
/** 按条数上限或保留天数分批清理历史匹配日志 */
export async function pruneWorkProductMatchLogs(
options: PruneWorkProductMatchLogsOptions = {},
): Promise<PruneWorkProductMatchLogsResult> {
let deletedByAge = 0
let deletedByCount = 0
const batchSize = Math.min(5000, Math.max(100, options.batchSize || 1000))
// 1. 按保留天数清理过期日志
if (options.retentionDays && options.retentionDays > 0) {
while (true) {
const res = await query(
`
WITH candidates AS (
SELECT ctid
FROM work_product_match_logs
WHERE created_at < NOW() - make_interval(days => $1::int)
LIMIT $2
)
DELETE FROM work_product_match_logs AS target
USING candidates
WHERE target.ctid = candidates.ctid
`,
[options.retentionDays, batchSize],
)
const count = res.rowCount || 0
deletedByAge += count
if (count < batchSize) break
}
}
// 2. 按最大条数清理超出上限的旧日志（保留最新的 maxRows 条）
if (options.maxRows && options.maxRows > 0) {
const cutoffRes = await query<{ id: string }>(
`
SELECT id
FROM work_product_match_logs
ORDER BY id DESC
OFFSET $1
LIMIT 1
`,
[options.maxRows],
)
const firstRow = cutoffRes.rows[0]
if (firstRow) {
const cutoffId = Number(firstRow.id)
while (true) {
const res = await query(
`
WITH candidates AS (
SELECT ctid
FROM work_product_match_logs
WHERE id <= $1
LIMIT $2
)
DELETE FROM work_product_match_logs AS target
USING candidates
WHERE target.ctid = candidates.ctid
`,
[cutoffId, batchSize],
)
const count = res.rowCount || 0
deletedByCount += count
if (count < batchSize) break
}
}
}
const remainingRes = await query<{ total: number }>(
"SELECT COUNT(*)::int AS total FROM work_product_match_logs",
)
const remainingTotal = Number(remainingRes.rows[0]?.total || 0)
return {
deletedByAge,
deletedByCount,
totalDeleted: deletedByAge + deletedByCount,
remainingTotal,
}
}
apps/backend/src/routes/admin/worker-platform/config.ts
+17
View File
@@ -14,6 +14,7 @@ import {
listAdminKuaishouMatchSources,
listAdminWorkCategories,
listAdminWorkProductMatchLogs,
pruneAdminWorkProductMatchLogs,
listAdminWorkProductRuleMappings,
listAdminWorkProductRules,
listAdminWorkerLevels,
@@ -341,6 +342,22 @@ router.get(
}),
)
router.post(
'/worker-platform/product-match-logs/cleanup',
requireAdminRoles(['admin', 'operator']),
createJsonHandler((req) => pruneAdminWorkProductMatchLogs(req.body || {}), {
successMessage: '匹配日志清理完成',
errorMessage: '清理匹配日志失败',
scope: '[admin/worker-platform/product-match-logs/cleanup]',
audit: (_req, data) => ({
action: 'worker_product_match_logs_cleaned',
targetType: 'worker_product_match_logs',
targetId: 'cleanup',
data: data && typeof data === 'object' ? (data as Record<string, unknown>) : {},
}),
}),
)
router.get(
'/worker-platform/finance-config',
requireAdminRoles(['admin', 'operator', 'support', 'finance']),
apps/backend/src/services/scheduler/data-retention-job.ts
+24 -13
View File
@@ -1,14 +1,17 @@
import { query } from '../../db/client.js'
import { runtimeConfig } from '../../config/runtime.js'
import { query } from "../../db/client.js"
import { runtimeConfig } from "../../config/runtime.js"
import { getWorkerProductMatchConfig } from "../worker-platform/worker-product-match-config-service.js"
import { pruneWorkProductMatchLogs } from "../../repositories/worker-platform/product-match-repo.js"
export type DataRetentionResult = {
status: 'ok'
status: "ok"
message: string
rawPayloadRows: number
taskEventRows: number
webhookEventRows: number
auditLogRows: number
matchLogRows: number
matchLogDeletedRows: number
}
/** 清理敏感原始报文并删除已过保留期的事件/审计记录。 */
@@ -19,29 +22,37 @@ export async function runDataRetentionJob(): Promise<DataRetentionResult> {
const webhookEventDays = positiveDays(retention.webhookEventDays, 180)
const auditLogDays = positiveDays(retention.auditLogDays, 365)
// 1. 按配置清理超出上限或过期的打手商品匹配日志
const matchConfig = getWorkerProductMatchConfig()
const matchLogPruneResult = await pruneWorkProductMatchLogs({
maxRows: matchConfig.maxMatchLogs,
retentionDays: matchConfig.matchLogRetentionDays,
})
// 2. 清理其他报文与日志
const rawPayloadRows =
(await scrubJsonColumnInBatches('orders', 'updated_at', rawPayloadDays)) +
(await scrubJsonColumnInBatches('kuaishou_industry_vouchers', 'updated_at', rawPayloadDays))
(await scrubJsonColumnInBatches("orders", "updated_at", rawPayloadDays)) +
(await scrubJsonColumnInBatches("kuaishou_industry_vouchers", "updated_at", rawPayloadDays))
const matchLogRows = await scrubJsonColumnInBatches(
'work_product_match_logs',
'created_at',
"work_product_match_logs",
"created_at",
rawPayloadDays,
)
const taskEventRows = await deleteOldRowsInBatches('task_events', taskEventDays)
const webhookEventRows = await deleteOldRowsInBatches('webhook_events', webhookEventDays)
const auditLogRows = await deleteOldRowsInBatches('admin_audit_logs', auditLogDays)
const taskEventRows = await deleteOldRowsInBatches("task_events", taskEventDays)
const webhookEventRows = await deleteOldRowsInBatches("webhook_events", webhookEventDays)
const auditLogRows = await deleteOldRowsInBatches("admin_audit_logs", auditLogDays)
return {
status: 'ok',
message: '数据保留清理完成',
status: "ok",
message: "数据保留清理完成",
rawPayloadRows,
matchLogRows,
matchLogDeletedRows: matchLogPruneResult.totalDeleted,
taskEventRows,
webhookEventRows,
auditLogRows,
}
}
async function scrubJsonColumnInBatches(
table: 'orders' | 'kuaishou_industry_vouchers' | 'work_product_match_logs',
dateColumn: 'created_at' | 'updated_at',
apps/backend/src/services/worker-platform/admin-worker-config-service.ts
+18
View File
@@ -1,3 +1,4 @@
import { pruneWorkProductMatchLogs } from '../../repositories/worker-platform/product-match-repo.js'
import type { JsonObject } from '../../types/json.js'
import {
getWorkerAnnouncementConfig,
@@ -35,3 +36,20 @@ export async function saveAdminWorkerAnnouncementConfig(payload: JsonObject = {}
export async function saveAdminWorkerProductMatchConfig(payload: JsonObject = {}) {
return saveWorkerProductMatchConfig(payload)
}
export async function pruneAdminWorkProductMatchLogs(payload: JsonObject = {}) {
const config = getWorkerProductMatchConfig()
const maxRows = payload.maxRows !== undefined ? Number(payload.maxRows) : config.maxMatchLogs
const retentionDays =
payload.retentionDays !== undefined
? Number(payload.retentionDays)
: config.matchLogRetentionDays
return pruneWorkProductMatchLogs({
maxRows: Number.isInteger(maxRows) && maxRows >= 0 ? maxRows : config.maxMatchLogs,
retentionDays:
Number.isInteger(retentionDays) && retentionDays >= 0
? retentionDays
: config.matchLogRetentionDays,
})
}
apps/backend/src/services/worker-platform/worker-product-match-config-service.ts
+29 -7
View File
@@ -1,9 +1,12 @@
import { APP_CONFIG_KEYS } from '../../config/app-config-keys.js'
import type { JsonObject } from '../../types/json.js'
import { readAppConfigEntry, saveAppConfigEntry } from '../config/app-config-store.js'
import { APP_CONFIG_KEYS } from "../../config/app-config-keys.js"
import type { JsonObject } from "../../types/json.js"
import { readAppConfigEntry, saveAppConfigEntry } from "../config/app-config-store.js"
import { pruneWorkProductMatchLogs } from "../../repositories/worker-platform/product-match-repo.js"
export type WorkerProductMatchConfig = {
legacyFallbackEnabled: boolean
maxMatchLogs: number
matchLogRetentionDays: number
}
export function getWorkerProductMatchConfig(): WorkerProductMatchConfig {
@@ -17,26 +20,45 @@ export function getWorkerProductMatchConfig(): WorkerProductMatchConfig {
export async function saveWorkerProductMatchConfig(
rawValue: unknown,
): Promise<WorkerProductMatchConfig> {
return saveAppConfigEntry({
const saved = await saveAppConfigEntry({
configKey: APP_CONFIG_KEYS.workerProductMatch,
value: rawValue,
normalize: normalizeWorkerProductMatchConfig,
})
if (saved.maxMatchLogs > 0 || saved.matchLogRetentionDays > 0) {
// 异步触发超出日志清理，避免阻塞保存请求
void pruneWorkProductMatchLogs({
maxRows: saved.maxMatchLogs,
retentionDays: saved.matchLogRetentionDays,
}).catch(() => {})
}
return saved
}
export function createDefaultWorkerProductMatchConfig(): WorkerProductMatchConfig {
return { legacyFallbackEnabled: false }
return {
legacyFallbackEnabled: false,
maxMatchLogs: 10000,
matchLogRetentionDays: 30,
}
}
export function normalizeWorkerProductMatchConfig(rawValue: unknown): WorkerProductMatchConfig {
const source = isPlainObject(rawValue) ? rawValue : {}
const maxLogs = Number(source.maxMatchLogs)
const retentionDays = Number(source.matchLogRetentionDays)
return {
// 默认关闭旧规则，避免大标题中的公共词触发错误接单。
legacyFallbackEnabled:
typeof source.legacyFallbackEnabled === 'boolean' ? source.legacyFallbackEnabled : false,
typeof source.legacyFallbackEnabled === "boolean" ? source.legacyFallbackEnabled : false,
// 匹配日志保留上限条数，默认 10000 条，0 为不限
maxMatchLogs: Number.isInteger(maxLogs) && maxLogs >= 0 ? maxLogs : 10000,
// 匹配日志保留天数，默认 30 天，0 为不限
matchLogRetentionDays:
Number.isInteger(retentionDays) && retentionDays >= 0 ? retentionDays : 30,
}
}
function isPlainObject(value: unknown): value is JsonObject {
return Object.prototype.toString.call(value) === '[object Object]'
return Object.prototype.toString.call(value) === "[object Object]"
}