diff --git a/apps/backend/src/repositories/worker-platform/product-match-repo.ts b/apps/backend/src/repositories/worker-platform/product-match-repo.ts index 81280e27..d22e999f 100644 --- a/apps/backend/src/repositories/worker-platform/product-match-repo.ts +++ b/apps/backend/src/repositories/worker-platform/product-match-repo.ts @@ -293,3 +293,97 @@ export async function listWorkProductMatchLogs({ ) return { items: result.rows, total: Number(totalResult.rows[0]?.total || 0) } } + +export type PruneWorkProductMatchLogsOptions = { + maxRows?: number + retentionDays?: number + batchSize?: number +} + +export type PruneWorkProductMatchLogsResult = { + deletedByAge: number + deletedByCount: number + totalDeleted: number + remainingTotal: number +} + +/** 按条数上限或保留天数分批清理历史匹配日志 */ +export async function pruneWorkProductMatchLogs( + options: PruneWorkProductMatchLogsOptions = {}, +): Promise { + let deletedByAge = 0 + let deletedByCount = 0 + const batchSize = Math.min(5000, Math.max(100, options.batchSize || 1000)) + + // 1. 按保留天数清理过期日志 + if (options.retentionDays && options.retentionDays > 0) { + while (true) { + const res = await query( + ` + WITH candidates AS ( + SELECT ctid + FROM work_product_match_logs + WHERE created_at < NOW() - make_interval(days => $1::int) + LIMIT $2 + ) + DELETE FROM work_product_match_logs AS target + USING candidates + WHERE target.ctid = candidates.ctid + `, + [options.retentionDays, batchSize], + ) + const count = res.rowCount || 0 + deletedByAge += count + if (count < batchSize) break + } + } + + // 2. 按最大条数清理超出上限的旧日志（保留最新的 maxRows 条） + if (options.maxRows && options.maxRows > 0) { + const cutoffRes = await query<{ id: string }>( + ` + SELECT id + FROM work_product_match_logs + ORDER BY id DESC + OFFSET $1 + LIMIT 1 + `, + [options.maxRows], + ) + const firstRow = cutoffRes.rows[0] + if (firstRow) { + const cutoffId = Number(firstRow.id) + while (true) { + const res = await query( + ` + WITH candidates AS ( + SELECT ctid + FROM work_product_match_logs + WHERE id <= $1 + LIMIT $2 + ) + DELETE FROM work_product_match_logs AS target + USING candidates + WHERE target.ctid = candidates.ctid + `, + [cutoffId, batchSize], + ) + const count = res.rowCount || 0 + deletedByCount += count + if (count < batchSize) break + } + } + } + + const remainingRes = await query<{ total: number }>( + "SELECT COUNT(*)::int AS total FROM work_product_match_logs", + ) + const remainingTotal = Number(remainingRes.rows[0]?.total || 0) + + return { + deletedByAge, + deletedByCount, + totalDeleted: deletedByAge + deletedByCount, + remainingTotal, + } +} diff --git a/apps/backend/src/routes/admin/worker-platform/config.ts b/apps/backend/src/routes/admin/worker-platform/config.ts index e9e379f0..02a6c6f4 100644 --- a/apps/backend/src/routes/admin/worker-platform/config.ts +++ b/apps/backend/src/routes/admin/worker-platform/config.ts @@ -14,6 +14,7 @@ import { listAdminKuaishouMatchSources, listAdminWorkCategories, listAdminWorkProductMatchLogs, + pruneAdminWorkProductMatchLogs, listAdminWorkProductRuleMappings, listAdminWorkProductRules, listAdminWorkerLevels, @@ -341,6 +342,22 @@ router.get( }), ) +router.post( + '/worker-platform/product-match-logs/cleanup', + requireAdminRoles(['admin', 'operator']), + createJsonHandler((req) => pruneAdminWorkProductMatchLogs(req.body || {}), { + successMessage: '匹配日志清理完成', + errorMessage: '清理匹配日志失败', + scope: '[admin/worker-platform/product-match-logs/cleanup]', + audit: (_req, data) => ({ + action: 'worker_product_match_logs_cleaned', + targetType: 'worker_product_match_logs', + targetId: 'cleanup', + data: data && typeof data === 'object' ? (data as Record) : {}, + }), + }), +) + router.get( '/worker-platform/finance-config', requireAdminRoles(['admin', 'operator', 'support', 'finance']), diff --git a/apps/backend/src/services/scheduler/data-retention-job.ts b/apps/backend/src/services/scheduler/data-retention-job.ts index 71980e86..537970f6 100644 --- a/apps/backend/src/services/scheduler/data-retention-job.ts +++ b/apps/backend/src/services/scheduler/data-retention-job.ts @@ -1,14 +1,17 @@ -import { query } from '../../db/client.js' -import { runtimeConfig } from '../../config/runtime.js' +import { query } from "../../db/client.js" +import { runtimeConfig } from "../../config/runtime.js" +import { getWorkerProductMatchConfig } from "../worker-platform/worker-product-match-config-service.js" +import { pruneWorkProductMatchLogs } from "../../repositories/worker-platform/product-match-repo.js" export type DataRetentionResult = { - status: 'ok' + status: "ok" message: string rawPayloadRows: number taskEventRows: number webhookEventRows: number auditLogRows: number matchLogRows: number + matchLogDeletedRows: number } /** 清理敏感原始报文并删除已过保留期的事件/审计记录。 */ @@ -19,29 +22,37 @@ export async function runDataRetentionJob(): Promise { const webhookEventDays = positiveDays(retention.webhookEventDays, 180) const auditLogDays = positiveDays(retention.auditLogDays, 365) + // 1. 按配置清理超出上限或过期的打手商品匹配日志 + const matchConfig = getWorkerProductMatchConfig() + const matchLogPruneResult = await pruneWorkProductMatchLogs({ + maxRows: matchConfig.maxMatchLogs, + retentionDays: matchConfig.matchLogRetentionDays, + }) + + // 2. 清理其他报文与日志 const rawPayloadRows = - (await scrubJsonColumnInBatches('orders', 'updated_at', rawPayloadDays)) + - (await scrubJsonColumnInBatches('kuaishou_industry_vouchers', 'updated_at', rawPayloadDays)) + (await scrubJsonColumnInBatches("orders", "updated_at", rawPayloadDays)) + + (await scrubJsonColumnInBatches("kuaishou_industry_vouchers", "updated_at", rawPayloadDays)) const matchLogRows = await scrubJsonColumnInBatches( - 'work_product_match_logs', - 'created_at', + "work_product_match_logs", + "created_at", rawPayloadDays, ) - const taskEventRows = await deleteOldRowsInBatches('task_events', taskEventDays) - const webhookEventRows = await deleteOldRowsInBatches('webhook_events', webhookEventDays) - const auditLogRows = await deleteOldRowsInBatches('admin_audit_logs', auditLogDays) + const taskEventRows = await deleteOldRowsInBatches("task_events", taskEventDays) + const webhookEventRows = await deleteOldRowsInBatches("webhook_events", webhookEventDays) + const auditLogRows = await deleteOldRowsInBatches("admin_audit_logs", auditLogDays) return { - status: 'ok', - message: '数据保留清理完成', + status: "ok", + message: "数据保留清理完成", rawPayloadRows, matchLogRows, + matchLogDeletedRows: matchLogPruneResult.totalDeleted, taskEventRows, webhookEventRows, auditLogRows, } } - async function scrubJsonColumnInBatches( table: 'orders' | 'kuaishou_industry_vouchers' | 'work_product_match_logs', dateColumn: 'created_at' | 'updated_at', diff --git a/apps/backend/src/services/worker-platform/admin-worker-config-service.ts b/apps/backend/src/services/worker-platform/admin-worker-config-service.ts index 7048a389..df870d1d 100644 --- a/apps/backend/src/services/worker-platform/admin-worker-config-service.ts +++ b/apps/backend/src/services/worker-platform/admin-worker-config-service.ts @@ -1,3 +1,4 @@ +import { pruneWorkProductMatchLogs } from '../../repositories/worker-platform/product-match-repo.js' import type { JsonObject } from '../../types/json.js' import { getWorkerAnnouncementConfig, @@ -35,3 +36,20 @@ export async function saveAdminWorkerAnnouncementConfig(payload: JsonObject = {} export async function saveAdminWorkerProductMatchConfig(payload: JsonObject = {}) { return saveWorkerProductMatchConfig(payload) } + +export async function pruneAdminWorkProductMatchLogs(payload: JsonObject = {}) { + const config = getWorkerProductMatchConfig() + const maxRows = payload.maxRows !== undefined ? Number(payload.maxRows) : config.maxMatchLogs + const retentionDays = + payload.retentionDays !== undefined + ? Number(payload.retentionDays) + : config.matchLogRetentionDays + + return pruneWorkProductMatchLogs({ + maxRows: Number.isInteger(maxRows) && maxRows >= 0 ? maxRows : config.maxMatchLogs, + retentionDays: + Number.isInteger(retentionDays) && retentionDays >= 0 + ? retentionDays + : config.matchLogRetentionDays, + }) +} diff --git a/apps/backend/src/services/worker-platform/worker-product-match-config-service.ts b/apps/backend/src/services/worker-platform/worker-product-match-config-service.ts index 33d65876..0cf5208c 100644 --- a/apps/backend/src/services/worker-platform/worker-product-match-config-service.ts +++ b/apps/backend/src/services/worker-platform/worker-product-match-config-service.ts @@ -1,9 +1,12 @@ -import { APP_CONFIG_KEYS } from '../../config/app-config-keys.js' -import type { JsonObject } from '../../types/json.js' -import { readAppConfigEntry, saveAppConfigEntry } from '../config/app-config-store.js' +import { APP_CONFIG_KEYS } from "../../config/app-config-keys.js" +import type { JsonObject } from "../../types/json.js" +import { readAppConfigEntry, saveAppConfigEntry } from "../config/app-config-store.js" +import { pruneWorkProductMatchLogs } from "../../repositories/worker-platform/product-match-repo.js" export type WorkerProductMatchConfig = { legacyFallbackEnabled: boolean + maxMatchLogs: number + matchLogRetentionDays: number } export function getWorkerProductMatchConfig(): WorkerProductMatchConfig { @@ -17,26 +20,45 @@ export function getWorkerProductMatchConfig(): WorkerProductMatchConfig { export async function saveWorkerProductMatchConfig( rawValue: unknown, ): Promise { - return saveAppConfigEntry({ + const saved = await saveAppConfigEntry({ configKey: APP_CONFIG_KEYS.workerProductMatch, value: rawValue, normalize: normalizeWorkerProductMatchConfig, }) + if (saved.maxMatchLogs > 0 || saved.matchLogRetentionDays > 0) { + // 异步触发超出日志清理，避免阻塞保存请求 + void pruneWorkProductMatchLogs({ + maxRows: saved.maxMatchLogs, + retentionDays: saved.matchLogRetentionDays, + }).catch(() => {}) + } + return saved } export function createDefaultWorkerProductMatchConfig(): WorkerProductMatchConfig { - return { legacyFallbackEnabled: false } + return { + legacyFallbackEnabled: false, + maxMatchLogs: 10000, + matchLogRetentionDays: 30, + } } export function normalizeWorkerProductMatchConfig(rawValue: unknown): WorkerProductMatchConfig { const source = isPlainObject(rawValue) ? rawValue : {} + const maxLogs = Number(source.maxMatchLogs) + const retentionDays = Number(source.matchLogRetentionDays) return { // 默认关闭旧规则，避免大标题中的公共词触发错误接单。 legacyFallbackEnabled: - typeof source.legacyFallbackEnabled === 'boolean' ? source.legacyFallbackEnabled : false, + typeof source.legacyFallbackEnabled === "boolean" ? source.legacyFallbackEnabled : false, + // 匹配日志保留上限条数，默认 10000 条，0 为不限 + maxMatchLogs: Number.isInteger(maxLogs) && maxLogs >= 0 ? maxLogs : 10000, + // 匹配日志保留天数，默认 30 天，0 为不限 + matchLogRetentionDays: + Number.isInteger(retentionDays) && retentionDays >= 0 ? retentionDays : 30, } } function isPlainObject(value: unknown): value is JsonObject { - return Object.prototype.toString.call(value) === '[object Object]' + return Object.prototype.toString.call(value) === "[object Object]" } diff --git a/apps/frontend/src/pages/admin/panels/ProductMatchPanel.tsx b/apps/frontend/src/pages/admin/panels/ProductMatchPanel.tsx index 78fd72da..9c9a340a 100644 --- a/apps/frontend/src/pages/admin/panels/ProductMatchPanel.tsx +++ b/apps/frontend/src/pages/admin/panels/ProductMatchPanel.tsx @@ -1,4 +1,4 @@ -import { PlayCircleOutlined, ReloadOutlined, SearchOutlined } from '@ant-design/icons' +import { DeleteOutlined, PlayCircleOutlined, ReloadOutlined, SearchOutlined, SettingOutlined } from '@ant-design/icons' import { useQuery, useQueryClient } from '@tanstack/react-query' import { App, @@ -6,7 +6,9 @@ import { Card, Descriptions, Input, + InputNumber, Modal, + Popconfirm, Select, Space, Switch, @@ -16,7 +18,7 @@ import { Typography, } from 'antd' import type { TableColumnsType } from 'antd' -import { useState } from 'react' +import { useEffect, useState } from 'react' import { useNavigate } from 'react-router' import JsonPreview from '@/components/admin/JsonPreview' @@ -27,6 +29,7 @@ import { fetchAdminKuaishouMatchSources, fetchAdminWorkerProductMatchConfig, fetchAdminWorkProductMatchLogs, + cleanupAdminWorkProductMatchLogs, saveAdminWorkerProductMatchConfig, testAdminKuaishouProductMatch, } from '@/services/admin' @@ -50,6 +53,11 @@ export default function ProductMatchPanel() { const [logKeyword, setLogKeyword] = useState('') const [logCreatedRange, setLogCreatedRange] = useState<[string, string] | null>(null) const [logPage, setLogPage] = useState(1) + const [maxLogs, setMaxLogs] = useState(10000) + const [retentionDays, setRetentionDays] = useState(30) + const [savingRetention, setSavingRetention] = useState(false) + const [cleaningLogs, setCleaningLogs] = useState(false) + const shopsQuery = useQuery({ queryKey: ['admin-kuaishou-industry-shops', 'product-match'], @@ -87,6 +95,58 @@ export default function ProductMatchPanel() { const logsPagination = logsQuery.data?.data.pagination const shops = shopsQuery.data?.data.shops || [] const matchConfig = matchConfigQuery.data?.data + + useEffect(() => { + if (matchConfig) { + if (typeof matchConfig.maxMatchLogs === "number") { + setMaxLogs(matchConfig.maxMatchLogs) + } + if (typeof matchConfig.matchLogRetentionDays === "number") { + setRetentionDays(matchConfig.matchLogRetentionDays) + } + } + }, [matchConfig]) + + async function handleSaveRetentionConfig() { + setSavingRetention(true) + try { + await saveAdminWorkerProductMatchConfig({ + legacyFallbackEnabled: matchConfig?.legacyFallbackEnabled ?? false, + maxMatchLogs: maxLogs, + matchLogRetentionDays: retentionDays, + }) + message.success("匹配日志保留策略已保存并已执行清理") + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ["admin-worker-platform-product-match-config"] }), + queryClient.invalidateQueries({ queryKey: ["admin-worker-platform-product-match-logs"] }), + ]) + } catch (error) { + message.error(error instanceof Error ? error.message : "保存配置失败") + } finally { + setSavingRetention(false) + } + } + + async function handleCleanupLogs() { + setCleaningLogs(true) + try { + const response = await cleanupAdminWorkProductMatchLogs({ + maxRows: maxLogs, + retentionDays: retentionDays, + }) + const result = response.data + message.success( + `成功清理 ${result.totalDeleted} 条历史匹配日志，剩余 ${result.remainingTotal} 条`, + ) + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ["admin-worker-platform-product-match-logs"] }), + ]) + } catch (error) { + message.error(error instanceof Error ? error.message : "清理日志失败") + } finally { + setCleaningLogs(false) + } + } const shopNameBySellerId = createShopNameBySellerId(shops) function bindToTemplate(source: KuaishouMatchSource) { @@ -391,10 +451,71 @@ export default function ProductMatchPanel() { > 刷新 - - 日志全量保留，可按结果、关键词与日期筛选 - +
+ + + + 日志保留策略： + + 最多保留 + setMaxLogs(val || 10000)} + addonAfter="条" + style={{ width: 140 }} + /> + 保留天数 + setRetentionDays(val || 30)} + addonAfter="天" + style={{ width: 115 }} + /> + + + （自动滚动保留最新记录，超出自动清理） + + + + + +
rowKey="logId" loading={logsQuery.isLoading} diff --git a/apps/frontend/src/services/admin/worker-platform.ts b/apps/frontend/src/services/admin/worker-platform.ts index 5104eec2..a2056267 100644 --- a/apps/frontend/src/services/admin/worker-platform.ts +++ b/apps/frontend/src/services/admin/worker-platform.ts @@ -343,6 +343,17 @@ export function fetchAdminWorkProductMatchLogs(params?: Record) ) } +export function cleanupAdminWorkProductMatchLogs( + payload: { maxRows?: number; retentionDays?: number } = {}, +) { + return apiPost<{ + deletedByAge: number + deletedByCount: number + totalDeleted: number + remainingTotal: number + }>('/api/v1/admin/worker-platform/product-match-logs/cleanup', payload) +} + export function reprocessAdminKuaishouWorkOrderMatches(limit = 100) { return apiPost<{ scannedCount: number; createdCount: number; skippedCount: number }>( '/api/v1/admin/worker-platform/orders/reprocess-matches', diff --git a/apps/frontend/src/types/worker-platform.ts b/apps/frontend/src/types/worker-platform.ts index b6597e7e..38503551 100644 --- a/apps/frontend/src/types/worker-platform.ts +++ b/apps/frontend/src/types/worker-platform.ts @@ -291,6 +291,8 @@ export type WorkerPlatformNotificationConfig = { export type WorkerProductMatchConfig = { legacyFallbackEnabled: boolean + maxMatchLogs?: number + matchLogRetentionDays?: number } export type WorkerAnnouncementConfig = {