Files
order_site/apps/backend/src/services/semi-auto/semi-auto-trace.ts
T

342 lines
10 KiB
TypeScript

import { AsyncLocalStorage } from 'node:async_hooks'
import crypto from 'node:crypto'
import { query } from '../../db/client.js'
import { logWarn } from '../../utils/logger.js'
/** 报文保留天数:180 天,由 startSemiAutoTraceCleanup 定时清理。 */
export const SEMI_AUTO_TRACE_RETENTION_DAYS = 180
const MAX_BODY_CHARS = 32_000
const MAX_REDACT_DEPTH = 6
const FINGERPRINT_LIMIT = 5_000
export type SemiAutoTraceChannel =
| 'customer_api'
| 'admin_api'
| 'huya_wup'
| 'huya_http'
| 'external_connector'
export type SemiAutoTraceContext = {
requestId: string
taskId?: number
orderNo?: string
source?: string
}
export type SemiAutoTraceEntry = {
taskId?: number
orderNo?: string
requestId?: string
direction: 'inbound' | 'outbound'
channel: SemiAutoTraceChannel | string
action: string
status?: 'ok' | 'error'
httpStatus?: number
durationMs?: number
requestSummary?: Record<string, unknown>
requestBody?: unknown
/** 请求体里的通用 `code` 字段按验证码处理;响应体中的业务 code 保持原样。 */
maskRequestCode?: boolean
responseSummary?: Record<string, unknown>
responseBody?: unknown
errorMessage?: string
/** 写操作/关键读:无条件记录(轮询类不要传)。 */
always?: boolean
/** 轮询类请求的状态指纹:与上次相同则不记录。 */
fingerprint?: string | undefined
}
const traceStorage = new AsyncLocalStorage<SemiAutoTraceContext>()
const lastFingerprints = new Map<string, string>()
/** Cookie / Token / 密钥字段:一律不落库。 */
const SENSITIVE_KEYS = new Set([
'cookie',
'set-cookie',
'authorization',
'proxyauthorization',
'token',
'accesstoken',
'access_token',
'refreshtoken',
'refresh_token',
'biztoken',
'udb_biztoken',
'password',
'passwd',
'secret',
'apikey',
'api_key',
'x-semi-auto-key',
'sig',
'qrcodetoken',
'qrcode_token',
'qrcontent',
'qr_content',
'qrimage',
'qr_image',
'query_string',
'smscode',
'sms_code',
'sessionid',
'session_id',
'baseinfo',
'cookies',
'cookie_header',
'cookieheader',
'ticket',
'credential_payload_json',
])
const REQUEST_ONLY_SENSITIVE_KEYS = new Set(['code', 'rid', 'mfa_id'])
export function runWithSemiAutoTrace<T>(context: SemiAutoTraceContext, fn: () => T): T {
return traceStorage.run(context, fn)
}
export function currentSemiAutoTrace(): SemiAutoTraceContext | undefined {
return traceStorage.getStore()
}
function maskValue(value: unknown): unknown {
if (value === null || value === undefined) return value
if (typeof value !== 'string') return '******'
return value.length > 0 ? '******' : ''
}
function normalizeKey(key: string): string {
return key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase()
}
/** 深度脱敏:Cookie / Token / 密钥字段直接替换为 ******。 */
export function redactTraceValue(
value: unknown,
depth = 0,
options: { maskRequestCode?: boolean } = {},
): unknown {
if (depth > MAX_REDACT_DEPTH) return '[max-depth]'
if (value === null || value === undefined) return value
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
return value
}
if (Array.isArray(value)) {
return value.slice(0, 50).map((item) => redactTraceValue(item, depth + 1, options))
}
if (typeof value === 'object') {
const source = value as Record<string, unknown>
const result: Record<string, unknown> = {}
for (const [key, current] of Object.entries(source)) {
const normalizedKey = normalizeKey(key)
result[key] =
SENSITIVE_KEYS.has(normalizedKey) ||
(options.maskRequestCode === true && REQUEST_ONLY_SENSITIVE_KEYS.has(normalizedKey))
? maskValue(current)
: redactTraceValue(current, depth + 1, options)
}
return result
}
return String(value)
}
export function stringifyTraceBody(
value: unknown,
options: { maskRequestCode?: boolean } = {},
): string {
if (value === undefined || value === null) return ''
if (typeof value === 'string') {
return value.length > MAX_BODY_CHARS
? `${value.slice(0, MAX_BODY_CHARS)}\n...[truncated]`
: value
}
try {
const text = JSON.stringify(redactTraceValue(value, 0, options), null, 2)
return text.length > MAX_BODY_CHARS ? `${text.slice(0, MAX_BODY_CHARS)}\n...[truncated]` : text
} catch {
return String(value).slice(0, MAX_BODY_CHARS)
}
}
function fingerprintKey(entry: SemiAutoTraceEntry): string {
return `${entry.taskId || 0}|${entry.channel}|${entry.action}`
}
/**
* 采样:写操作与关键读一律记录;轮询类只在状态变化或失败时记录,避免每 2 秒一条。
*/
export function shouldRecordTrace(entry: SemiAutoTraceEntry): boolean {
if (entry.always) return true
if ((entry.status || 'ok') === 'error') return true
const fingerprint = String(entry.fingerprint || '')
if (!fingerprint) return true
const key = fingerprintKey(entry)
const previous = lastFingerprints.get(key)
if (previous === fingerprint) return false
lastFingerprints.set(key, fingerprint)
if (lastFingerprints.size > FINGERPRINT_LIMIT) lastFingerprints.clear()
return true
}
/** 记录一条报文:异步落库,失败只记日志,绝不阻断主流程。 */
export function recordSemiAutoTrace(entry: SemiAutoTraceEntry): void {
const context = currentSemiAutoTrace()
const taskId = Number(entry.taskId || context?.taskId || 0)
if (!Number.isFinite(taskId) || taskId <= 0) return
if (!shouldRecordTrace(entry)) return
const orderNo = entry.orderNo || context?.orderNo || ''
const requestId = entry.requestId || context?.requestId || ''
const status = entry.status === 'error' ? 'error' : 'ok'
const requestBody = stringifyTraceBody(entry.requestBody, {
maskRequestCode: entry.maskRequestCode === true,
})
const responseBody = stringifyTraceBody(entry.responseBody)
void query(
`INSERT INTO semi_auto_trace_logs (
task_id, order_no, request_id, direction, channel, action, status,
http_status, duration_ms, request_summary, request_body,
response_summary, response_body, error_message, created_at
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::jsonb,$11,$12::jsonb,$13,$14,NOW())`,
[
taskId,
orderNo,
requestId,
entry.direction,
entry.channel,
entry.action,
status,
entry.httpStatus ?? null,
entry.durationMs ?? null,
JSON.stringify(redactTraceValue(entry.requestSummary || {})),
requestBody,
JSON.stringify(redactTraceValue(entry.responseSummary || {})),
responseBody,
String(entry.errorMessage || ''),
],
).catch((error) => {
logWarn('semi-auto/trace', '写入报文失败', {
taskId,
action: entry.action,
error: error instanceof Error ? error.message : String(error),
})
})
}
/** 供调用方快速生成一个轮询指纹。 */
export function traceFingerprint(value: unknown): string {
if (value === null || value === undefined) return ''
let text = ''
if (typeof value === 'string') {
text = value
} else {
try {
text = JSON.stringify(value)
} catch {
text = String(value)
}
}
return crypto.createHash('sha1').update(text).digest('hex')
}
export type SemiAutoTraceQuery = {
page?: number
pageSize?: number
channel?: string
status?: string
requestId?: string
keyword?: string
}
export async function querySemiAutoTraces(taskId: number, input: SemiAutoTraceQuery = {}) {
const page = Math.max(1, Math.floor(Number(input.page) || 1))
const pageSize = Math.min(200, Math.max(1, Math.floor(Number(input.pageSize) || 50)))
const conditions = ['task_id = $1']
const params: unknown[] = [taskId]
if (input.channel) {
params.push(String(input.channel))
conditions.push(`channel = $${params.length}`)
}
if (input.status) {
params.push(String(input.status))
conditions.push(`status = $${params.length}`)
}
if (input.requestId) {
params.push(String(input.requestId))
conditions.push(`request_id = $${params.length}`)
}
if (input.keyword) {
params.push(`%${String(input.keyword).trim()}%`)
conditions.push(
`(action ILIKE $${params.length} OR request_body ILIKE $${params.length} OR response_body ILIKE $${params.length})`,
)
}
const where = conditions.join(' AND ')
const count = await query(
`SELECT COUNT(*)::int AS total FROM semi_auto_trace_logs WHERE ${where}`,
params,
)
const rows = await query(
`SELECT id, order_no, request_id, direction, channel, action, status,
http_status, duration_ms, request_summary, request_body,
response_summary, response_body, error_message, created_at
FROM semi_auto_trace_logs
WHERE ${where}
ORDER BY id DESC
LIMIT $${params.length + 1} OFFSET $${params.length + 2}`,
[...params, pageSize, (page - 1) * pageSize],
)
const summary = await query(
`SELECT channel, status, COUNT(*)::int AS total
FROM semi_auto_trace_logs
WHERE task_id = $1
GROUP BY channel, status`,
[taskId],
)
return {
pagination: { page, pageSize, total: Number(count.rows[0]?.total || 0) },
summary: summary.rows,
items: rows.rows.map((row) => ({
id: Number(row.id),
orderNo: row.order_no,
requestId: row.request_id,
direction: row.direction,
channel: row.channel,
action: row.action,
status: row.status,
httpStatus: row.http_status,
durationMs: row.duration_ms,
requestSummary: row.request_summary || {},
requestBody: row.request_body || '',
responseSummary: row.response_summary || {},
responseBody: row.response_body || '',
errorMessage: row.error_message || '',
createdAt: row.created_at,
})),
}
}
export async function cleanupSemiAutoTraces(retentionDays = SEMI_AUTO_TRACE_RETENTION_DAYS) {
const days = Math.max(1, Math.floor(Number(retentionDays) || SEMI_AUTO_TRACE_RETENTION_DAYS))
const result = await query(
`DELETE FROM semi_auto_trace_logs WHERE created_at < NOW() - ($1 || ' days')::interval`,
[String(days)],
)
return result.rowCount || 0
}
let cleanupStarted = false
/** 启动时清一次,之后每 6 小时清理超过保留期的报文。 */
export function startSemiAutoTraceCleanup(): void {
if (cleanupStarted) return
cleanupStarted = true
const run = () => {
void cleanupSemiAutoTraces().catch((error) => {
logWarn('semi-auto/trace', '清理历史报文失败', {
error: error instanceof Error ? error.message : String(error),
})
})
}
run()
const timer = setInterval(run, 6 * 60 * 60 * 1000)
timer.unref?.()
}