feat(semi-auto): 新增履约历史与全链路报文留痕
运营端历史抽屉展示状态流转与全链路报文:客户/运营接口入站 + 虎牙 WUP/livelink、外部 Connector 出站;轮询按状态变化采样,Cookie/Token/短信验证码不落库,报文保留 180 天。同步收敛登录/绑定事件,避免轮询重复写时间线。
This commit is contained in:
@@ -0,0 +1,326 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
import crypto from 'node:crypto'
|
||||
|
||||
import { query } from '../../db/client.js'
|
||||
import { logWarn } from '../../utils/logger.js'
|
||||
|
||||
/** 报文保留天数:180 天,由 startSemiAutoTraceCleanup 定时清理。 */
|
||||
export const SEMI_AUTO_TRACE_RETENTION_DAYS = 180
|
||||
const MAX_BODY_CHARS = 32_000
|
||||
const MAX_REDACT_DEPTH = 6
|
||||
const FINGERPRINT_LIMIT = 5_000
|
||||
|
||||
export type SemiAutoTraceChannel =
|
||||
| 'customer_api'
|
||||
| 'admin_api'
|
||||
| 'huya_wup'
|
||||
| 'huya_http'
|
||||
| 'external_connector'
|
||||
|
||||
export type SemiAutoTraceContext = {
|
||||
requestId: string
|
||||
taskId?: number
|
||||
orderNo?: string
|
||||
source?: string
|
||||
}
|
||||
|
||||
export type SemiAutoTraceEntry = {
|
||||
taskId?: number
|
||||
orderNo?: string
|
||||
requestId?: string
|
||||
direction: 'inbound' | 'outbound'
|
||||
channel: SemiAutoTraceChannel | string
|
||||
action: string
|
||||
status?: 'ok' | 'error'
|
||||
httpStatus?: number
|
||||
durationMs?: number
|
||||
requestSummary?: Record<string, unknown>
|
||||
requestBody?: unknown
|
||||
responseSummary?: Record<string, unknown>
|
||||
responseBody?: unknown
|
||||
errorMessage?: string
|
||||
/** 写操作/关键读:无条件记录(轮询类不要传)。 */
|
||||
always?: boolean
|
||||
/** 轮询类请求的状态指纹:与上次相同则不记录。 */
|
||||
fingerprint?: string | undefined
|
||||
}
|
||||
|
||||
const traceStorage = new AsyncLocalStorage<SemiAutoTraceContext>()
|
||||
const lastFingerprints = new Map<string, string>()
|
||||
|
||||
/** Cookie / Token / 密钥字段:一律不落库。 */
|
||||
const SENSITIVE_KEYS = new Set([
|
||||
'cookie',
|
||||
'set-cookie',
|
||||
'authorization',
|
||||
'proxyauthorization',
|
||||
'token',
|
||||
'accesstoken',
|
||||
'access_token',
|
||||
'refreshtoken',
|
||||
'refresh_token',
|
||||
'biztoken',
|
||||
'udb_biztoken',
|
||||
'password',
|
||||
'passwd',
|
||||
'secret',
|
||||
'apikey',
|
||||
'api_key',
|
||||
'x-semi-auto-key',
|
||||
'sig',
|
||||
'qrcodetoken',
|
||||
'qrcode_token',
|
||||
'qrcontent',
|
||||
'qr_content',
|
||||
'qrimage',
|
||||
'qr_image',
|
||||
'query_string',
|
||||
'smscode',
|
||||
'sms_code',
|
||||
'sessionid',
|
||||
'session_id',
|
||||
'baseinfo',
|
||||
'cookies',
|
||||
'cookie_header',
|
||||
'cookieheader',
|
||||
'ticket',
|
||||
'credential_payload_json',
|
||||
])
|
||||
|
||||
export function runWithSemiAutoTrace<T>(context: SemiAutoTraceContext, fn: () => T): T {
|
||||
return traceStorage.run(context, fn)
|
||||
}
|
||||
|
||||
export function currentSemiAutoTrace(): SemiAutoTraceContext | undefined {
|
||||
return traceStorage.getStore()
|
||||
}
|
||||
|
||||
function maskValue(value: unknown): unknown {
|
||||
if (value === null || value === undefined) return value
|
||||
if (typeof value !== 'string') return '******'
|
||||
return value.length > 0 ? '******' : ''
|
||||
}
|
||||
|
||||
function normalizeKey(key: string): string {
|
||||
return key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase()
|
||||
}
|
||||
|
||||
/** 深度脱敏:Cookie / Token / 密钥字段直接替换为 ******。 */
|
||||
export function redactTraceValue(value: unknown, depth = 0): unknown {
|
||||
if (depth > MAX_REDACT_DEPTH) return '[max-depth]'
|
||||
if (value === null || value === undefined) return value
|
||||
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
||||
return value
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.slice(0, 50).map((item) => redactTraceValue(item, depth + 1))
|
||||
}
|
||||
if (typeof value === 'object') {
|
||||
const source = value as Record<string, unknown>
|
||||
const result: Record<string, unknown> = {}
|
||||
for (const [key, current] of Object.entries(source)) {
|
||||
result[key] = SENSITIVE_KEYS.has(normalizeKey(key))
|
||||
? maskValue(current)
|
||||
: redactTraceValue(current, depth + 1)
|
||||
}
|
||||
return result
|
||||
}
|
||||
return String(value)
|
||||
}
|
||||
|
||||
export function stringifyTraceBody(value: unknown): string {
|
||||
if (value === undefined || value === null) return ''
|
||||
if (typeof value === 'string') {
|
||||
return value.length > MAX_BODY_CHARS
|
||||
? `${value.slice(0, MAX_BODY_CHARS)}\n...[truncated]`
|
||||
: value
|
||||
}
|
||||
try {
|
||||
const text = JSON.stringify(redactTraceValue(value), null, 2)
|
||||
return text.length > MAX_BODY_CHARS ? `${text.slice(0, MAX_BODY_CHARS)}\n...[truncated]` : text
|
||||
} catch {
|
||||
return String(value).slice(0, MAX_BODY_CHARS)
|
||||
}
|
||||
}
|
||||
|
||||
function fingerprintKey(entry: SemiAutoTraceEntry): string {
|
||||
return `${entry.taskId || 0}|${entry.channel}|${entry.action}`
|
||||
}
|
||||
|
||||
/**
|
||||
* 采样:写操作与关键读一律记录;轮询类只在状态变化或失败时记录,避免每 2 秒一条。
|
||||
*/
|
||||
export function shouldRecordTrace(entry: SemiAutoTraceEntry): boolean {
|
||||
if (entry.always) return true
|
||||
if ((entry.status || 'ok') === 'error') return true
|
||||
const fingerprint = String(entry.fingerprint || '')
|
||||
if (!fingerprint) return true
|
||||
const key = fingerprintKey(entry)
|
||||
const previous = lastFingerprints.get(key)
|
||||
if (previous === fingerprint) return false
|
||||
lastFingerprints.set(key, fingerprint)
|
||||
if (lastFingerprints.size > FINGERPRINT_LIMIT) lastFingerprints.clear()
|
||||
return true
|
||||
}
|
||||
|
||||
/** 记录一条报文:异步落库,失败只记日志,绝不阻断主流程。 */
|
||||
export function recordSemiAutoTrace(entry: SemiAutoTraceEntry): void {
|
||||
const context = currentSemiAutoTrace()
|
||||
const taskId = Number(entry.taskId || context?.taskId || 0)
|
||||
if (!Number.isFinite(taskId) || taskId <= 0) return
|
||||
if (!shouldRecordTrace(entry)) return
|
||||
const orderNo = entry.orderNo || context?.orderNo || ''
|
||||
const requestId = entry.requestId || context?.requestId || ''
|
||||
const status = entry.status === 'error' ? 'error' : 'ok'
|
||||
const requestBody = stringifyTraceBody(entry.requestBody)
|
||||
const responseBody = stringifyTraceBody(entry.responseBody)
|
||||
void query(
|
||||
`INSERT INTO semi_auto_trace_logs (
|
||||
task_id, order_no, request_id, direction, channel, action, status,
|
||||
http_status, duration_ms, request_summary, request_body,
|
||||
response_summary, response_body, error_message, created_at
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::jsonb,$11,$12::jsonb,$13,$14,NOW())`,
|
||||
[
|
||||
taskId,
|
||||
orderNo,
|
||||
requestId,
|
||||
entry.direction,
|
||||
entry.channel,
|
||||
entry.action,
|
||||
status,
|
||||
entry.httpStatus ?? null,
|
||||
entry.durationMs ?? null,
|
||||
JSON.stringify(redactTraceValue(entry.requestSummary || {})),
|
||||
requestBody,
|
||||
JSON.stringify(redactTraceValue(entry.responseSummary || {})),
|
||||
responseBody,
|
||||
String(entry.errorMessage || ''),
|
||||
],
|
||||
).catch((error) => {
|
||||
logWarn('semi-auto/trace', '写入报文失败', {
|
||||
taskId,
|
||||
action: entry.action,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/** 供调用方快速生成一个轮询指纹。 */
|
||||
export function traceFingerprint(value: unknown): string {
|
||||
if (value === null || value === undefined) return ''
|
||||
let text = ''
|
||||
if (typeof value === 'string') {
|
||||
text = value
|
||||
} else {
|
||||
try {
|
||||
text = JSON.stringify(value)
|
||||
} catch {
|
||||
text = String(value)
|
||||
}
|
||||
}
|
||||
return crypto.createHash('sha1').update(text).digest('hex')
|
||||
}
|
||||
|
||||
export type SemiAutoTraceQuery = {
|
||||
page?: number
|
||||
pageSize?: number
|
||||
channel?: string
|
||||
status?: string
|
||||
requestId?: string
|
||||
keyword?: string
|
||||
}
|
||||
|
||||
export async function querySemiAutoTraces(taskId: number, input: SemiAutoTraceQuery = {}) {
|
||||
const page = Math.max(1, Math.floor(Number(input.page) || 1))
|
||||
const pageSize = Math.min(200, Math.max(1, Math.floor(Number(input.pageSize) || 50)))
|
||||
const conditions = ['task_id = $1']
|
||||
const params: unknown[] = [taskId]
|
||||
if (input.channel) {
|
||||
params.push(String(input.channel))
|
||||
conditions.push(`channel = $${params.length}`)
|
||||
}
|
||||
if (input.status) {
|
||||
params.push(String(input.status))
|
||||
conditions.push(`status = $${params.length}`)
|
||||
}
|
||||
if (input.requestId) {
|
||||
params.push(String(input.requestId))
|
||||
conditions.push(`request_id = $${params.length}`)
|
||||
}
|
||||
if (input.keyword) {
|
||||
params.push(`%${String(input.keyword).trim()}%`)
|
||||
conditions.push(
|
||||
`(action ILIKE $${params.length} OR request_body ILIKE $${params.length} OR response_body ILIKE $${params.length})`,
|
||||
)
|
||||
}
|
||||
const where = conditions.join(' AND ')
|
||||
const count = await query(
|
||||
`SELECT COUNT(*)::int AS total FROM semi_auto_trace_logs WHERE ${where}`,
|
||||
params,
|
||||
)
|
||||
const rows = await query(
|
||||
`SELECT id, order_no, request_id, direction, channel, action, status,
|
||||
http_status, duration_ms, request_summary, request_body,
|
||||
response_summary, response_body, error_message, created_at
|
||||
FROM semi_auto_trace_logs
|
||||
WHERE ${where}
|
||||
ORDER BY id DESC
|
||||
LIMIT $${params.length + 1} OFFSET $${params.length + 2}`,
|
||||
[...params, pageSize, (page - 1) * pageSize],
|
||||
)
|
||||
const summary = await query(
|
||||
`SELECT channel, status, COUNT(*)::int AS total
|
||||
FROM semi_auto_trace_logs
|
||||
WHERE task_id = $1
|
||||
GROUP BY channel, status`,
|
||||
[taskId],
|
||||
)
|
||||
return {
|
||||
pagination: { page, pageSize, total: Number(count.rows[0]?.total || 0) },
|
||||
summary: summary.rows,
|
||||
items: rows.rows.map((row) => ({
|
||||
id: Number(row.id),
|
||||
orderNo: row.order_no,
|
||||
requestId: row.request_id,
|
||||
direction: row.direction,
|
||||
channel: row.channel,
|
||||
action: row.action,
|
||||
status: row.status,
|
||||
httpStatus: row.http_status,
|
||||
durationMs: row.duration_ms,
|
||||
requestSummary: row.request_summary || {},
|
||||
requestBody: row.request_body || '',
|
||||
responseSummary: row.response_summary || {},
|
||||
responseBody: row.response_body || '',
|
||||
errorMessage: row.error_message || '',
|
||||
createdAt: row.created_at,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
export async function cleanupSemiAutoTraces(retentionDays = SEMI_AUTO_TRACE_RETENTION_DAYS) {
|
||||
const days = Math.max(1, Math.floor(Number(retentionDays) || SEMI_AUTO_TRACE_RETENTION_DAYS))
|
||||
const result = await query(
|
||||
`DELETE FROM semi_auto_trace_logs WHERE created_at < NOW() - ($1 || ' days')::interval`,
|
||||
[String(days)],
|
||||
)
|
||||
return result.rowCount || 0
|
||||
}
|
||||
|
||||
let cleanupStarted = false
|
||||
|
||||
/** 启动时清一次,之后每 6 小时清理超过保留期的报文。 */
|
||||
export function startSemiAutoTraceCleanup(): void {
|
||||
if (cleanupStarted) return
|
||||
cleanupStarted = true
|
||||
const run = () => {
|
||||
void cleanupSemiAutoTraces().catch((error) => {
|
||||
logWarn('semi-auto/trace', '清理历史报文失败', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
})
|
||||
}
|
||||
run()
|
||||
const timer = setInterval(run, 6 * 60 * 60 * 1000)
|
||||
timer.unref?.()
|
||||
}
|
||||
Reference in New Issue
Block a user