342 lines
10 KiB
TypeScript
342 lines
10 KiB
TypeScript
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
import crypto from 'node:crypto'
|
|
|
|
import { query } from '../../db/client.js'
|
|
import { logWarn } from '../../utils/logger.js'
|
|
|
|
/** 报文保留天数:180 天,由 startSemiAutoTraceCleanup 定时清理。 */
|
|
export const SEMI_AUTO_TRACE_RETENTION_DAYS = 180
|
|
const MAX_BODY_CHARS = 32_000
|
|
const MAX_REDACT_DEPTH = 6
|
|
const FINGERPRINT_LIMIT = 5_000
|
|
|
|
export type SemiAutoTraceChannel =
|
|
| 'customer_api'
|
|
| 'admin_api'
|
|
| 'huya_wup'
|
|
| 'huya_http'
|
|
| 'external_connector'
|
|
|
|
export type SemiAutoTraceContext = {
|
|
requestId: string
|
|
taskId?: number
|
|
orderNo?: string
|
|
source?: string
|
|
}
|
|
|
|
export type SemiAutoTraceEntry = {
|
|
taskId?: number
|
|
orderNo?: string
|
|
requestId?: string
|
|
direction: 'inbound' | 'outbound'
|
|
channel: SemiAutoTraceChannel | string
|
|
action: string
|
|
status?: 'ok' | 'error'
|
|
httpStatus?: number
|
|
durationMs?: number
|
|
requestSummary?: Record<string, unknown>
|
|
requestBody?: unknown
|
|
/** 请求体里的通用 `code` 字段按验证码处理;响应体中的业务 code 保持原样。 */
|
|
maskRequestCode?: boolean
|
|
responseSummary?: Record<string, unknown>
|
|
responseBody?: unknown
|
|
errorMessage?: string
|
|
/** 写操作/关键读:无条件记录(轮询类不要传)。 */
|
|
always?: boolean
|
|
/** 轮询类请求的状态指纹:与上次相同则不记录。 */
|
|
fingerprint?: string | undefined
|
|
}
|
|
|
|
const traceStorage = new AsyncLocalStorage<SemiAutoTraceContext>()
|
|
const lastFingerprints = new Map<string, string>()
|
|
|
|
/** Cookie / Token / 密钥字段:一律不落库。 */
|
|
const SENSITIVE_KEYS = new Set([
|
|
'cookie',
|
|
'set-cookie',
|
|
'authorization',
|
|
'proxyauthorization',
|
|
'token',
|
|
'accesstoken',
|
|
'access_token',
|
|
'refreshtoken',
|
|
'refresh_token',
|
|
'biztoken',
|
|
'udb_biztoken',
|
|
'password',
|
|
'passwd',
|
|
'secret',
|
|
'apikey',
|
|
'api_key',
|
|
'x-semi-auto-key',
|
|
'sig',
|
|
'qrcodetoken',
|
|
'qrcode_token',
|
|
'qrcontent',
|
|
'qr_content',
|
|
'qrimage',
|
|
'qr_image',
|
|
'query_string',
|
|
'smscode',
|
|
'sms_code',
|
|
'sessionid',
|
|
'session_id',
|
|
'baseinfo',
|
|
'cookies',
|
|
'cookie_header',
|
|
'cookieheader',
|
|
'ticket',
|
|
'credential_payload_json',
|
|
])
|
|
const REQUEST_ONLY_SENSITIVE_KEYS = new Set(['code', 'rid', 'mfa_id'])
|
|
|
|
export function runWithSemiAutoTrace<T>(context: SemiAutoTraceContext, fn: () => T): T {
|
|
return traceStorage.run(context, fn)
|
|
}
|
|
|
|
export function currentSemiAutoTrace(): SemiAutoTraceContext | undefined {
|
|
return traceStorage.getStore()
|
|
}
|
|
|
|
function maskValue(value: unknown): unknown {
|
|
if (value === null || value === undefined) return value
|
|
if (typeof value !== 'string') return '******'
|
|
return value.length > 0 ? '******' : ''
|
|
}
|
|
|
|
function normalizeKey(key: string): string {
|
|
return key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase()
|
|
}
|
|
|
|
/** 深度脱敏:Cookie / Token / 密钥字段直接替换为 ******。 */
|
|
export function redactTraceValue(
|
|
value: unknown,
|
|
depth = 0,
|
|
options: { maskRequestCode?: boolean } = {},
|
|
): unknown {
|
|
if (depth > MAX_REDACT_DEPTH) return '[max-depth]'
|
|
if (value === null || value === undefined) return value
|
|
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
|
return value
|
|
}
|
|
if (Array.isArray(value)) {
|
|
return value.slice(0, 50).map((item) => redactTraceValue(item, depth + 1, options))
|
|
}
|
|
if (typeof value === 'object') {
|
|
const source = value as Record<string, unknown>
|
|
const result: Record<string, unknown> = {}
|
|
for (const [key, current] of Object.entries(source)) {
|
|
const normalizedKey = normalizeKey(key)
|
|
result[key] =
|
|
SENSITIVE_KEYS.has(normalizedKey) ||
|
|
(options.maskRequestCode === true && REQUEST_ONLY_SENSITIVE_KEYS.has(normalizedKey))
|
|
? maskValue(current)
|
|
: redactTraceValue(current, depth + 1, options)
|
|
}
|
|
return result
|
|
}
|
|
return String(value)
|
|
}
|
|
|
|
export function stringifyTraceBody(
|
|
value: unknown,
|
|
options: { maskRequestCode?: boolean } = {},
|
|
): string {
|
|
if (value === undefined || value === null) return ''
|
|
if (typeof value === 'string') {
|
|
return value.length > MAX_BODY_CHARS
|
|
? `${value.slice(0, MAX_BODY_CHARS)}\n...[truncated]`
|
|
: value
|
|
}
|
|
try {
|
|
const text = JSON.stringify(redactTraceValue(value, 0, options), null, 2)
|
|
return text.length > MAX_BODY_CHARS ? `${text.slice(0, MAX_BODY_CHARS)}\n...[truncated]` : text
|
|
} catch {
|
|
return String(value).slice(0, MAX_BODY_CHARS)
|
|
}
|
|
}
|
|
|
|
function fingerprintKey(entry: SemiAutoTraceEntry): string {
|
|
return `${entry.taskId || 0}|${entry.channel}|${entry.action}`
|
|
}
|
|
|
|
/**
|
|
* 采样:写操作与关键读一律记录;轮询类只在状态变化或失败时记录,避免每 2 秒一条。
|
|
*/
|
|
export function shouldRecordTrace(entry: SemiAutoTraceEntry): boolean {
|
|
if (entry.always) return true
|
|
if ((entry.status || 'ok') === 'error') return true
|
|
const fingerprint = String(entry.fingerprint || '')
|
|
if (!fingerprint) return true
|
|
const key = fingerprintKey(entry)
|
|
const previous = lastFingerprints.get(key)
|
|
if (previous === fingerprint) return false
|
|
lastFingerprints.set(key, fingerprint)
|
|
if (lastFingerprints.size > FINGERPRINT_LIMIT) lastFingerprints.clear()
|
|
return true
|
|
}
|
|
|
|
/** 记录一条报文:异步落库,失败只记日志,绝不阻断主流程。 */
|
|
export function recordSemiAutoTrace(entry: SemiAutoTraceEntry): void {
|
|
const context = currentSemiAutoTrace()
|
|
const taskId = Number(entry.taskId || context?.taskId || 0)
|
|
if (!Number.isFinite(taskId) || taskId <= 0) return
|
|
if (!shouldRecordTrace(entry)) return
|
|
const orderNo = entry.orderNo || context?.orderNo || ''
|
|
const requestId = entry.requestId || context?.requestId || ''
|
|
const status = entry.status === 'error' ? 'error' : 'ok'
|
|
const requestBody = stringifyTraceBody(entry.requestBody, {
|
|
maskRequestCode: entry.maskRequestCode === true,
|
|
})
|
|
const responseBody = stringifyTraceBody(entry.responseBody)
|
|
void query(
|
|
`INSERT INTO semi_auto_trace_logs (
|
|
task_id, order_no, request_id, direction, channel, action, status,
|
|
http_status, duration_ms, request_summary, request_body,
|
|
response_summary, response_body, error_message, created_at
|
|
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::jsonb,$11,$12::jsonb,$13,$14,NOW())`,
|
|
[
|
|
taskId,
|
|
orderNo,
|
|
requestId,
|
|
entry.direction,
|
|
entry.channel,
|
|
entry.action,
|
|
status,
|
|
entry.httpStatus ?? null,
|
|
entry.durationMs ?? null,
|
|
JSON.stringify(redactTraceValue(entry.requestSummary || {})),
|
|
requestBody,
|
|
JSON.stringify(redactTraceValue(entry.responseSummary || {})),
|
|
responseBody,
|
|
String(entry.errorMessage || ''),
|
|
],
|
|
).catch((error) => {
|
|
logWarn('semi-auto/trace', '写入报文失败', {
|
|
taskId,
|
|
action: entry.action,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
})
|
|
})
|
|
}
|
|
|
|
/** 供调用方快速生成一个轮询指纹。 */
|
|
export function traceFingerprint(value: unknown): string {
|
|
if (value === null || value === undefined) return ''
|
|
let text = ''
|
|
if (typeof value === 'string') {
|
|
text = value
|
|
} else {
|
|
try {
|
|
text = JSON.stringify(value)
|
|
} catch {
|
|
text = String(value)
|
|
}
|
|
}
|
|
return crypto.createHash('sha1').update(text).digest('hex')
|
|
}
|
|
|
|
export type SemiAutoTraceQuery = {
|
|
page?: number
|
|
pageSize?: number
|
|
channel?: string
|
|
status?: string
|
|
requestId?: string
|
|
keyword?: string
|
|
}
|
|
|
|
export async function querySemiAutoTraces(taskId: number, input: SemiAutoTraceQuery = {}) {
|
|
const page = Math.max(1, Math.floor(Number(input.page) || 1))
|
|
const pageSize = Math.min(200, Math.max(1, Math.floor(Number(input.pageSize) || 50)))
|
|
const conditions = ['task_id = $1']
|
|
const params: unknown[] = [taskId]
|
|
if (input.channel) {
|
|
params.push(String(input.channel))
|
|
conditions.push(`channel = $${params.length}`)
|
|
}
|
|
if (input.status) {
|
|
params.push(String(input.status))
|
|
conditions.push(`status = $${params.length}`)
|
|
}
|
|
if (input.requestId) {
|
|
params.push(String(input.requestId))
|
|
conditions.push(`request_id = $${params.length}`)
|
|
}
|
|
if (input.keyword) {
|
|
params.push(`%${String(input.keyword).trim()}%`)
|
|
conditions.push(
|
|
`(action ILIKE $${params.length} OR request_body ILIKE $${params.length} OR response_body ILIKE $${params.length})`,
|
|
)
|
|
}
|
|
const where = conditions.join(' AND ')
|
|
const count = await query(
|
|
`SELECT COUNT(*)::int AS total FROM semi_auto_trace_logs WHERE ${where}`,
|
|
params,
|
|
)
|
|
const rows = await query(
|
|
`SELECT id, order_no, request_id, direction, channel, action, status,
|
|
http_status, duration_ms, request_summary, request_body,
|
|
response_summary, response_body, error_message, created_at
|
|
FROM semi_auto_trace_logs
|
|
WHERE ${where}
|
|
ORDER BY id DESC
|
|
LIMIT $${params.length + 1} OFFSET $${params.length + 2}`,
|
|
[...params, pageSize, (page - 1) * pageSize],
|
|
)
|
|
const summary = await query(
|
|
`SELECT channel, status, COUNT(*)::int AS total
|
|
FROM semi_auto_trace_logs
|
|
WHERE task_id = $1
|
|
GROUP BY channel, status`,
|
|
[taskId],
|
|
)
|
|
return {
|
|
pagination: { page, pageSize, total: Number(count.rows[0]?.total || 0) },
|
|
summary: summary.rows,
|
|
items: rows.rows.map((row) => ({
|
|
id: Number(row.id),
|
|
orderNo: row.order_no,
|
|
requestId: row.request_id,
|
|
direction: row.direction,
|
|
channel: row.channel,
|
|
action: row.action,
|
|
status: row.status,
|
|
httpStatus: row.http_status,
|
|
durationMs: row.duration_ms,
|
|
requestSummary: row.request_summary || {},
|
|
requestBody: row.request_body || '',
|
|
responseSummary: row.response_summary || {},
|
|
responseBody: row.response_body || '',
|
|
errorMessage: row.error_message || '',
|
|
createdAt: row.created_at,
|
|
})),
|
|
}
|
|
}
|
|
|
|
export async function cleanupSemiAutoTraces(retentionDays = SEMI_AUTO_TRACE_RETENTION_DAYS) {
|
|
const days = Math.max(1, Math.floor(Number(retentionDays) || SEMI_AUTO_TRACE_RETENTION_DAYS))
|
|
const result = await query(
|
|
`DELETE FROM semi_auto_trace_logs WHERE created_at < NOW() - ($1 || ' days')::interval`,
|
|
[String(days)],
|
|
)
|
|
return result.rowCount || 0
|
|
}
|
|
|
|
let cleanupStarted = false
|
|
|
|
/** 启动时清一次,之后每 6 小时清理超过保留期的报文。 */
|
|
export function startSemiAutoTraceCleanup(): void {
|
|
if (cleanupStarted) return
|
|
cleanupStarted = true
|
|
const run = () => {
|
|
void cleanupSemiAutoTraces().catch((error) => {
|
|
logWarn('semi-auto/trace', '清理历史报文失败', {
|
|
error: error instanceof Error ? error.message : String(error),
|
|
})
|
|
})
|
|
}
|
|
run()
|
|
const timer = setInterval(run, 6 * 60 * 60 * 1000)
|
|
timer.unref?.()
|
|
}
|