Files

686 lines
19 KiB
TypeScript

import fs from 'node:fs/promises'
import path from 'node:path'
import process from 'node:process'
import util from 'node:util'
import { PROJECT_ROOT, runtimeConfig } from '../config/runtime.js'
import type { HttpErrorLike } from './http.js'
import { maskSecret } from './masking.js'
import { asJsonObject } from '../types/json.js'
type LogLevel = 'debug' | 'info' | 'warn' | 'error'
type LogChannel = 'app' | 'integration'
type LogDetail = unknown
type InlineFields = Record<string, string | number | boolean | null | undefined>
type ExternalHttpPacketDetail = Record<string, unknown>
type LogEntry = {
time: string
level: LogLevel
scope: string
message: string
pid: number
detail?: unknown
}
const LOG_LEVEL_PRIORITY: Record<LogLevel, number> = {
debug: 10,
info: 20,
warn: 30,
error: 40,
}
const LEVEL_LABELS: Record<LogLevel, string> = {
debug: 'DEBUG',
info: 'INFO ',
warn: 'WARN ',
error: 'ERROR',
}
const ANSI = {
reset: '\u001B[0m',
dim: '\u001B[2m',
bold: '\u001B[1m',
gray: '\u001B[90m',
cyan: '\u001B[36m',
blue: '\u001B[34m',
yellow: '\u001B[33m',
red: '\u001B[31m',
}
const DATA_ROOT = resolveDataRoot()
const LOG_DIR = path.join(DATA_ROOT, 'logs')
const ACTIVE_LOG_LEVEL = normalizeLogLevel(runtimeConfig.logging?.level)
/** integration 通道独立阈值,默认 warn(仅错误/告警落盘) */
const ACTIVE_INTEGRATION_LOG_LEVEL = normalizeLogLevel(
runtimeConfig.logging?.integrationLevel ?? 'warn',
)
const DEFAULT_LOG_RETENTION_DAYS = 30
const LOG_RETENTION_DAYS = normalizeLogRetentionDays(runtimeConfig.logging?.retentionDays)
const LEGACY_LOG_FILES = new Set(['app.log', 'integration.log'])
const SENSITIVE_KEY_PATTERN =
/token|secret|password|passwd|pwd|cookie|authorization|credential|card|cards|cardno|cardpwd|apikey|api_key|devicekey|session|x-sign|(?:^|[_-])sign(?:ature)?(?:[_-]|$)/i
const SENSITIVE_URL_PARAM_PATTERN =
/([?&](?:token|secret|password|passwd|pwd|cookie|authorization|api[_-]?key|device[_-]?key|sign|signature|code)=)([^&\s,;"']+)/gi
const SENSITIVE_JSON_FIELD_PATTERN =
/("(?:token|secret|password|passwd|pwd|cookie|authorization|credential|card|cards|cardNo|cardPwd|apiKey|api_key|deviceKey|device_key|session|x-sign|sign|signature)"\s*:\s*")([^"]*)(")/gi
const MAX_SANITIZE_DEPTH = 8
let writeQueue = Promise.resolve()
let lastCleanupDateKey = ''
export function createRequestId(prefix = 'req'): string {
return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`
}
export function logDebug(scope: unknown, message: unknown, detail: LogDetail = undefined) {
return writeLog('debug', scope, message, detail)
}
export function logInfo(scope: unknown, message: unknown, detail: LogDetail = undefined) {
return writeLog('info', scope, message, detail)
}
export function logWarn(scope: unknown, message: unknown, detail: LogDetail = undefined) {
return writeLog('warn', scope, message, detail)
}
export function logError(scope: unknown, message: unknown, detail: LogDetail = undefined) {
return writeLog('error', scope, message, detail)
}
export function logIntegration(
scope: unknown,
message: unknown,
detail: LogDetail = undefined,
{ level = 'info' }: { level?: LogLevel } = {},
) {
// 默认 integration 阈值为 warn:失败类文案即使调用方传 info 也提升,避免漏记异常
return writeLog(resolveIntegrationLogLevel(level, message), scope, message, detail, {
channel: 'integration',
})
}
function resolveIntegrationLogLevel(level: LogLevel, message: unknown): LogLevel {
const normalized = normalizeLogLevel(level)
if (normalized === 'warn' || normalized === 'error') {
return normalized
}
const text = String(message || '')
if (/失败|异常|错误|超时|拒绝|expired|fail|error|timeout|denied/i.test(text)) {
return 'error'
}
return normalized
}
export function logExternalHttpPacket(
scope: unknown,
message: unknown,
detail: ExternalHttpPacketDetail = {},
{ level = 'info' }: { level?: LogLevel } = {},
) {
return logIntegration(scope, message, normalizeExternalHttpPacketDetail(detail), { level })
}
export function logCapture(
scope: unknown,
message: unknown,
detail: LogDetail = undefined,
{ channel = 'capture', sanitize = false }: { channel?: string; sanitize?: boolean } = {},
): LogEntry | null {
const entry = {
time: new Date().toISOString(),
level: 'info' as LogLevel,
scope: String(scope || 'capture').trim() || 'capture',
message: String(message || '').trim() || '-',
pid: process.pid,
detail: normalizeLogValue(detail, sanitize !== false),
}
enqueueFileWrite(entry, resolveLogFilePath(channel, entry.time), sanitize !== false, true)
return entry
}
function writeLog(
level: LogLevel,
scope: unknown,
message: unknown,
detail: LogDetail,
{ channel = 'app' }: { channel?: LogChannel } = {},
): LogEntry | null {
const configuredLevel =
channel === 'integration' ? ACTIVE_INTEGRATION_LOG_LEVEL : ACTIVE_LOG_LEVEL
if (!shouldWriteLog(level, configuredLevel)) {
return null
}
const entry = {
time: new Date().toISOString(),
level: normalizeLogLevel(level),
scope: String(scope || 'app').trim() || 'app',
message: String(message || '').trim() || '-',
pid: process.pid,
detail: normalizeLogValue(detail),
}
writeConsole(entry)
enqueueFileWrite(entry, resolveLogFilePath(channel, entry.time), true)
return entry
}
function enqueueFileWrite(
entry: LogEntry,
filePath: string,
sanitize: boolean,
jsonLine = false,
): void {
const line = jsonLine ? JSON.stringify(entry) : formatLogEntry(entry, { color: false, sanitize })
const dateKey = extractLogDateKey(entry.time)
writeQueue = writeQueue
.then(async () => {
await fs.mkdir(LOG_DIR, { recursive: true })
await fs.appendFile(filePath, `${line}\n`, 'utf8')
await cleanupExpiredLogsIfNeeded(dateKey)
})
.catch((error) => {
const reason = error instanceof Error ? error.message : String(error || '未知错误')
console.error('[logger] failed to write log file:', reason)
})
}
function writeConsole(entry: LogEntry): void {
const logger = resolveConsoleMethod(entry.level)
logger(formatLogEntry(entry, { color: supportsAnsiColor() }))
}
function resolveConsoleMethod(level: unknown) {
const normalized = normalizeLogLevel(level)
if (normalized === 'error') {
return console.error
}
if (normalized === 'warn') {
return console.warn
}
return console.log
}
export function normalizeLogLevel(level: unknown): LogLevel {
const normalized = String(level || '')
.trim()
.toLowerCase()
return isLogLevel(normalized) ? normalized : 'info'
}
export function shouldWriteLog(
level: unknown,
configuredLevel: unknown = ACTIVE_LOG_LEVEL,
): boolean {
const normalizedLevel = normalizeLogLevel(level)
const normalizedConfigured = normalizeLogLevel(configuredLevel)
return LOG_LEVEL_PRIORITY[normalizedLevel] >= LOG_LEVEL_PRIORITY[normalizedConfigured]
}
export function formatLogEntry(
entry: Partial<LogEntry>,
{ color = false, sanitize = true }: { color?: boolean; sanitize?: boolean } = {},
): string {
const normalizedEntry = {
time: String(entry?.time || new Date().toISOString()),
level: normalizeLogLevel(entry?.level),
scope: String(entry?.scope || 'app').trim() || 'app',
message: String(entry?.message || '').trim() || '-',
pid: Number(entry?.pid || process.pid),
detail: sanitize !== false ? sanitizeLogValue(entry?.detail) : entry?.detail,
}
const timestamp = colorize(formatLogTimestamp(normalizedEntry.time), ANSI.gray, color)
const level = colorize(
LEVEL_LABELS[normalizedEntry.level],
resolveLevelColor(normalizedEntry.level),
color,
)
const scope = colorize(normalizedEntry.scope, ANSI.cyan, color)
const message = colorize(normalizedEntry.message, ANSI.bold, color)
const separator = colorize('|', ANSI.dim, color)
const { inline, block } = splitDetailPayload(normalizedEntry.detail)
const inlineFields = formatInlineFields({
pid: normalizedEntry.pid,
...inline,
})
const firstLine = [timestamp, level, scope, message, separator, inlineFields]
.filter(Boolean)
.join(' ')
if (!block) {
return firstLine
}
const inspected = util.inspect(block, {
depth: 8,
colors: color,
compact: false,
breakLength: 120,
maxArrayLength: 100,
})
return `${firstLine}\n${indentBlock(inspected, ' ')}`
}
export function resolveLogFilePath(channel = 'app', time = new Date().toISOString()): string {
const normalizedChannel = normalizeLogChannelName(channel)
return path.join(LOG_DIR, `${normalizedChannel}-${extractLogDateKey(time)}.log`)
}
function normalizeLogChannelName(value: unknown): string {
const normalized = String(value || '')
.trim()
.toLowerCase()
if (!normalized) {
return 'app'
}
if (normalized === 'integration') {
return 'integration'
}
return /^[a-z0-9-]{1,40}$/.test(normalized) ? normalized : 'app'
}
export function resolveExpiredLogFilenames(
fileNames: string[] = [],
referenceTime = new Date().toISOString(),
retentionDays = LOG_RETENTION_DAYS,
): string[] {
const normalizedRetentionDays = normalizeLogRetentionDays(retentionDays)
const cutoffDateKey = toDateKey(offsetDate(referenceTime, -(normalizedRetentionDays - 1)))
return (Array.isArray(fileNames) ? fileNames : []).filter((fileName) =>
shouldDeleteLogFileByDate(fileName, cutoffDateKey),
)
}
export function formatLogTimestamp(value: unknown): string {
const date = new Date(value instanceof Date ? value : String(value || ''))
if (Number.isNaN(date.getTime())) {
return String(value || '').trim() || new Date().toISOString()
}
return `${formatLocalDate(date)} ${formatLocalTime(date)}.${String(date.getMilliseconds()).padStart(3, '0')} ${formatTimezoneOffset(date)}`
}
function resolveDataRoot(): string {
const configured = String(runtimeConfig.data?.root || '').trim()
return configured ? path.resolve(configured) : path.resolve(PROJECT_ROOT, 'data')
}
function normalizeLogValue(value: unknown, sanitize = true): unknown {
if (typeof value === 'undefined') {
return undefined
}
const normalizeCurrent = (current: unknown): unknown => {
if (current instanceof Error) {
const currentError = current as Error & HttpErrorLike
return {
name: currentError.name,
message: sanitize ? sanitizeLogString(currentError.message) : currentError.message,
stack: sanitize ? sanitizeLogString(currentError.stack) : currentError.stack,
statusCode: currentError.statusCode,
errorCode: currentError.errorCode,
context: sanitize ? sanitizeLogValue(currentError.context) : currentError.context,
}
}
if (typeof current === 'bigint') {
return String(current)
}
return current
}
try {
return sanitize
? sanitizeLogValue(
JSON.parse(JSON.stringify(value, (_key, current) => normalizeCurrent(current))),
)
: JSON.parse(JSON.stringify(value, (_key, current) => normalizeCurrent(current)))
} catch {
return sanitize ? sanitizeLogString(String(value)) : String(value)
}
}
function sanitizeLogValue(value: unknown, key = '', depth = 0): unknown {
if (typeof value === 'undefined') {
return undefined
}
if (value === null) {
return null
}
if (isSensitiveLogKey(key)) {
return maskSecret(value)
}
if (typeof value === 'string') {
return sanitizeLogString(value)
}
if (typeof value === 'number' || typeof value === 'boolean') {
return value
}
if (typeof value === 'bigint') {
return String(value)
}
if (depth >= MAX_SANITIZE_DEPTH) {
return '[MaxDepth]'
}
if (Array.isArray(value)) {
return value.map((item) => sanitizeLogValue(item, key, depth + 1))
}
if (value && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value).map(([currentKey, currentValue]) => [
currentKey,
sanitizeLogValue(currentValue, currentKey, depth + 1),
]),
)
}
return sanitizeLogString(String(value))
}
function sanitizeLogString(value: unknown): string {
return String(value || '')
.replace(
/(Bearer\s+)([A-Za-z0-9._~+/=-]+)/gi,
(_matched, prefix, secret) => `${prefix}${maskSecret(secret)}`,
)
.replace(
SENSITIVE_URL_PARAM_PATTERN,
(_matched, prefix, secret) => `${prefix}${maskSecret(secret)}`,
)
.replace(
SENSITIVE_JSON_FIELD_PATTERN,
(_matched, prefix, secret, suffix) => `${prefix}${maskSecret(secret)}${suffix}`,
)
.replace(
/\b((?:token|secret|password|passwd|pwd|cookie|authorization|api[_-]?key|device[_-]?key)=)([^&\s,;]+)/gi,
(_matched, prefix, secret) => `${prefix}${maskSecret(secret)}`,
)
}
function isSensitiveLogKey(key: unknown): boolean {
return SENSITIVE_KEY_PATTERN.test(String(key || '').trim())
}
function normalizeExternalHttpPacketDetail(
detail: ExternalHttpPacketDetail,
): ExternalHttpPacketDetail {
const source = asJsonObject(detail)
const normalized: ExternalHttpPacketDetail = {}
for (const [key, value] of Object.entries(source)) {
normalized[key] = normalizeExternalPacketValue(value)
}
return normalized
}
function normalizeExternalPacketValue(value: unknown): unknown {
if (typeof value === 'undefined' || value === null) {
return value
}
if (value instanceof Error) {
const currentError = value as Error & HttpErrorLike
return {
name: value.name,
message: value.message,
stack: value.stack,
statusCode: currentError.statusCode,
errorCode: currentError.errorCode,
context: normalizeExternalPacketValue(currentError.context),
}
}
if (typeof value === 'string') {
return value
}
if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint') {
return value
}
if (Array.isArray(value)) {
return value.map((item) => normalizeExternalPacketValue(item))
}
if (value && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value).map(([key, currentValue]) => [
key,
normalizeExternalPacketValue(currentValue),
]),
)
}
return String(value)
}
function splitDetailPayload(detail: unknown): { inline: InlineFields; block: unknown | null } {
if (typeof detail === 'undefined') {
return {
inline: {},
block: null,
}
}
if (isScalarLogValue(detail)) {
return {
inline: { detail },
block: null,
}
}
if (Array.isArray(detail)) {
return {
inline: {},
block: detail,
}
}
if (detail && typeof detail === 'object') {
const inline: InlineFields = {}
const block: Record<string, unknown> = {}
for (const [key, value] of Object.entries(detail)) {
if (isScalarLogValue(value)) {
inline[key] = value
continue
}
block[key] = value
}
return {
inline,
block: Object.keys(block).length > 0 ? block : null,
}
}
return {
inline: { detail: String(detail) },
block: null,
}
}
function formatInlineFields(fields: InlineFields): string {
return Object.entries(fields)
.filter(([, value]) => typeof value !== 'undefined' && value !== '')
.map(([key, value]) => `${key}=${formatInlineValue(value)}`)
.join(' ')
}
function formatInlineValue(value: unknown): string {
if (value === null) {
return 'null'
}
if (typeof value === 'number' || typeof value === 'boolean') {
return String(value)
}
const text = String(value || '')
return /^[A-Za-z0-9_./:@-]+$/.test(text) ? text : JSON.stringify(text)
}
function isScalarLogValue(value: unknown): value is string | number | boolean | null {
return value === null || ['string', 'number', 'boolean'].includes(typeof value)
}
function indentBlock(text: unknown, indent: string): string {
return String(text || '')
.split('\n')
.map((line) => `${indent}${line}`)
.join('\n')
}
function supportsAnsiColor(): boolean {
return process.env.NO_COLOR !== '1' && Boolean(process.stdout.isTTY || process.stderr.isTTY)
}
function resolveLevelColor(level: unknown): string {
switch (normalizeLogLevel(level)) {
case 'debug':
return ANSI.blue
case 'warn':
return ANSI.yellow
case 'error':
return ANSI.red
default:
return ANSI.cyan
}
}
function colorize(text: string, ansiCode: string, enabled: boolean): string {
if (!enabled || !ansiCode) {
return text
}
return `${ansiCode}${text}${ANSI.reset}`
}
function formatLocalDate(date: Date): string {
return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`
}
function formatLocalTime(date: Date): string {
return `${String(date.getHours()).padStart(2, '0')}:${String(date.getMinutes()).padStart(2, '0')}:${String(date.getSeconds()).padStart(2, '0')}`
}
function formatTimezoneOffset(date: Date): string {
const totalMinutes = -date.getTimezoneOffset()
const sign = totalMinutes >= 0 ? '+' : '-'
const absoluteMinutes = Math.abs(totalMinutes)
const hours = String(Math.floor(absoluteMinutes / 60)).padStart(2, '0')
const minutes = String(absoluteMinutes % 60).padStart(2, '0')
return `${sign}${hours}:${minutes}`
}
async function cleanupExpiredLogsIfNeeded(dateKey: string): Promise<void> {
if (!dateKey || dateKey === lastCleanupDateKey) {
return
}
lastCleanupDateKey = dateKey
try {
const fileNames = await fs.readdir(LOG_DIR)
const expired = resolveExpiredLogFilenames(fileNames, dateKey, LOG_RETENTION_DAYS)
await Promise.all(
expired.map((fileName) => fs.rm(path.join(LOG_DIR, fileName), { force: true })),
)
} catch (error) {
const reason = error instanceof Error ? error.message : String(error || '未知错误')
console.error('[logger] failed to cleanup old log files:', reason)
}
}
function extractLogDateKey(time: unknown): string {
return toDateKey(time)
}
function toDateKey(value: unknown): string {
const date = parseLogDate(value)
if (Number.isNaN(date.getTime())) {
return formatLocalDate(new Date())
}
return formatLocalDate(date)
}
function offsetDate(value: unknown, offsetDays: unknown): Date {
const date = parseLogDate(value)
if (Number.isNaN(date.getTime())) {
return new Date()
}
date.setDate(date.getDate() + Number(offsetDays || 0))
return date
}
function parseLogDate(value: unknown): Date {
if (value instanceof Date) {
return new Date(value.getTime())
}
const text = String(value || '').trim()
const matchedDateKey = /^(\d{4})-(\d{2})-(\d{2})$/.exec(text)
if (matchedDateKey) {
return new Date(
Number(matchedDateKey[1]),
Number(matchedDateKey[2]) - 1,
Number(matchedDateKey[3]),
)
}
return new Date(text)
}
function normalizeLogRetentionDays(value: unknown): number {
const parsed = Number(value)
if (!Number.isFinite(parsed)) {
return DEFAULT_LOG_RETENTION_DAYS
}
return Math.max(1, Math.floor(parsed))
}
function shouldDeleteLogFileByDate(fileName: unknown, cutoffDateKey: string): boolean {
if (LEGACY_LOG_FILES.has(String(fileName || '').trim())) {
return true
}
const matched = /^([a-z0-9-]{1,40})-(\d{4}-\d{2}-\d{2})\.log$/.exec(String(fileName || '').trim())
if (!matched) {
return false
}
return String(matched[2] || '') < cutoffDateKey
}
function isLogLevel(value: string): value is LogLevel {
return ['debug', 'info', 'warn', 'error'].includes(value)
}