优化：降低访问日志噪声并增加日志文件轮转

2026-08-30 16:59:11 +08:00
parent 6ca54e468e
commit 25b89bb503
20 changed files with 2334 additions and 4 deletions
@@ -40,6 +40,9 @@ npm test
 | --- | --- |
 | `PORT` | 服务端口（Compose 中常由 `BACKEND_PORT` 映射） |
 | `LOG_LEVEL` | `debug` / `info` / `warn` / `error` |
+| `LOG_MAX_FILE_SIZE_MB` | 单个应用日志文件大小上限，默认 `20` MB，超出后轮转 |
+| `LOG_MAX_FILES` | 同一日期最多保留的日志文件数量，默认 `5`（含当前文件） |
+| `LOG_RETENTION_DAYS` | 日志日期保留天数，默认 `30` |
 | `DATABASE_URL` | PostgreSQL 连接串 |
 | `ADMIN_SESSION_SECRET` | 后台登录态签名密钥 |
 | `ADMIN_DEFAULT_USERS_JSON` | 默认后台用户 |
@@ -130,6 +133,6 @@ docker compose -f ../../docker-compose.dev.yml exec -T backend npm run db:migrat
 - `data/logs/app-YYYY-MM-DD.log`
 - `data/logs/integration-YYYY-MM-DD.log`

-按天切分，默认清理过期日志。成功的 `/health*` 探活默认不写 access log。
+按天切分并按大小轮转，默认单文件 20MB、单日最多 5 个分片、保留 30 天。成功的 `GET/HEAD` 请求以 `DEBUG` 记录，生产默认 `LOG_LEVEL=info` 不落盘；写请求和所有 4xx/5xx 仍保留 access log。成功的 `/health*` 探活默认不写 access log。

 `data/*.json` 为本地运行配置，可能含敏感信息，默认不提交；请从 `*.example.json` 复制后填写。
@@ -19,6 +19,8 @@ export function createDefaultRuntimeConfig(projectRoot: string): RuntimeConfig {
      // 对接日志默认只保留 warn/error，避免 integration-*.log 被 info 刷爆
      integrationLevel: 'warn',
      retentionDays: 30,
+      maxFileSizeMb: 20,
+      maxFiles: 5,
    },

    retention: {
@@ -13,6 +13,8 @@ test('applyEnvOverrides maps typed environment values without mutating base conf
    DATA_ROOT: './tmp/backend-data',
    DATABASE_SSL: 'yes',
    LOG_RETENTION_DAYS: '45',
+    LOG_MAX_FILE_SIZE_MB: '12',
+    LOG_MAX_FILES: '4',
    ADMIN_DEFAULT_USERS_JSON: '[{"username":"admin","password":"secret","role":"admin"}]',
    KAQUAN91_USER_ID: 'kaquan-user',
    KAQUAN91_SECRET: 'kaquan-secret',
@@ -23,6 +25,8 @@ test('applyEnvOverrides maps typed environment values without mutating base conf
  assert.equal(config.data.root, path.resolve('./tmp/backend-data'))
  assert.equal(config.database.ssl, true)
  assert.equal(config.logging.retentionDays, 45)
+  assert.equal(config.logging.maxFileSizeMb, 12)
+  assert.equal(config.logging.maxFiles, 4)
  assert.deepEqual(config.admin.defaultUsers, [
    { username: 'admin', password: 'secret', role: 'admin' },
  ])
@@ -33,6 +33,8 @@ export const ENV_OVERRIDES: readonly EnvOverride[] = [
  stringEnv('LOG_LEVEL', ['logging', 'level']),
  stringEnv('LOG_INTEGRATION_LEVEL', ['logging', 'integrationLevel']),
  integerEnv('LOG_RETENTION_DAYS', ['logging', 'retentionDays']),
+  integerEnv('LOG_MAX_FILE_SIZE_MB', ['logging', 'maxFileSizeMb']),
+  integerEnv('LOG_MAX_FILES', ['logging', 'maxFiles']),
  integerEnv('RAW_PAYLOAD_RETENTION_DAYS', ['retention', 'rawPayloadDays']),
  integerEnv('TASK_EVENT_RETENTION_DAYS', ['retention', 'taskEventDays']),
  integerEnv('WEBHOOK_EVENT_RETENTION_DAYS', ['retention', 'webhookEventDays']),
@@ -52,6 +52,8 @@ export function validateRuntimeConfig(
    })
  }
  requireInteger(issues, 'database.maxConnections', config.database?.maxConnections, { min: 1 })
+  requireOptionalInteger(issues, 'logging.maxFileSizeMb', config.logging?.maxFileSizeMb, { min: 1 })
+  requireOptionalInteger(issues, 'logging.maxFiles', config.logging?.maxFiles, { min: 2 })
  requireOptionalInteger(issues, 'database.idleTimeoutMs', config.database?.idleTimeoutMs, {
    min: 1,
  })
@@ -0,0 +1,15 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+
+import { resolveAccessLogLevel } from './access-log.js'
+
+test('resolveAccessLogLevel sends successful reads to debug', () => {
+  assert.equal(resolveAccessLogLevel('GET', 200), 'debug')
+  assert.equal(resolveAccessLogLevel('head', 304), 'debug')
+})
+
+test('resolveAccessLogLevel keeps writes and failures at info', () => {
+  assert.equal(resolveAccessLogLevel('POST', 200), 'info')
+  assert.equal(resolveAccessLogLevel('GET', 404), 'info')
+  assert.equal(resolveAccessLogLevel('GET', 500), 'info')
+})
@@ -1,5 +1,5 @@
 import type { Request, Response, NextFunction } from 'express'
-import { createRequestId, logInfo } from '../utils/logger.js'
+import { createRequestId, logDebug, logInfo } from '../utils/logger.js'

 export function accessLogMiddleware(req: Request, res: Response, next: NextFunction): void {
  const startedAt = Date.now()
@@ -13,7 +13,9 @@ export function accessLogMiddleware(req: Request, res: Response, next: NextFunct
      return
    }

-    logInfo('[http/access]', 'request completed', {
+    const writeLog =
+      resolveAccessLogLevel(req.method, res.statusCode) === 'debug' ? logDebug : logInfo
+    writeLog('[http/access]', 'request completed', {
      requestId,
      method: req.method,
      originalUrl: req.originalUrl,
@@ -33,6 +35,18 @@ export function accessLogMiddleware(req: Request, res: Response, next: NextFunct
  next()
 }

+export function resolveAccessLogLevel(method: unknown, statusCode: unknown): 'debug' | 'info' {
+  const normalizedMethod = String(method || '')
+    .trim()
+    .toUpperCase()
+  const normalizedStatus = Number(statusCode || 0)
+  return ['GET', 'HEAD'].includes(normalizedMethod) &&
+    normalizedStatus >= 200 &&
+    normalizedStatus < 400
+    ? 'debug'
+    : 'info'
+}
+
 function resolveRequestId(req: Request): string {
  const fromHeader = String(req.headers['x-request-id'] || '').trim()
  return fromHeader || createRequestId('req')
@@ -31,6 +31,10 @@ export type RuntimeConfig = {
    /** integration 通道（外部对接日志文件）最低级别，默认 warn */
    integrationLevel: string
    retentionDays: number
+    /** 单个日志文件大小上限（MB），超出后轮转。 */
+    maxFileSizeMb?: number
+    /** 同一日期最多保留的日志文件数量（含当前文件）。 */
+    maxFiles?: number
  }
  retention: {
    rawPayloadDays: number
@@ -77,6 +77,16 @@ test('resolveExpiredLogFilenames supports explicit retention days', () => {
  assert.deepEqual(expired, ['app-2026-04-07.log'])
 })

+test('resolveExpiredLogFilenames includes rotated files', () => {
+  const expired = resolveExpiredLogFilenames(
+    ['app-2026-04-14.1.log', 'app-2026-04-07.5.log', 'app-2026-04-14.log'],
+    '2026-04-14',
+    7,
+  )
+
+  assert.deepEqual(expired, ['app-2026-04-07.5.log'])
+})
+
 test('formatLogTimestamp renders readable local timestamp with timezone offset', () => {
  const rendered = formatLogTimestamp('2026-04-14T10:36:24.974Z')
  assert.match(rendered, /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} [+-]\d{2}:\d{2}$/)
@@ -56,6 +56,11 @@ const ACTIVE_INTEGRATION_LOG_LEVEL = normalizeLogLevel(
 )
 const DEFAULT_LOG_RETENTION_DAYS = 30
 const LOG_RETENTION_DAYS = normalizeLogRetentionDays(runtimeConfig.logging?.retentionDays)
+const DEFAULT_LOG_MAX_FILE_SIZE_MB = 20
+const DEFAULT_LOG_MAX_FILES = 5
+const LOG_MAX_FILE_SIZE_BYTES =
+  normalizeLogMaxFileSizeMb(runtimeConfig.logging?.maxFileSizeMb) * 1024 * 1024
+const LOG_MAX_FILES = normalizeLogMaxFiles(runtimeConfig.logging?.maxFiles)
 const LEGACY_LOG_FILES = new Set(['app.log', 'integration.log'])
 const SENSITIVE_KEY_PATTERN =
  /token|secret|password|passwd|pwd|cookie|authorization|credential|card|cards|cardno|cardpwd|apikey|api_key|devicekey|session|x-sign|(?:^|[_-])sign(?:ature)?(?:[_-]|$)/i
@@ -181,6 +186,7 @@ function enqueueFileWrite(
  writeQueue = writeQueue
    .then(async () => {
      await fs.mkdir(LOG_DIR, { recursive: true })
+      await rotateLogFileIfNeeded(filePath, Buffer.byteLength(`${line}\n`, 'utf8'))
      await fs.appendFile(filePath, `${line}\n`, 'utf8')
      await cleanupExpiredLogsIfNeeded(dateKey)
    })
@@ -666,12 +672,78 @@ function normalizeLogRetentionDays(value: unknown): number {
  return Math.max(1, Math.floor(parsed))
 }

+function normalizeLogMaxFileSizeMb(value: unknown): number {
+  const parsed = Number(value)
+  if (!Number.isFinite(parsed)) {
+    return DEFAULT_LOG_MAX_FILE_SIZE_MB
+  }
+
+  return Math.max(1, Math.floor(parsed))
+}
+
+function normalizeLogMaxFiles(value: unknown): number {
+  const parsed = Number(value)
+  if (!Number.isFinite(parsed)) {
+    return DEFAULT_LOG_MAX_FILES
+  }
+
+  return Math.max(2, Math.floor(parsed))
+}
+
+async function rotateLogFileIfNeeded(filePath: string, nextLineBytes: number): Promise<void> {
+  if (!filePath.endsWith('.log') || LOG_MAX_FILE_SIZE_BYTES <= 0) {
+    return
+  }
+
+  let currentSize = 0
+  try {
+    currentSize = (await fs.stat(filePath)).size
+  } catch (error) {
+    if (isFileMissingError(error)) {
+      return
+    }
+    throw error
+  }
+
+  if (currentSize === 0 || currentSize + nextLineBytes <= LOG_MAX_FILE_SIZE_BYTES) {
+    return
+  }
+
+  for (let index = LOG_MAX_FILES - 1; index >= 1; index -= 1) {
+    await renameIfPresent(
+      rotatedLogFilePath(filePath, index),
+      rotatedLogFilePath(filePath, index + 1),
+    )
+  }
+  await renameIfPresent(filePath, rotatedLogFilePath(filePath, 1))
+}
+
+function rotatedLogFilePath(filePath: string, index: number): string {
+  return filePath.replace(/\.log$/, `.${index}.log`)
+}
+
+async function renameIfPresent(source: string, target: string): Promise<void> {
+  try {
+    await fs.rename(source, target)
+  } catch (error) {
+    if (!isFileMissingError(error)) {
+      throw error
+    }
+  }
+}
+
+function isFileMissingError(error: unknown): boolean {
+  return Boolean(error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')
+}
+
 function shouldDeleteLogFileByDate(fileName: unknown, cutoffDateKey: string): boolean {
  if (LEGACY_LOG_FILES.has(String(fileName || '').trim())) {
    return true
  }

-  const matched = /^([a-z0-9-]{1,40})-(\d{4}-\d{2}-\d{2})\.log$/.exec(String(fileName || '').trim())
+  const matched = /^([a-z0-9-]{1,40})-(\d{4}-\d{2}-\d{2})(?:\.\d+)?\.log$/.exec(
+    String(fileName || '').trim(),
+  )

  if (!matched) {
    return false