fix: 免审验收容错与成本中心归一回填超时修复

- worker：等级升级计数移出验收事务，瞬时错误重试一次，最终失败转人工待办且不再返回 500 - worker：待验收阶段修改验收资料时保留 reviewMode/evidenceStatus/evidenceDueAt - cost-center：迁移 107 新增人工待归一覆盖部分索引（生产实测候选聚合 19.6s → 391ms） - cost-center：归一人工商品/按工单回填商品/模板预览前端超时放宽到 300s - cost-center：批处理接口语句超时放宽到 60s（SET LOCAL，在线接口仍为 15s）
2026-10-29 14:33:35 +08:00
parent 814b357d46
commit 979a1bfa41
13 changed files with 617 additions and 68 deletions
@@ -0,0 +1,63 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+
+import { isTransientDbError, retryOnTransientDbError } from './db-retry.js'
+
+test('statement_timeout / 死锁属于可重试的瞬时错误', () => {
+  assert.equal(isTransientDbError({ code: '57014' }), true)
+  assert.equal(isTransientDbError(new Error('canceling statement due to statement timeout')), true)
+  assert.equal(isTransientDbError({ code: '40P01' }), true)
+})
+
+test('唯一键冲突等业务错误不重试', () => {
+  assert.equal(isTransientDbError({ code: '23505', message: 'duplicate key' }), false)
+  assert.equal(isTransientDbError(new Error('bad request')), false)
+})
+
+test('瞬时错误重试一次后成功', async () => {
+  let calls = 0
+  const result = await retryOnTransientDbError(
+    async () => {
+      calls += 1
+      if (calls === 1) {
+        throw Object.assign(new Error('canceling statement due to statement timeout'), {
+          code: '57014',
+        })
+      }
+      return 'ok'
+    },
+    { attempts: 2, delayMs: 0 },
+  )
+
+  assert.equal(result, 'ok')
+  assert.equal(calls, 2)
+})
+
+test('非瞬时错误只调用一次', async () => {
+  let calls = 0
+  await assert.rejects(
+    retryOnTransientDbError(
+      async () => {
+        calls += 1
+        throw Object.assign(new Error('duplicate key'), { code: '23505' })
+      },
+      { attempts: 2, delayMs: 0 },
+    ),
+  )
+  assert.equal(calls, 1)
+})
+
+test('重试次数用尽后抛出最后一次错误', async () => {
+  let calls = 0
+  await assert.rejects(
+    retryOnTransientDbError(
+      async () => {
+        calls += 1
+        throw { code: '40P01', message: 'deadlock detected' }
+      },
+      { attempts: 2, delayMs: 0 },
+    ),
+    (error: { code?: string }) => error.code === '40P01',
+  )
+  assert.equal(calls, 2)
+})
@@ -0,0 +1,66 @@
+/**
+ * 数据库瞬时错误的重试口径。
+ *
+ * 业务写事务（比如免审核验收结算）在库被回填/批量同步压住时会撞上
+ * `statement_timeout`，这类错误整体回滚、重放是安全的；死锁/序列化冲突同理。
+ * 参数错误、唯一键冲突等非瞬时错误不能重试，否则只会放大问题。
+ */
+const TRANSIENT_DB_ERROR_CODES = new Set([
+  '57014', // query_canceled（statement_timeout）
+  '40001', // serialization_failure
+  '40P01', // deadlock_detected
+  '55P03', // lock_not_available
+  '57P01', // admin_shutdown
+  '08000', // connection_exception
+  '08003', // connection_does_not_exist
+  '08006', // connection_failure
+  '08001', // sqlclient_unable_to_establish_sqlconnection
+])
+
+export function isTransientDbError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') return false
+  const code = String((error as { code?: unknown }).code || '')
+  if (TRANSIENT_DB_ERROR_CODES.has(code)) return true
+  const message = String((error as { message?: unknown }).message || '').toLowerCase()
+  return (
+    message.includes('statement timeout') ||
+    message.includes('canceling statement due to statement timeout') ||
+    message.includes('deadlock detected') ||
+    message.includes('connection terminated unexpectedly')
+  )
+}
+
+export type DbRetryOptions = {
+  /** 总尝试次数（含首次），默认 2 次。 */
+  attempts?: number
+  /** 两次尝试之间的等待毫秒数，默认 300ms。 */
+  delayMs?: number
+  onRetry?: (error: unknown, nextAttempt: number) => void
+}
+
+export async function retryOnTransientDbError<T>(
+  fn: () => Promise<T>,
+  options: DbRetryOptions = {},
+): Promise<T> {
+  const attempts = Math.max(1, Math.floor(Number(options.attempts ?? 2) || 2))
+  const delayMs = Math.max(0, Number(options.delayMs ?? 300) || 0)
+
+  let lastError: unknown
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      return await fn()
+    } catch (error) {
+      lastError = error
+      if (attempt >= attempts || !isTransientDbError(error)) break
+      options.onRetry?.(error, attempt + 1)
+      if (delayMs > 0) await sleep(delayMs)
+    }
+  }
+  throw lastError
+}
+
+function sleep(ms: number) {
+  return new Promise<void>((resolve) => {
+    setTimeout(resolve, ms)
+  })
+}