优化接单平台工单查询性能

2026-09-04 21:22:35 +08:00
parent 625df52992
commit 84a8f138d4
7 changed files with 158 additions and 82 deletions
@@ -10,7 +10,12 @@ import { fileURLToPath } from 'node:url'
 const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url))
 const BACKEND_ROOT = path.resolve(SCRIPT_DIR, '..')

-const checks: Array<{ name: string; file: string; patterns: string[] }> = [
+const checks: Array<{
+  name: string
+  file: string
+  patterns?: string[]
+  forbiddenPatterns?: string[]
+}> = [
  {
    name: '慢查询阈值配置',
    file: path.join(BACKEND_ROOT, 'src/config/env-overrides.ts'),
@@ -35,6 +40,12 @@ const checks: Array<{ name: string; file: string; patterns: string[] }> = [
    file: path.join(BACKEND_ROOT, 'src/repositories/worker-platform/work-order-share-repo.ts'),
    patterns: ['workOrderIds: number[]', 'wos.work_order_id = ANY($2::bigint[])'],
  },
+  {
+    name: '接单工单宽详情分页禁止窗口总数',
+    file: path.join(BACKEND_ROOT, 'src/repositories/worker-platform/work-order-query-repo.ts'),
+    patterns: ['countWorkOrdersWithWhere', 'includeTotal'],
+    forbiddenPatterns: ['COUNT(*) OVER' + '()', 'LIST_WORK_ORDER_SELECT'],
+  },
  {
    name: '数据保留清理扫描索引',
    file: path.join(BACKEND_ROOT, 'src/db/migrations/067_data_retention_indexes.sql'),
@@ -55,10 +66,27 @@ const checks: Array<{ name: string; file: string; patterns: string[] }> = [
      "WHERE event_type LIKE 'timeout_%'",
    ],
  },
+  {
+    name: '接单工单搜索表达式索引重建',
+    file: path.join(BACKEND_ROOT, 'src/db/migrations/073_rebuild_work_order_search_index.sql'),
+    patterns: [
+      'DROP INDEX IF EXISTS idx_work_orders_search_text_trgm',
+      'CREATE INDEX idx_work_orders_search_text_trgm',
+      "material_json->'collect'->'fields'->>'gameId'",
+    ],
+  },
 ]

 const failures: string[] = []

+// 窗口总数会在 LIMIT/OFFSET 之前消费全部候选行；全后端 SQL 都禁止引入该写法。
+const countWindowPattern = /COUNT\s*\(\s*\*\s*\)\s*OVER\s*\(/i
+for (const file of listSourceFiles(path.join(BACKEND_ROOT, 'src'))) {
+  if (countWindowPattern.test(fs.readFileSync(file, 'utf8'))) {
+    failures.push(`禁止 COUNT(*) OVER() 窗口分页: ${file}`)
+  }
+}
+
 // 已应用的历史迁移必须保持字节不变，所有结构或数据修复都必须新增迁移文件。
 try {
  const diffOutputs = [
@@ -89,11 +117,16 @@ for (const check of checks) {
    continue
  }
  const source = fs.readFileSync(check.file, 'utf8')
-  for (const pattern of check.patterns) {
+  for (const pattern of check.patterns || []) {
    if (!source.includes(pattern)) {
      failures.push(`${check.name}: 缺少约束 ${pattern} (${check.file})`)
    }
  }
+  for (const pattern of check.forbiddenPatterns || []) {
+    if (source.includes(pattern)) {
+      failures.push(`${check.name}: 出现禁止 SQL ${pattern} (${check.file})`)
+    }
+  }
 }

 if (failures.length > 0) {
@@ -103,3 +136,16 @@ if (failures.length > 0) {
 } else {
  console.info('[sql-guard] SQL 约束检查通过')
 }
+
+function listSourceFiles(directory: string): string[] {
+  const files: string[] = []
+  for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
+    const file = path.join(directory, entry.name)
+    if (entry.isDirectory()) {
+      files.push(...listSourceFiles(file))
+    } else if (entry.isFile() && /\.(?:ts|sql)$/.test(entry.name)) {
+      files.push(file)
+    }
+  }
+  return files
+}
@@ -0,0 +1,23 @@
+-- 修复接单工单搜索表达式索引漂移。
+-- 部署中曾存在索引表达式与当前查询不一致的数据库；新增迁移确保已执行 031 的环境也会重建。
+DROP INDEX IF EXISTS idx_work_orders_search_text_trgm;
+
+CREATE INDEX idx_work_orders_search_text_trgm
+  ON work_orders USING gin (
+    LOWER(
+      COALESCE(work_order_no, '') || ' ' ||
+      COALESCE(platform_order_id, '') || ' ' ||
+      COALESCE(product_name, '') || ' ' ||
+      COALESCE(material_json->'collect'->'fields'->>'gameId', '') || ' ' ||
+      COALESCE(material_json->'collect'->'fields'->>'gameNickname', '') || ' ' ||
+      COALESCE(material_json->'collect'->'fields'->>'gameAccount', '') || ' ' ||
+      COALESCE(material_json->'collect'->'fields'->>'roleName', '') || ' ' ||
+      COALESCE(material_json->'fields'->>'gameId', '') || ' ' ||
+      COALESCE(material_json->'fields'->>'gameNickname', '') || ' ' ||
+      COALESCE(material_json->>'gameId', '') || ' ' ||
+      COALESCE(material_json->>'gameNickname', '')
+    ) gin_trgm_ops
+  );
+
+COMMENT ON INDEX idx_work_orders_search_text_trgm IS
+  '接单工单模糊搜索：订单号、商品名、游戏 ID/昵称及历史字段';
@@ -418,6 +418,8 @@ export type GrabWorkOrderResult = {
 export type ListInput = {
  page?: number
  pageSize?: number
+  /** 列表调用方已有同筛选条件的聚合总数时，可跳过独立 COUNT。 */
+  includeTotal?: boolean
  /** 单状态筛选（兼容 worker 用户等其他列表） */
  status?: string
  /** 支持多个状态同时筛选（空数组表示全部状态） */
@@ -58,13 +58,6 @@ export const WORK_ORDER_SELECT = `
  ) share_summary ON TRUE
 `

-// Keep the page and total in one filtered query. This avoids running the same
-// joins and share aggregation twice for every list request.
-const LIST_WORK_ORDER_SELECT = WORK_ORDER_SELECT.replace(
-  '  SELECT\n',
-  '  SELECT\n    COUNT(*) OVER()::int AS total_count,\n',
-)
-
 type WorkerOrderOverviewRow = {
  status: string
  order_count: number
@@ -234,6 +227,7 @@ export async function listWorkOrdersByPlatformOrderId(
 export async function listWorkOrders({
  page = 1,
  pageSize = 20,
+  includeTotal = true,
  statuses = [],
  status = '',
  workOrderId = 0,
@@ -271,10 +265,13 @@ export async function listWorkOrders({
    vipEvidencePending,
    acceptanceMode,
    giftPhase,
-    useOperationalStatus,
+    // 无状态筛选时运营状态不影响列表入选，不应为此聚合整张拼单表。
+    useOperationalStatus:
+      useOperationalStatus && (effectiveStatuses.length > 0 || vipEvidencePending),
    createdFrom,
    createdTo,
  })
+  const countParams = [...params]
  const offset = (page - 1) * pageSize
  const orderBy =
    sort === 'worker_claimed_at_desc' && workerSharingId
@@ -289,33 +286,42 @@ export async function listWorkOrders({
      : 'wo.id DESC'
  const effectiveOrderBy = pinnedFirst ? `wo.pinned_at DESC NULLS LAST, ${orderBy}` : orderBy
  params.push(pageSize, offset)
-  const itemsResult = await query<WorkOrderRow>(
-    `${LIST_WORK_ORDER_SELECT}
+  // 宽详情只查当前页；不将 COUNT 窗口放进详情查询，避免 LIMIT 前处理全量 JSON。
+  const [itemsResult, total] = await Promise.all([
+    query<WorkOrderRow>(
+      `${WORK_ORDER_SELECT}
       ${joins}
       ${whereClause}
       ORDER BY ${effectiveOrderBy}
       LIMIT $${params.length - 1} OFFSET $${params.length}`,
      params,
-  )
-  // A page beyond the end has no row from which to read the window count.
-  // Keep that uncommon pagination edge case correct without restoring a
-  // second query for normal requests.
-  let total = Number(
-    (itemsResult.rows[0] as WorkOrderRow & { total_count?: number })?.total_count || 0,
-  )
-  if (itemsResult.rows.length === 0 && page > 1) {
-    const totalResult = await query<{ total: number }>(
-      `SELECT COUNT(*)::int AS total FROM work_orders wo ${joins} ${whereClause}`,
-      params.slice(0, -2),
-    )
-    total = Number(totalResult.rows[0]?.total || 0)
-  }
+    ),
+    includeTotal ? countWorkOrdersWithWhere(joins, whereClause, countParams) : Promise.resolve(0),
+  ])
  return {
    items: itemsResult.rows,
    total,
  }
 }

+/** 只返回筛选范围总数，不加载工单 JSON、取消事件和当前页关联字段。 */
+export async function countWorkOrders(input: ListInput = {}): Promise<number> {
+  const { joins, whereClause, params } = buildWorkOrderWhere(input)
+  return countWorkOrdersWithWhere(joins, whereClause, params)
+}
+
+async function countWorkOrdersWithWhere(
+  joins: string,
+  whereClause: string,
+  params: unknown[],
+): Promise<number> {
+  const result = await query<{ total: number }>(
+    `SELECT COUNT(*)::int AS total FROM work_orders wo ${joins} ${whereClause}`,
+    params,
+  )
+  return Number(result.rows[0]?.total || 0)
+}
+
 /** 按筛选条件汇总工单，统计范围不受当前分页影响。 */
 export async function getWorkOrderStatistics(input: ListInput = {}): Promise<WorkOrderStatistics> {
  const { joins, whereClause, params } = buildWorkOrderWhere(input)
@@ -55,12 +55,15 @@ export async function listAdminWorkOrders(query: JsonObject = {}) {
    ...(createdFrom ? { createdFrom } : {}),
    ...(createdTo ? { createdTo } : {}),
  }
-  const { items, total } = await listWorkOrders({ page, pageSize, ...filters })
-  const statistics = await getWorkOrderStatistics(filters)
-  const pendingUnfreezeByOrderId = await sumPendingUnfreezeByOrderIds(
-    items.map((item) => Number(item.id)),
-  )
-  const shares = await listWorkOrderSharesByOrderIds(items.map((item) => Number(item.id)))
+  const [{ items }, statistics] = await Promise.all([
+    listWorkOrders({ page, pageSize, includeTotal: false, ...filters }),
+    getWorkOrderStatistics(filters),
+  ])
+  const itemIds = items.map((item) => Number(item.id))
+  const [pendingUnfreezeByOrderId, shares] = await Promise.all([
+    sumPendingUnfreezeByOrderIds(itemIds),
+    listWorkOrderSharesByOrderIds(itemIds),
+  ])
  const sharesByOrderId = new Map<number, typeof shares>()
  for (const share of shares) {
    const orderId = Number(share.work_order_id)
@@ -73,7 +76,7 @@ export async function listAdminWorkOrders(query: JsonObject = {}) {
      ...mapWorkOrderAdmin(item, sharesByOrderId.get(Number(item.id)) || []),
      pendingUnfreezeAmount: pendingUnfreezeByOrderId.get(Number(item.id)) || 0,
    })),
-    pagination: { page, pageSize, total },
+    pagination: { page, pageSize, total: statistics.totalCount },
    statistics,
  }
 }
@@ -1,57 +1,52 @@
 import { WORK_ORDER_STATUS } from '../../domain/work-order-status.js'
-import { listWorkOrders, listWorkerUsers } from '../../repositories/worker-platform/index.js'
+import { countWorkOrders, listWorkerUsers } from '../../repositories/worker-platform/index.js'
 import { publishWorkerHallRealtimeRefresh } from '../realtime/realtime-event-service.js'
 import { getWorkerHallConfig, saveWorkerHallConfig } from './worker-hall-config-service.js'

 export async function getAdminWorkerPlatformSummary() {
  const hallConfig = getWorkerHallConfig()
-  const [pendingWorkers, pendingMaterial, openOrders, visibleOpenOrders, inProgressOrders] =
-    await Promise.all([
+  const [
+    pendingWorkers,
+    pendingMaterialCount,
+    openOrderCount,
+    visibleOpenOrderCount,
+    inProgressOrderCount,
+  ] = await Promise.all([
    listWorkerUsers({ page: 1, pageSize: 1, status: 'pending_review' }),
-      listWorkOrders({ page: 1, pageSize: 1, statuses: [WORK_ORDER_STATUS.PENDING_MATERIAL] }),
-      listWorkOrders({
-        page: 1,
-        pageSize: 1,
+    countWorkOrders({ statuses: [WORK_ORDER_STATUS.PENDING_MATERIAL] }),
+    countWorkOrders({
      statuses: [WORK_ORDER_STATUS.OPEN],
      excludeFilledSharing: true,
    }),
-      listWorkOrders({
-        page: 1,
-        pageSize: 1,
+    countWorkOrders({
      statuses: [WORK_ORDER_STATUS.OPEN],
      hallCandidateLimit: hallConfig.maxVisibleOrders,
      hallCategoryLimits: hallConfig.categoryMaxVisibleOrders,
      excludeFilledSharing: true,
    }),
-      listWorkOrders({
-        page: 1,
-        pageSize: 1,
+    countWorkOrders({
      statuses: [WORK_ORDER_STATUS.IN_PROGRESS],
      useOperationalStatus: true,
    }),
  ])
  return {
    pendingWorkerCount: pendingWorkers.total,
-    pendingMaterialCount: pendingMaterial.total,
-    openOrderCount: visibleOpenOrders.total,
-    queuedHallOrderCount: Math.max(0, openOrders.total - visibleOpenOrders.total),
+    pendingMaterialCount,
+    openOrderCount: visibleOpenOrderCount,
+    queuedHallOrderCount: Math.max(0, openOrderCount - visibleOpenOrderCount),
    hallMaxVisibleOrders: hallConfig.maxVisibleOrders,
-    inProgressOrderCount: inProgressOrders.total,
+    inProgressOrderCount,
  }
 }

 export async function getAdminWorkerHallConfig() {
  const config = getWorkerHallConfig()
-  const [openOrders, visibleOpenOrders] = await Promise.all([
-    listWorkOrders({
-      page: 1,
-      pageSize: 1,
+  const [openOrderCount, visibleOpenOrderCount] = await Promise.all([
+    countWorkOrders({
      statuses: [WORK_ORDER_STATUS.OPEN],
      excludeFilledSharing: true,
    }),
-    listWorkOrders({
-      page: 1,
-      pageSize: 1,
+    countWorkOrders({
      statuses: [WORK_ORDER_STATUS.OPEN],
      hallCandidateLimit: config.maxVisibleOrders,
      hallCategoryLimits: config.categoryMaxVisibleOrders,
@@ -60,8 +55,8 @@ export async function getAdminWorkerHallConfig() {
  ])
  return {
    ...config,
-    visibleOrderCount: visibleOpenOrders.total,
-    queuedOrderCount: Math.max(0, openOrders.total - visibleOpenOrders.total),
+    visibleOrderCount: visibleOpenOrderCount,
+    queuedOrderCount: Math.max(0, openOrderCount - visibleOpenOrderCount),
  }
 }

@@ -5,11 +5,12 @@
 新增或修改 SQL 必须满足：

 1. 列表查询必须有明确的分页、最大数量或日期范围；禁止无界返回历史数据。
-2. `GROUP BY`、`COUNT(DISTINCT)`、窗口函数和跨表聚合必须附带 `EXPLAIN (ANALYZE, BUFFERS)` 结果。
-3. 高频接口禁止在公共 `SELECT` 中叠加未必要的 LATERAL、相关子查询或全表聚合。
-4. 当前页关联数据必须按当前页 ID 批量查询，不得读取用户全部历史记录再在应用层过滤。
-5. 新增索引必须通过数据库 migration 提交，并说明覆盖的查询条件；大表生产索引应评估锁影响。
-6. SQL 必须使用参数绑定，禁止把请求参数直接拼接到 SQL 值中。
+2. 禁止在包含 `SELECT *`、JSON 宽字段、LATERAL 或详情关联的列表查询上使用 `COUNT(*) OVER()` 后再 `LIMIT/OFFSET`。窗口总数会迫使数据库在分页前处理全部候选宽记录；应使用独立窄 `COUNT`、复用同筛选条件的聚合总数，或先分页 ID 再查当前页详情。
+3. `GROUP BY`、`COUNT(DISTINCT)`、窗口函数和跨表聚合必须附带 `EXPLAIN (ANALYZE, BUFFERS)` 结果。
+4. 高频接口禁止在公共 `SELECT` 中叠加未必要的 LATERAL、相关子查询或全表聚合。
+5. 当前页关联数据必须按当前页 ID 批量查询，不得读取用户全部历史记录再在应用层过滤。
+6. 新增索引必须通过数据库 migration 提交，并说明覆盖的查询条件；大表生产索引应评估锁影响。
+7. SQL 必须使用参数绑定，禁止把请求参数直接拼接到 SQL 值中。

 后端 `npm run check` 会执行 `npm run check:sql`，检查关键性能防线没有被重构移除。该检查不能替代真实数据量下的 `EXPLAIN`。