From 84a8f138d48c2afff23020687d7738ccad90a599 Mon Sep 17 00:00:00 2001 From: yml2213 Date: Fri, 4 Sep 2026 21:22:35 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=8E=A5=E5=8D=95=E5=B9=B3?= =?UTF-8?q?=E5=8F=B0=E5=B7=A5=E5=8D=95=E6=9F=A5=E8=AF=A2=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/scripts/check-sql-guard.ts | 50 ++++++++++++- .../073_rebuild_work_order_search_index.sql | 23 ++++++ .../src/repositories/worker-platform/types.ts | 2 + .../worker-platform/work-order-query-repo.ts | 64 ++++++++-------- .../admin-work-order-query-service.ts | 17 +++-- .../admin-worker-platform-summary-service.ts | 73 +++++++++---------- docs/数据库SQL约束与运维.md | 11 +-- 7 files changed, 158 insertions(+), 82 deletions(-) create mode 100644 apps/backend/src/db/migrations/073_rebuild_work_order_search_index.sql diff --git a/apps/backend/scripts/check-sql-guard.ts b/apps/backend/scripts/check-sql-guard.ts index 5bfbeaaf..d64bad14 100644 --- a/apps/backend/scripts/check-sql-guard.ts +++ b/apps/backend/scripts/check-sql-guard.ts @@ -10,7 +10,12 @@ import { fileURLToPath } from 'node:url' const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)) const BACKEND_ROOT = path.resolve(SCRIPT_DIR, '..') -const checks: Array<{ name: string; file: string; patterns: string[] }> = [ +const checks: Array<{ + name: string + file: string + patterns?: string[] + forbiddenPatterns?: string[] +}> = [ { name: '慢查询阈值配置', file: path.join(BACKEND_ROOT, 'src/config/env-overrides.ts'), @@ -35,6 +40,12 @@ const checks: Array<{ name: string; file: string; patterns: string[] }> = [ file: path.join(BACKEND_ROOT, 'src/repositories/worker-platform/work-order-share-repo.ts'), patterns: ['workOrderIds: number[]', 'wos.work_order_id = ANY($2::bigint[])'], }, + { + name: '接单工单宽详情分页禁止窗口总数', + file: path.join(BACKEND_ROOT, 'src/repositories/worker-platform/work-order-query-repo.ts'), + patterns: ['countWorkOrdersWithWhere', 'includeTotal'], + forbiddenPatterns: ['COUNT(*) OVER' + '()', 'LIST_WORK_ORDER_SELECT'], + }, { name: '数据保留清理扫描索引', file: path.join(BACKEND_ROOT, 'src/db/migrations/067_data_retention_indexes.sql'), @@ -55,10 +66,27 @@ const checks: Array<{ name: string; file: string; patterns: string[] }> = [ "WHERE event_type LIKE 'timeout_%'", ], }, + { + name: '接单工单搜索表达式索引重建', + file: path.join(BACKEND_ROOT, 'src/db/migrations/073_rebuild_work_order_search_index.sql'), + patterns: [ + 'DROP INDEX IF EXISTS idx_work_orders_search_text_trgm', + 'CREATE INDEX idx_work_orders_search_text_trgm', + "material_json->'collect'->'fields'->>'gameId'", + ], + }, ] const failures: string[] = [] +// 窗口总数会在 LIMIT/OFFSET 之前消费全部候选行；全后端 SQL 都禁止引入该写法。 +const countWindowPattern = /COUNT\s*\(\s*\*\s*\)\s*OVER\s*\(/i +for (const file of listSourceFiles(path.join(BACKEND_ROOT, 'src'))) { + if (countWindowPattern.test(fs.readFileSync(file, 'utf8'))) { + failures.push(`禁止 COUNT(*) OVER() 窗口分页: ${file}`) + } +} + // 已应用的历史迁移必须保持字节不变，所有结构或数据修复都必须新增迁移文件。 try { const diffOutputs = [ @@ -89,11 +117,16 @@ for (const check of checks) { continue } const source = fs.readFileSync(check.file, 'utf8') - for (const pattern of check.patterns) { + for (const pattern of check.patterns || []) { if (!source.includes(pattern)) { failures.push(`${check.name}: 缺少约束 ${pattern} (${check.file})`) } } + for (const pattern of check.forbiddenPatterns || []) { + if (source.includes(pattern)) { + failures.push(`${check.name}: 出现禁止 SQL ${pattern} (${check.file})`) + } + } } if (failures.length > 0) { @@ -103,3 +136,16 @@ if (failures.length > 0) { } else { console.info('[sql-guard] SQL 约束检查通过') } + +function listSourceFiles(directory: string): string[] { + const files: string[] = [] + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name) + if (entry.isDirectory()) { + files.push(...listSourceFiles(file)) + } else if (entry.isFile() && /\.(?:ts|sql)$/.test(entry.name)) { + files.push(file) + } + } + return files +} diff --git a/apps/backend/src/db/migrations/073_rebuild_work_order_search_index.sql b/apps/backend/src/db/migrations/073_rebuild_work_order_search_index.sql new file mode 100644 index 00000000..89b9099c --- /dev/null +++ b/apps/backend/src/db/migrations/073_rebuild_work_order_search_index.sql @@ -0,0 +1,23 @@ +-- 修复接单工单搜索表达式索引漂移。 +-- 部署中曾存在索引表达式与当前查询不一致的数据库；新增迁移确保已执行 031 的环境也会重建。 +DROP INDEX IF EXISTS idx_work_orders_search_text_trgm; + +CREATE INDEX idx_work_orders_search_text_trgm + ON work_orders USING gin ( + LOWER( + COALESCE(work_order_no, '') || ' ' || + COALESCE(platform_order_id, '') || ' ' || + COALESCE(product_name, '') || ' ' || + COALESCE(material_json->'collect'->'fields'->>'gameId', '') || ' ' || + COALESCE(material_json->'collect'->'fields'->>'gameNickname', '') || ' ' || + COALESCE(material_json->'collect'->'fields'->>'gameAccount', '') || ' ' || + COALESCE(material_json->'collect'->'fields'->>'roleName', '') || ' ' || + COALESCE(material_json->'fields'->>'gameId', '') || ' ' || + COALESCE(material_json->'fields'->>'gameNickname', '') || ' ' || + COALESCE(material_json->>'gameId', '') || ' ' || + COALESCE(material_json->>'gameNickname', '') + ) gin_trgm_ops + ); + +COMMENT ON INDEX idx_work_orders_search_text_trgm IS + '接单工单模糊搜索：订单号、商品名、游戏 ID/昵称及历史字段'; diff --git a/apps/backend/src/repositories/worker-platform/types.ts b/apps/backend/src/repositories/worker-platform/types.ts index 848086d6..bee23f83 100644 --- a/apps/backend/src/repositories/worker-platform/types.ts +++ b/apps/backend/src/repositories/worker-platform/types.ts @@ -418,6 +418,8 @@ export type GrabWorkOrderResult = { export type ListInput = { page?: number pageSize?: number + /** 列表调用方已有同筛选条件的聚合总数时，可跳过独立 COUNT。 */ + includeTotal?: boolean /** 单状态筛选（兼容 worker 用户等其他列表） */ status?: string /** 支持多个状态同时筛选（空数组表示全部状态） */ diff --git a/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts b/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts index b84151c2..58960a70 100644 --- a/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts +++ b/apps/backend/src/repositories/worker-platform/work-order-query-repo.ts @@ -58,13 +58,6 @@ export const WORK_ORDER_SELECT = ` ) share_summary ON TRUE ` -// Keep the page and total in one filtered query. This avoids running the same -// joins and share aggregation twice for every list request. -const LIST_WORK_ORDER_SELECT = WORK_ORDER_SELECT.replace( - ' SELECT\n', - ' SELECT\n COUNT(*) OVER()::int AS total_count,\n', -) - type WorkerOrderOverviewRow = { status: string order_count: number @@ -234,6 +227,7 @@ export async function listWorkOrdersByPlatformOrderId( export async function listWorkOrders({ page = 1, pageSize = 20, + includeTotal = true, statuses = [], status = '', workOrderId = 0, @@ -271,10 +265,13 @@ export async function listWorkOrders({ vipEvidencePending, acceptanceMode, giftPhase, - useOperationalStatus, + // 无状态筛选时运营状态不影响列表入选，不应为此聚合整张拼单表。 + useOperationalStatus: + useOperationalStatus && (effectiveStatuses.length > 0 || vipEvidencePending), createdFrom, createdTo, }) + const countParams = [...params] const offset = (page - 1) * pageSize const orderBy = sort === 'worker_claimed_at_desc' && workerSharingId @@ -289,33 +286,42 @@ export async function listWorkOrders({ : 'wo.id DESC' const effectiveOrderBy = pinnedFirst ? `wo.pinned_at DESC NULLS LAST, ${orderBy}` : orderBy params.push(pageSize, offset) - const itemsResult = await query( - `${LIST_WORK_ORDER_SELECT} - ${joins} - ${whereClause} - ORDER BY ${effectiveOrderBy} - LIMIT $${params.length - 1} OFFSET $${params.length}`, - params, - ) - // A page beyond the end has no row from which to read the window count. - // Keep that uncommon pagination edge case correct without restoring a - // second query for normal requests. - let total = Number( - (itemsResult.rows[0] as WorkOrderRow & { total_count?: number })?.total_count || 0, - ) - if (itemsResult.rows.length === 0 && page > 1) { - const totalResult = await query<{ total: number }>( - `SELECT COUNT(*)::int AS total FROM work_orders wo ${joins} ${whereClause}`, - params.slice(0, -2), - ) - total = Number(totalResult.rows[0]?.total || 0) - } + // 宽详情只查当前页；不将 COUNT 窗口放进详情查询，避免 LIMIT 前处理全量 JSON。 + const [itemsResult, total] = await Promise.all([ + query( + `${WORK_ORDER_SELECT} + ${joins} + ${whereClause} + ORDER BY ${effectiveOrderBy} + LIMIT $${params.length - 1} OFFSET $${params.length}`, + params, + ), + includeTotal ? countWorkOrdersWithWhere(joins, whereClause, countParams) : Promise.resolve(0), + ]) return { items: itemsResult.rows, total, } } +/** 只返回筛选范围总数，不加载工单 JSON、取消事件和当前页关联字段。 */ +export async function countWorkOrders(input: ListInput = {}): Promise { + const { joins, whereClause, params } = buildWorkOrderWhere(input) + return countWorkOrdersWithWhere(joins, whereClause, params) +} + +async function countWorkOrdersWithWhere( + joins: string, + whereClause: string, + params: unknown[], +): Promise { + const result = await query<{ total: number }>( + `SELECT COUNT(*)::int AS total FROM work_orders wo ${joins} ${whereClause}`, + params, + ) + return Number(result.rows[0]?.total || 0) +} + /** 按筛选条件汇总工单，统计范围不受当前分页影响。 */ export async function getWorkOrderStatistics(input: ListInput = {}): Promise { const { joins, whereClause, params } = buildWorkOrderWhere(input) diff --git a/apps/backend/src/services/worker-platform/admin-work-order-query-service.ts b/apps/backend/src/services/worker-platform/admin-work-order-query-service.ts index fc99dbbb..3a208df1 100644 --- a/apps/backend/src/services/worker-platform/admin-work-order-query-service.ts +++ b/apps/backend/src/services/worker-platform/admin-work-order-query-service.ts @@ -55,12 +55,15 @@ export async function listAdminWorkOrders(query: JsonObject = {}) { ...(createdFrom ? { createdFrom } : {}), ...(createdTo ? { createdTo } : {}), } - const { items, total } = await listWorkOrders({ page, pageSize, ...filters }) - const statistics = await getWorkOrderStatistics(filters) - const pendingUnfreezeByOrderId = await sumPendingUnfreezeByOrderIds( - items.map((item) => Number(item.id)), - ) - const shares = await listWorkOrderSharesByOrderIds(items.map((item) => Number(item.id))) + const [{ items }, statistics] = await Promise.all([ + listWorkOrders({ page, pageSize, includeTotal: false, ...filters }), + getWorkOrderStatistics(filters), + ]) + const itemIds = items.map((item) => Number(item.id)) + const [pendingUnfreezeByOrderId, shares] = await Promise.all([ + sumPendingUnfreezeByOrderIds(itemIds), + listWorkOrderSharesByOrderIds(itemIds), + ]) const sharesByOrderId = new Map() for (const share of shares) { const orderId = Number(share.work_order_id) @@ -73,7 +76,7 @@ export async function listAdminWorkOrders(query: JsonObject = {}) { ...mapWorkOrderAdmin(item, sharesByOrderId.get(Number(item.id)) || []), pendingUnfreezeAmount: pendingUnfreezeByOrderId.get(Number(item.id)) || 0, })), - pagination: { page, pageSize, total }, + pagination: { page, pageSize, total: statistics.totalCount }, statistics, } } diff --git a/apps/backend/src/services/worker-platform/admin-worker-platform-summary-service.ts b/apps/backend/src/services/worker-platform/admin-worker-platform-summary-service.ts index 11787f54..cb35f61a 100644 --- a/apps/backend/src/services/worker-platform/admin-worker-platform-summary-service.ts +++ b/apps/backend/src/services/worker-platform/admin-worker-platform-summary-service.ts @@ -1,57 +1,52 @@ import { WORK_ORDER_STATUS } from '../../domain/work-order-status.js' -import { listWorkOrders, listWorkerUsers } from '../../repositories/worker-platform/index.js' +import { countWorkOrders, listWorkerUsers } from '../../repositories/worker-platform/index.js' import { publishWorkerHallRealtimeRefresh } from '../realtime/realtime-event-service.js' import { getWorkerHallConfig, saveWorkerHallConfig } from './worker-hall-config-service.js' export async function getAdminWorkerPlatformSummary() { const hallConfig = getWorkerHallConfig() - const [pendingWorkers, pendingMaterial, openOrders, visibleOpenOrders, inProgressOrders] = - await Promise.all([ - listWorkerUsers({ page: 1, pageSize: 1, status: 'pending_review' }), - listWorkOrders({ page: 1, pageSize: 1, statuses: [WORK_ORDER_STATUS.PENDING_MATERIAL] }), - listWorkOrders({ - page: 1, - pageSize: 1, - statuses: [WORK_ORDER_STATUS.OPEN], - excludeFilledSharing: true, - }), - listWorkOrders({ - page: 1, - pageSize: 1, - statuses: [WORK_ORDER_STATUS.OPEN], - hallCandidateLimit: hallConfig.maxVisibleOrders, - hallCategoryLimits: hallConfig.categoryMaxVisibleOrders, - excludeFilledSharing: true, - }), - listWorkOrders({ - page: 1, - pageSize: 1, - statuses: [WORK_ORDER_STATUS.IN_PROGRESS], - useOperationalStatus: true, - }), - ]) + const [ + pendingWorkers, + pendingMaterialCount, + openOrderCount, + visibleOpenOrderCount, + inProgressOrderCount, + ] = await Promise.all([ + listWorkerUsers({ page: 1, pageSize: 1, status: 'pending_review' }), + countWorkOrders({ statuses: [WORK_ORDER_STATUS.PENDING_MATERIAL] }), + countWorkOrders({ + statuses: [WORK_ORDER_STATUS.OPEN], + excludeFilledSharing: true, + }), + countWorkOrders({ + statuses: [WORK_ORDER_STATUS.OPEN], + hallCandidateLimit: hallConfig.maxVisibleOrders, + hallCategoryLimits: hallConfig.categoryMaxVisibleOrders, + excludeFilledSharing: true, + }), + countWorkOrders({ + statuses: [WORK_ORDER_STATUS.IN_PROGRESS], + useOperationalStatus: true, + }), + ]) return { pendingWorkerCount: pendingWorkers.total, - pendingMaterialCount: pendingMaterial.total, - openOrderCount: visibleOpenOrders.total, - queuedHallOrderCount: Math.max(0, openOrders.total - visibleOpenOrders.total), + pendingMaterialCount, + openOrderCount: visibleOpenOrderCount, + queuedHallOrderCount: Math.max(0, openOrderCount - visibleOpenOrderCount), hallMaxVisibleOrders: hallConfig.maxVisibleOrders, - inProgressOrderCount: inProgressOrders.total, + inProgressOrderCount, } } export async function getAdminWorkerHallConfig() { const config = getWorkerHallConfig() - const [openOrders, visibleOpenOrders] = await Promise.all([ - listWorkOrders({ - page: 1, - pageSize: 1, + const [openOrderCount, visibleOpenOrderCount] = await Promise.all([ + countWorkOrders({ statuses: [WORK_ORDER_STATUS.OPEN], excludeFilledSharing: true, }), - listWorkOrders({ - page: 1, - pageSize: 1, + countWorkOrders({ statuses: [WORK_ORDER_STATUS.OPEN], hallCandidateLimit: config.maxVisibleOrders, hallCategoryLimits: config.categoryMaxVisibleOrders, @@ -60,8 +55,8 @@ export async function getAdminWorkerHallConfig() { ]) return { ...config, - visibleOrderCount: visibleOpenOrders.total, - queuedOrderCount: Math.max(0, openOrders.total - visibleOpenOrders.total), + visibleOrderCount: visibleOpenOrderCount, + queuedOrderCount: Math.max(0, openOrderCount - visibleOpenOrderCount), } } diff --git a/docs/数据库SQL约束与运维.md b/docs/数据库SQL约束与运维.md index 78f3a4c0..1c53a193 100644 --- a/docs/数据库SQL约束与运维.md +++ b/docs/数据库SQL约束与运维.md @@ -5,11 +5,12 @@ 新增或修改 SQL 必须满足： 1. 列表查询必须有明确的分页、最大数量或日期范围；禁止无界返回历史数据。 -2. `GROUP BY`、`COUNT(DISTINCT)`、窗口函数和跨表聚合必须附带 `EXPLAIN (ANALYZE, BUFFERS)` 结果。 -3. 高频接口禁止在公共 `SELECT` 中叠加未必要的 LATERAL、相关子查询或全表聚合。 -4. 当前页关联数据必须按当前页 ID 批量查询，不得读取用户全部历史记录再在应用层过滤。 -5. 新增索引必须通过数据库 migration 提交，并说明覆盖的查询条件；大表生产索引应评估锁影响。 -6. SQL 必须使用参数绑定，禁止把请求参数直接拼接到 SQL 值中。 +2. 禁止在包含 `SELECT *`、JSON 宽字段、LATERAL 或详情关联的列表查询上使用 `COUNT(*) OVER()` 后再 `LIMIT/OFFSET`。窗口总数会迫使数据库在分页前处理全部候选宽记录；应使用独立窄 `COUNT`、复用同筛选条件的聚合总数，或先分页 ID 再查当前页详情。 +3. `GROUP BY`、`COUNT(DISTINCT)`、窗口函数和跨表聚合必须附带 `EXPLAIN (ANALYZE, BUFFERS)` 结果。 +4. 高频接口禁止在公共 `SELECT` 中叠加未必要的 LATERAL、相关子查询或全表聚合。 +5. 当前页关联数据必须按当前页 ID 批量查询，不得读取用户全部历史记录再在应用层过滤。 +6. 新增索引必须通过数据库 migration 提交，并说明覆盖的查询条件；大表生产索引应评估锁影响。 +7. SQL 必须使用参数绑定，禁止把请求参数直接拼接到 SQL 值中。 后端 `npm run check` 会执行 `npm run check:sql`，检查关键性能防线没有被重构移除。该检查不能替代真实数据量下的 `EXPLAIN`。