diff --git a/.env.mac-docker.example b/.env.mac-docker.example index 5f0022fd..4ef44611 100644 --- a/.env.mac-docker.example +++ b/.env.mac-docker.example @@ -20,6 +20,8 @@ DATABASE_MAX_CONNECTIONS=8 DATABASE_IDLE_TIMEOUT_MS=30000 DATABASE_CONNECTION_TIMEOUT_MS=5000 DATABASE_STATEMENT_TIMEOUT_MS=15000 +DATABASE_SLOW_QUERY_THRESHOLD_MS=500 +POSTGRES_LOG_MIN_DURATION_STATEMENT_MS=500 POSTGRES_PORT=5432 # Logging diff --git a/.env.server.example b/.env.server.example index 14039e98..be140725 100644 --- a/.env.server.example +++ b/.env.server.example @@ -27,6 +27,10 @@ DATABASE_MAX_CONNECTIONS=8 DATABASE_IDLE_TIMEOUT_MS=30000 DATABASE_CONNECTION_TIMEOUT_MS=5000 DATABASE_STATEMENT_TIMEOUT_MS=15000 +# 慢查询告警阈值；超过该时长的 SQL 只记录指纹和耗时，不记录参数值 +DATABASE_SLOW_QUERY_THRESHOLD_MS=500 +# PostgreSQL 原生慢查询日志阈值（毫秒）；-1 关闭 +POSTGRES_LOG_MIN_DURATION_STATEMENT_MS=500 # Logging LOG_LEVEL=info diff --git a/apps/backend/package.json b/apps/backend/package.json index 44ddc55f..313a135f 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -9,6 +9,7 @@ "db:migrate": "tsx src/db/migrate.ts", "db:migrate:create": "tsx scripts/create-migration.ts", "db:migrate:status": "tsx scripts/migration-status.ts", + "check:sql": "tsx scripts/check-sql-guard.ts", "dev": "tsx watch --clear-screen=false src/index.ts", "format": "prettier --config ../../.prettierrc.json --ignore-path ../../.prettierignore --write \"src/**/*.{ts,js,json}\" \"scripts/**/*.{ts,js}\" eslint.config.js package.json tsconfig.json tsconfig.build.json tsconfig.eslint.json", "format:check": "prettier --config ../../.prettierrc.json --ignore-path ../../.prettierignore --check \"src/**/*.{ts,js,json}\" \"scripts/**/*.{ts,js}\" eslint.config.js package.json tsconfig.json tsconfig.build.json tsconfig.eslint.json", @@ -23,7 +24,7 @@ "test:industry:gen": "tsx scripts/gen-test-cases.ts", "test:industry:curl": "tsx scripts/curl-send-callback.ts", "typecheck": "tsc -p tsconfig.json --noEmit", - "check": "npm run format:check && npm run lint:check && npm run typecheck && npm test && npm run build", + "check": "npm run format:check && npm run check:sql && npm run lint:check && npm run typecheck && npm test && npm run build", "start": "node dist/index.js", "start:src": "tsx src/index.ts", "mock:feifei": "tsx scripts/mock-feifei-claim.ts" diff --git a/apps/backend/scripts/check-sql-guard.ts b/apps/backend/scripts/check-sql-guard.ts new file mode 100644 index 00000000..0488d6ff --- /dev/null +++ b/apps/backend/scripts/check-sql-guard.ts @@ -0,0 +1,59 @@ +/** + * 数据库 SQL 约束检查。 + * 该脚本检查关键防线是否被后续重构移除，不尝试替代 EXPLAIN 和生产监控。 + */ +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)) +const BACKEND_ROOT = path.resolve(SCRIPT_DIR, '..') + +const checks: Array<{ name: string; file: string; patterns: string[] }> = [ + { + name: '慢查询阈值配置', + file: path.join(BACKEND_ROOT, 'src/config/env-overrides.ts'), + patterns: ['DATABASE_SLOW_QUERY_THRESHOLD_MS', 'slowQueryThresholdMs'], + }, + { + name: '慢查询日志与参数脱敏', + file: path.join(BACKEND_ROOT, 'src/db/client.ts'), + patterns: ['logSlowQuery', 'fingerprintSql', '参数值始终不写入日志'], + }, + { + name: '高频 pending 查询索引', + file: path.join(BACKEND_ROOT, 'src/db/migrations/058_database_query_guardrails.sql'), + patterns: [ + 'idx_worker_cancel_requests_pending_worker_order', + 'idx_worker_feedbacks_pending_worker_order', + 'idx_work_order_events_order_type_id', + ], + }, + { + name: '打手订单拼单查询限定当前页', + file: path.join(BACKEND_ROOT, 'src/repositories/worker-platform/work-order-share-repo.ts'), + patterns: ['workOrderIds: number[]', 'wos.work_order_id = ANY($2::bigint[])'], + }, +] + +const failures: string[] = [] +for (const check of checks) { + if (!fs.existsSync(check.file)) { + failures.push(`${check.name}: 文件不存在 ${check.file}`) + continue + } + const source = fs.readFileSync(check.file, 'utf8') + for (const pattern of check.patterns) { + if (!source.includes(pattern)) { + failures.push(`${check.name}: 缺少约束 ${pattern} (${check.file})`) + } + } +} + +if (failures.length > 0) { + console.error('[sql-guard] SQL 约束检查失败') + for (const failure of failures) console.error(`[sql-guard] ${failure}`) + process.exitCode = 1 +} else { + console.info('[sql-guard] SQL 约束检查通过') +} diff --git a/apps/backend/src/config/defaults.ts b/apps/backend/src/config/defaults.ts index 0d7d9724..5b380b1e 100644 --- a/apps/backend/src/config/defaults.ts +++ b/apps/backend/src/config/defaults.ts @@ -26,6 +26,7 @@ export function createDefaultRuntimeConfig(projectRoot: string): RuntimeConfig { idleTimeoutMs: 30_000, connectionTimeoutMs: 5_000, statementTimeoutMs: 15_000, + slowQueryThresholdMs: 500, }, storage: { diff --git a/apps/backend/src/config/env-documentation.test.ts b/apps/backend/src/config/env-documentation.test.ts index f79a4e8a..0faea8bf 100644 --- a/apps/backend/src/config/env-documentation.test.ts +++ b/apps/backend/src/config/env-documentation.test.ts @@ -24,6 +24,7 @@ const DEPLOYMENT_ONLY_ENV_NAMES = [ 'MINIO_ROOT_USER', 'NPM_CONFIG_REGISTRY', 'POSTGRES_DB', + 'POSTGRES_LOG_MIN_DURATION_STATEMENT_MS', 'POSTGRES_PASSWORD', 'POSTGRES_PORT', 'POSTGRES_USER', diff --git a/apps/backend/src/config/env-overrides.ts b/apps/backend/src/config/env-overrides.ts index dc61532c..6bd94d32 100644 --- a/apps/backend/src/config/env-overrides.ts +++ b/apps/backend/src/config/env-overrides.ts @@ -38,6 +38,7 @@ export const ENV_OVERRIDES: readonly EnvOverride[] = [ integerEnv('DATABASE_IDLE_TIMEOUT_MS', ['database', 'idleTimeoutMs']), integerEnv('DATABASE_CONNECTION_TIMEOUT_MS', ['database', 'connectionTimeoutMs']), integerEnv('DATABASE_STATEMENT_TIMEOUT_MS', ['database', 'statementTimeoutMs']), + integerEnv('DATABASE_SLOW_QUERY_THRESHOLD_MS', ['database', 'slowQueryThresholdMs']), stringEnv('STORAGE_ENDPOINT', ['storage', 'endpoint']), stringEnv('STORAGE_BUCKET', ['storage', 'bucket']), stringEnv('STORAGE_ACCESS_KEY_ID', ['storage', 'accessKeyId']), diff --git a/apps/backend/src/config/runtime-validation.ts b/apps/backend/src/config/runtime-validation.ts index 43bdc468..c1fa79bf 100644 --- a/apps/backend/src/config/runtime-validation.ts +++ b/apps/backend/src/config/runtime-validation.ts @@ -62,6 +62,12 @@ export function validateRuntimeConfig( min: 1, }, ) + requireOptionalInteger( + issues, + 'database.slowQueryThresholdMs', + config.database?.slowQueryThresholdMs, + { min: 0 }, + ) requireInteger(issues, 'storage.maxUploadSizeMb', config.storage?.maxUploadSizeMb, { min: 1, max: 100, diff --git a/apps/backend/src/db/client.ts b/apps/backend/src/db/client.ts index 89dd34ff..6e90c564 100644 --- a/apps/backend/src/db/client.ts +++ b/apps/backend/src/db/client.ts @@ -2,12 +2,13 @@ import { Pool } from 'pg' import type { PoolClient, QueryResult, QueryResultRow } from 'pg' import { runtimeConfig } from '../config/runtime.js' +import { logWarn } from '../utils/logger.js' let poolInstance: Pool | null = null export function getDb(): Pool { if (!poolInstance) { - poolInstance = new Pool({ + const pool = new Pool({ connectionString: String(runtimeConfig.database?.url || '').trim(), ssl: runtimeConfig.database?.ssl ? { rejectUnauthorized: false } : false, max: Number(runtimeConfig.database?.maxConnections || 10), @@ -15,6 +16,12 @@ export function getDb(): Pool { connectionTimeoutMillis: Number(runtimeConfig.database?.connectionTimeoutMs || 5_000), statement_timeout: Number(runtimeConfig.database?.statementTimeoutMs || 15_000), }) + pool.on('error', (error) => { + void logWarn('[db/pool]', '数据库连接池出现空闲连接错误', { + error: error instanceof Error ? error.message : String(error), + }) + }) + poolInstance = pool } return poolInstance @@ -30,7 +37,40 @@ export async function query( params: unknown[] = [], ): Promise> { const pool = getDb() - return pool.query(text, params) + const startedAt = process.hrtime.bigint() + try { + const result = await pool.query(text, params) + logSlowQuery(text, startedAt, result.rowCount ?? 0) + return result + } catch (error) { + logSlowQuery(text, startedAt, 0, error) + throw error + } +} + +function logSlowQuery(text: string, startedAt: bigint, rowCount: number, error?: unknown) { + const thresholdMs = Number(runtimeConfig.database?.slowQueryThresholdMs ?? 500) + if (thresholdMs <= 0) return + const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000 + if (durationMs < thresholdMs && !error) return + void logWarn('[db/query]', error ? '数据库 SQL 执行失败' : '数据库慢查询', { + durationMs: Math.round(durationMs * 100) / 100, + thresholdMs, + rowCount, + sql: fingerprintSql(text), + error: error instanceof Error ? error.message : undefined, + }) +} + +/** 只保留可聚合的 SQL 形状，参数值始终不写入日志。 */ +function fingerprintSql(text: string) { + return String(text || '') + .replace(/--[^\r\n]*/g, ' ') + .replace(/\/\*[\s\S]*?\*\//g, ' ') + .replace(/\$\d+/g, '?') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 500) } export async function withTransaction(fn: (client: PoolClient) => Promise): Promise { diff --git a/apps/backend/src/db/migrations/058_database_query_guardrails.sql b/apps/backend/src/db/migrations/058_database_query_guardrails.sql new file mode 100644 index 00000000..60bdc23b --- /dev/null +++ b/apps/backend/src/db/migrations/058_database_query_guardrails.sql @@ -0,0 +1,32 @@ +-- 数据库可观测性与高频查询索引保护。 +-- PostgreSQL 容器通过 shared_preload_libraries 预加载；托管数据库若无权限，跳过扩展创建并由平台侧开启。 +DO $$ +BEGIN + CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +EXCEPTION + WHEN insufficient_privilege OR undefined_file THEN + RAISE NOTICE '无法创建 pg_stat_statements，请在数据库平台侧启用该扩展'; +END +$$; + +-- 打手订单状态计算中的 pending 撤单/反馈 EXISTS。 +CREATE INDEX IF NOT EXISTS idx_worker_cancel_requests_pending_worker_order + ON worker_order_cancel_requests(worker_id, work_order_id) + WHERE status = 'pending'; + +CREATE INDEX IF NOT EXISTS idx_worker_feedbacks_pending_worker_order + ON worker_order_feedbacks(worker_id, work_order_id) + WHERE status = 'pending'; + +-- 工单公共详情和资料更新角标按工单、事件类型取最新事件。 +CREATE INDEX IF NOT EXISTS idx_work_order_events_order_type_id + ON work_order_events(work_order_id, event_type, id DESC); + +-- 排行榜按打手和验收时间筛选；全部周期仍需汇总表或缓存，索引不能替代聚合。 +CREATE INDEX IF NOT EXISTS idx_work_orders_accepted_worker_at + ON work_orders(assigned_worker_id, accepted_at DESC) + WHERE status = 'accepted' AND assigned_worker_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_work_order_shares_accepted_worker_at + ON work_order_shares(worker_id, accepted_at DESC) + WHERE status = 'accepted'; diff --git a/apps/backend/src/repositories/worker-platform/work-order-share-repo.ts b/apps/backend/src/repositories/worker-platform/work-order-share-repo.ts index 2959d675..752b5e43 100644 --- a/apps/backend/src/repositories/worker-platform/work-order-share-repo.ts +++ b/apps/backend/src/repositories/worker-platform/work-order-share-repo.ts @@ -46,12 +46,16 @@ export async function listWorkOrderSharesByOrderIds( export async function listWorkerSharesByWorker( workerId: number | string, + workOrderIds: number[], ): Promise { + if (workOrderIds.length === 0) return [] const result = await query( `${WORK_ORDER_SHARE_SELECT} - WHERE wos.worker_id = $1 AND wos.status != 'cancelled' + WHERE wos.worker_id = $1 + AND wos.work_order_id = ANY($2::bigint[]) + AND wos.status != 'cancelled' ORDER BY wos.id DESC`, - [Number(workerId)], + [Number(workerId), workOrderIds], ) return result.rows } diff --git a/apps/backend/src/services/worker-platform/worker-my-order-service.ts b/apps/backend/src/services/worker-platform/worker-my-order-service.ts index 906cb5bf..18de7f7f 100644 --- a/apps/backend/src/services/worker-platform/worker-my-order-service.ts +++ b/apps/backend/src/services/worker-platform/worker-my-order-service.ts @@ -61,9 +61,9 @@ export async function listWorkerMyOrders(query: JsonObject = {}, session: Worker sort: 'worker_claimed_at_desc', }) const permissions = resolveWorkerPermissions(worker) - const myShares = await listWorkerSharesByWorker(worker.id) - const myShareByOrderId = new Map(myShares.map((share) => [Number(share.work_order_id), share])) const workOrderIds = items.map((item) => Number(item.id)) + const myShares = await listWorkerSharesByWorker(worker.id, workOrderIds) + const myShareByOrderId = new Map(myShares.map((share) => [Number(share.work_order_id), share])) const [notes, views, pendingCancelOrderIds, pendingFeedbackOrderIds, materialEventAtByOrderId] = await Promise.all([ listWorkerWorkOrderNotes(worker.id, workOrderIds), diff --git a/apps/backend/src/types/runtime-config.ts b/apps/backend/src/types/runtime-config.ts index 8cebe280..d3f5bfb1 100644 --- a/apps/backend/src/types/runtime-config.ts +++ b/apps/backend/src/types/runtime-config.ts @@ -37,6 +37,8 @@ export type RuntimeConfig = { idleTimeoutMs?: number connectionTimeoutMs?: number statementTimeoutMs?: number + /** 超过该时长的 SQL 记录慢查询告警；0 表示关闭。 */ + slowQueryThresholdMs?: number } storage: { endpoint: string diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 14f6c1cb..1d66fa29 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -7,6 +7,16 @@ x-json-log-rotation: &json-log-rotation services: postgres: image: docker.m.daocloud.io/library/postgres:16-bookworm + command: + [ + "postgres", + "-c", + "shared_preload_libraries=pg_stat_statements", + "-c", + "pg_stat_statements.track=all", + "-c", + "log_min_duration_statement=${POSTGRES_LOG_MIN_DURATION_STATEMENT_MS:-500}", + ] restart: unless-stopped logging: *json-log-rotation environment: diff --git a/docker-compose.yml b/docker-compose.yml index 7d497fd1..124be1fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,17 @@ x-json-log-rotation: &json-log-rotation services: postgres: image: docker.m.daocloud.io/library/postgres:16-bookworm + # 开启 pg_stat_statements，便于定位生产环境累计最耗时的 SQL。 + command: + [ + "postgres", + "-c", + "shared_preload_libraries=pg_stat_statements", + "-c", + "pg_stat_statements.track=all", + "-c", + "log_min_duration_statement=${POSTGRES_LOG_MIN_DURATION_STATEMENT_MS:-500}", + ] restart: unless-stopped logging: *json-log-rotation environment: diff --git a/docs/数据库SQL约束与运维.md b/docs/数据库SQL约束与运维.md new file mode 100644 index 00000000..eef94d9f --- /dev/null +++ b/docs/数据库SQL约束与运维.md @@ -0,0 +1,40 @@ +# 数据库 SQL 约束与运维 + +## 提交前约束 + +新增或修改 SQL 必须满足： + +1. 列表查询必须有明确的分页、最大数量或日期范围；禁止无界返回历史数据。 +2. `GROUP BY`、`COUNT(DISTINCT)`、窗口函数和跨表聚合必须附带 `EXPLAIN (ANALYZE, BUFFERS)` 结果。 +3. 高频接口禁止在公共 `SELECT` 中叠加未必要的 LATERAL、相关子查询或全表聚合。 +4. 当前页关联数据必须按当前页 ID 批量查询，不得读取用户全部历史记录再在应用层过滤。 +5. 新增索引必须通过数据库 migration 提交，并说明覆盖的查询条件；大表生产索引应评估锁影响。 +6. SQL 必须使用参数绑定，禁止把请求参数直接拼接到 SQL 值中。 + +后端 `npm run check` 会执行 `npm run check:sql`，检查关键性能防线没有被重构移除。该检查不能替代真实数据量下的 `EXPLAIN`。 + +## 生产观测 + +生产 PostgreSQL 应启用 `pg_stat_statements` 和慢查询日志。Compose 默认通过 `POSTGRES_LOG_MIN_DURATION_STATEMENT_MS`（默认 500ms）配置 PostgreSQL 原生慢查询日志；应用默认将执行超过 `DATABASE_SLOW_QUERY_THRESHOLD_MS`（默认 500ms）的 SQL 记录为慢查询，只记录 SQL 形状、耗时和返回行数，不记录参数值。 + +常用排查 SQL： + +```sql +SELECT calls, total_exec_time, mean_exec_time, rows, query +FROM pg_stat_statements +ORDER BY total_exec_time DESC +LIMIT 30; + +SELECT pid, state, wait_event_type, wait_event, + now() - query_start AS duration, query +FROM pg_stat_activity +WHERE datname = current_database() AND state <> 'idle' +ORDER BY query_start; +``` + +## 容量边界 + +- `DATABASE_STATEMENT_TIMEOUT_MS` 防止单条 SQL 长时间占用连接。 +- `DATABASE_MAX_CONNECTIONS` 按后端实例数总量评估；扩容后端实例时不能简单地每实例增加连接数。 +- CPU 限额只能防止数据库拖垮整机，不能替代 SQL 优化。 +- 数据量持续增长后，排行榜和历史统计应迁移到汇总表、缓存或只读副本，不应继续依赖在线全量聚合。