生产实测(2026-10-03,order_site):postgres 在 22.6 小时里烧了 12.2 CPU·小时 (均 0.54 核),却因 2 核配额累计被 cgroup 节流 9,337 秒;top 12 条语句占全部 DB CPU 的 69.4%,其中三条最热的就是「工单列表 / 计数 / 大厅候选集」。 一、查询形状(根因,不是缺索引) 接单大厅那条 COUNT 被调 71,539 次、均 103ms,EXPLAIN(ANALYZE,BUFFERS) 显示 82.8ms 里 35ms 花在「把整张 work_order_shares 聚合一遍再 join」: Index Only Scan rows=22,987 → GroupAggregate → 最终只用到 11 行;同一条 SQL 里 还有第二遍同样的全局聚合(filter_share_summary)。已改成按工单限定的关联子查询 (走 idx_work_order_shares_work_order): - buildWorkOrderWhere 的 excludeFilledSharing 过滤条件; - buildHallCandidateIdsSql 的 candidate_shares。 同参数实测:50.3ms → 2.1ms、35.9ms → 3.8ms、28.4ms → 1.8ms,buffer 10,145 → 187, 三种参数下结果逐行一致(11/11、5/5、11/11)。 有意保留 operational_share_summary 那一处全局聚合:全表统计(103,073 行)时 关联子查询要 103,073 次探测(330ms),一次聚合只要 55ms;已有用例钉住「只允许剩一处」。 二、容器与数据库参数 - POSTGRES_CPUS 2 → 3(4 核机器,同机其它容器实测只用 0.2-0.4 核且几乎不节流); - max_parallel_workers_per_gather 2 → 1(>1 时一条查询要 leader+2 worker=3 核,必撞配额); - checkpoint_timeout 300 → 900s、max_wal_size 1GB → 4GB(43 天累计 21,778 次检查点, sar 里就是每 20-40 分钟一轮写突发)。 三、迁移 092:热表 autovacuum 调参 可见性图陈旧导致 index-only scan 回表 5,580 次;work_orders/order_items/orders/ kuaishou_industry_vouchers/fulfillment_tasks 死元组 1.9 万-5.3 万、上次清理在前一天 甚至 9/29。把这些表的 scale_factor 0.2→0.05、analyze 0.1→0.02、cost_limit 200→800, 把「攒批猛扫」拆成「小步勤清」,并打开 5s 以上的 autovacuum 日志便于观测。 用例:新增 share-summary-scope.test.ts(钉住热路径不再出现整表聚合、只保留一处且有注释说明), env-documentation 白名单补 3 个新增部署变量。全量 599 通过,typecheck/lint/check:sql 通过。
168 lines
7.1 KiB
YAML
168 lines
7.1 KiB
YAML
x-json-log-rotation: &json-log-rotation
|
||
driver: json-file
|
||
options:
|
||
max-size: "20m"
|
||
max-file: "5"
|
||
|
||
services:
|
||
postgres:
|
||
image: docker.m.daocloud.io/library/postgres:16-bookworm
|
||
# 开启 pg_stat_statements,便于定位生产环境累计最耗时的 SQL。
|
||
# 内存类参数默认按 4 核 8G 且与其他服务共用的服务器设置(见 .env.server.example)。
|
||
command:
|
||
[
|
||
"postgres",
|
||
"-c",
|
||
"shared_preload_libraries=pg_stat_statements",
|
||
"-c",
|
||
"pg_stat_statements.track=all",
|
||
"-c",
|
||
"log_min_duration_statement=${POSTGRES_LOG_MIN_DURATION_STATEMENT_MS:-500}",
|
||
"-c",
|
||
"shared_buffers=${POSTGRES_SHARED_BUFFERS:-384MB}",
|
||
"-c",
|
||
"effective_cache_size=${POSTGRES_EFFECTIVE_CACHE_SIZE:-4GB}",
|
||
"-c",
|
||
"work_mem=${POSTGRES_WORK_MEM:-16MB}",
|
||
"-c",
|
||
"maintenance_work_mem=${POSTGRES_MAINTENANCE_WORK_MEM:-256MB}",
|
||
# 并行度 1:生产实测单条大查询一旦开 2 个 parallel worker,就需要
|
||
# 「leader + 2 worker = 3 核」,而容器配额只有 2 核 → 必然撞配额被节流
|
||
# (累计 nr_throttled 94,197 次 / throttled_usec 9,337s)。重查询已优化到
|
||
# 10ms 量级,用不到并行。
|
||
"-c",
|
||
"max_parallel_workers_per_gather=${POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER:-1}",
|
||
# 检查点:默认 5 分钟一次 + max_wal_size 1GB,生产 43 天累计 21,778 次检查点,
|
||
# sar 里就是每 20–40 分钟一轮写突发(wtps 130、8MB/s)。放宽到 15 分钟 / 4GB。
|
||
"-c",
|
||
"checkpoint_timeout=${POSTGRES_CHECKPOINT_TIMEOUT:-900}",
|
||
"-c",
|
||
"max_wal_size=${POSTGRES_MAX_WAL_SIZE:-4GB}",
|
||
]
|
||
restart: unless-stopped
|
||
# 让 PostgreSQL 优雅关闭有足够时间完成检查点;无限制时 OOM 可能拖垮整机。
|
||
mem_limit: ${POSTGRES_MEM_LIMIT:-1.5g}
|
||
# 3 核:这台机器 4 核,postgres 是最大单一消耗者(22.6h 烧 12.2 CPU·h、均 0.54 核),
|
||
# 却被 2 核配额卡住,累计 2.6 小时处于被节流状态。同机其它容器实测只用 0.2–0.4 核,
|
||
# 且几乎不节流,所以这里让出 1 核给数据库的突发。
|
||
cpus: ${POSTGRES_CPUS:-3}
|
||
shm_size: 512mb
|
||
stop_grace_period: 60s
|
||
logging: *json-log-rotation
|
||
environment:
|
||
TZ: ${TZ:-Asia/Shanghai}
|
||
POSTGRES_DB: ${POSTGRES_DB:-order_site}
|
||
POSTGRES_USER: ${POSTGRES_USER:-postgres}
|
||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
|
||
healthcheck:
|
||
test:
|
||
[
|
||
"CMD-SHELL",
|
||
"pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-order_site}",
|
||
]
|
||
interval: 5s
|
||
timeout: 5s
|
||
retries: 20
|
||
volumes:
|
||
- postgres_data:/var/lib/postgresql/data
|
||
|
||
backend:
|
||
build:
|
||
context: .
|
||
dockerfile: deploy/docker/backend.Dockerfile
|
||
args:
|
||
# 默认阿里云公共源;阿里云 ECS 内网可设 DEBIAN_MIRROR=mirrors.cloud.aliyuncs.com
|
||
DEBIAN_MIRROR: ${DEBIAN_MIRROR:-mirrors.aliyun.com}
|
||
NPM_REGISTRY: ${NPM_REGISTRY:-https://registry.npmmirror.com}
|
||
restart: unless-stopped
|
||
# 30s 让后端在 SIGTERM 后完成优雅关闭(HTTP 连接收尾 + 数据库连接池释放)。
|
||
# 内存限额 768m → 1g:生产实测十天从重启后的 ~194MB 涨到 754MB(98%),
|
||
# 贴着 cgroup 上限时 V8 会持续 GC 抖动、接口整体变慢(且未必 OOM,所以日志里看不到崩溃)。
|
||
# 服务器 .env 里若仍写着 BACKEND_MEM_LIMIT=768m,会覆盖这里的默认值,需一并调整。
|
||
mem_limit: ${BACKEND_MEM_LIMIT:-1g}
|
||
cpus: ${BACKEND_CPUS:-1.5}
|
||
stop_grace_period: 30s
|
||
env_file:
|
||
- .env
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
healthcheck:
|
||
test:
|
||
["CMD-SHELL", "curl -fsS http://127.0.0.1:3000/health/ready || exit 1"]
|
||
interval: 5s
|
||
timeout: 5s
|
||
retries: 20
|
||
start_period: 20s
|
||
environment:
|
||
TZ: ${TZ:-Asia/Shanghai}
|
||
PORT: ${BACKEND_PORT:-3000}
|
||
# V8 老生代上限留出余量(限额 1g → 堆 768m):让 GC 在撞上 cgroup 墙之前主动回收,
|
||
# 避免"贴顶后才疯狂 GC"的抖动。native/external 内存与堆外缓冲还需要剩余那部分。
|
||
NODE_OPTIONS: ${BACKEND_NODE_OPTIONS:---max-old-space-size=768}
|
||
HTTP_TRUST_PROXY: ${HTTP_TRUST_PROXY:-1}
|
||
DATABASE_URL: ${DATABASE_URL:-postgres://postgres:postgres@postgres:5432/order_site}
|
||
DATABASE_SSL: ${DATABASE_SSL:-false}
|
||
DATABASE_MAX_CONNECTIONS: ${DATABASE_MAX_CONNECTIONS:-8}
|
||
DATABASE_IDLE_TIMEOUT_MS: ${DATABASE_IDLE_TIMEOUT_MS:-30000}
|
||
DATABASE_CONNECTION_TIMEOUT_MS: ${DATABASE_CONNECTION_TIMEOUT_MS:-5000}
|
||
DATABASE_STATEMENT_TIMEOUT_MS: ${DATABASE_STATEMENT_TIMEOUT_MS:-15000}
|
||
DATA_ROOT: /app/data
|
||
LOG_RETENTION_DAYS: ${LOG_RETENTION_DAYS:-30}
|
||
LOG_MAX_FILE_SIZE_MB: ${LOG_MAX_FILE_SIZE_MB:-20}
|
||
LOG_MAX_FILES: ${LOG_MAX_FILES:-5}
|
||
CLAIM_BASE_URL: ${CLAIM_BASE_URL:-http://localhost/#/claim}
|
||
# 生产图片存储只走阿里云 OSS;MinIO 仅保留在本地开发编排(docker-compose.dev.yml)。
|
||
STORAGE_MODE: ${STORAGE_MODE:-oss}
|
||
STORAGE_ENDPOINT: ${STORAGE_ENDPOINT:-http://minio:9000}
|
||
STORAGE_BUCKET: ${STORAGE_BUCKET:-order-site}
|
||
STORAGE_ACCESS_KEY_ID: ${STORAGE_ACCESS_KEY_ID:-minioadmin}
|
||
STORAGE_SECRET_ACCESS_KEY: ${STORAGE_SECRET_ACCESS_KEY:-minioadmin}
|
||
STORAGE_REGION: ${STORAGE_REGION:-us-east-1}
|
||
STORAGE_MAX_UPLOAD_SIZE_MB: ${STORAGE_MAX_UPLOAD_SIZE_MB:-10}
|
||
STORAGE_OSS_ENDPOINT: ${STORAGE_OSS_ENDPOINT:-}
|
||
STORAGE_OSS_BUCKET: ${STORAGE_OSS_BUCKET:-}
|
||
STORAGE_OSS_ACCESS_KEY_ID: ${STORAGE_OSS_ACCESS_KEY_ID:-}
|
||
STORAGE_OSS_SECRET_ACCESS_KEY: ${STORAGE_OSS_SECRET_ACCESS_KEY:-}
|
||
STORAGE_OSS_REGION: ${STORAGE_OSS_REGION:-oss-cn-hangzhou}
|
||
# ADMIN_SESSION_SECRET / ADMIN_DEFAULT_USERS_JSON 仅走 env_file。
|
||
# 勿在此处用 ${ADMIN_DEFAULT_USERS_JSON} 再注入:JSON 会被 YAML 解析破坏,导致生产校验失败。
|
||
KUASHOU_INDUSTRY_RATE_LIMIT_MAX: ${KUASHOU_INDUSTRY_RATE_LIMIT_MAX:-120}
|
||
logging: *json-log-rotation
|
||
volumes:
|
||
- ./apps/backend/data:/app/data
|
||
|
||
web:
|
||
build:
|
||
context: .
|
||
dockerfile: deploy/docker/frontend.Dockerfile
|
||
args:
|
||
APP_DOMAIN: ${APP_DOMAIN:-localhost}
|
||
VITE_WORKER_SITE_URL: ${WORKER_SITE_ADDR:-https://worker.khhao.com}
|
||
ALPINE_MIRROR: ${ALPINE_MIRROR:-mirrors.aliyun.com}
|
||
NPM_REGISTRY: ${NPM_REGISTRY:-https://registry.npmmirror.com}
|
||
restart: unless-stopped
|
||
mem_limit: ${WEB_MEM_LIMIT:-256m}
|
||
cpus: ${WEB_CPUS:-0.5}
|
||
stop_grace_period: 20s
|
||
logging: *json-log-rotation
|
||
depends_on:
|
||
backend:
|
||
condition: service_healthy
|
||
ports:
|
||
- "80:80"
|
||
- "443:443"
|
||
environment:
|
||
TZ: ${TZ:-Asia/Shanghai}
|
||
APP_DOMAIN: ${APP_DOMAIN:-localhost}
|
||
CADDY_SITE_ADDR: ${CADDY_SITE_ADDR:-http://localhost}
|
||
WORKER_SITE_ADDR: ${WORKER_SITE_ADDR:-https://worker.khhao.com}
|
||
volumes:
|
||
- caddy_data:/data
|
||
- caddy_config:/config
|
||
|
||
volumes:
|
||
postgres_data:
|
||
caddy_data:
|
||
caddy_config:
|