fix(db): 禁止修改已发布迁移并分批兼容旧库

This commit is contained in:
yml2213
2026-11-07 17:51:34 +08:00
parent 7b68d9b60a
commit 8ef8c61862
12 changed files with 253 additions and 123 deletions
+3 -2
View File
@@ -134,14 +134,15 @@ SKIP_CHECKS=1 git push
| 文件 | 含义 |
| --- | --- |
| `001_init.sql` | 空库基线,建立完整业务结构 |
| `002_xxx.sql` ~ `121_xxx.sql` | 后续结构与数据增量;当前生产发布基线为 119,120–121 为本次履约路由升级 |
| `002_xxx.sql` 起 | 后续结构与数据增量;`main` 中已有的迁移视为已发布且不可修改 |
约定:
1. **不要修改**已经应用到共享/生产环境的历史迁移文件。
1. **不要修改** `main` 中已经存在的历史迁移文件;`npm run check:sql` 会自动以 `main` 为基线保护全部已发布迁移。
2. 结构变更一律新增迁移文件,不要回写 `001_init.sql`。
3. 空库按序号依次执行 `001 → 002 → …`;已有库只执行尚未记录的增量。
4. 后端启动时自动 `up`;也可手动执行。
5. 旧库跨版本升级需要兼容清理时,使用显式前置脚本,不改历史迁移;当前 `107–113` 的兼容准备见 `deploy/sql/prepare-113-finance-cutover.sql`。
常用命令(容器内):
+6 -16
View File
@@ -7,6 +7,8 @@ import path from 'node:path'
import { execFileSync } from 'node:child_process'
import { fileURLToPath } from 'node:url'
import { changedPublishedMigrations } from '../src/db/migration-integrity.js'
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url))
const BACKEND_ROOT = path.resolve(SCRIPT_DIR, '..')
@@ -92,11 +94,6 @@ const failures: string[] = []
// 用「已发布基线」而不是 HEAD 做比较,是因为以 HEAD 为基线时,本次提交新增的
// 迁移也会被算成历史迁移,合并提交后必然自相矛盾。
const MIGRATION_BASELINE_REF = resolveMigrationBaselineRef()
// 生产当前已发布到 119;120+ 仍允许在部署前重写。部署 120 后应将此值提升到 120。
const PUBLISHED_MIGRATION_MAX_VERSION = Math.max(
0,
Math.trunc(Number(process.env.PUBLISHED_MIGRATION_MAX_VERSION) || 119),
)
function resolveMigrationBaselineRef(): string | null {
const candidates = [process.env.MIGRATION_BASELINE_REF, 'main', 'origin/main'].filter(
@@ -124,25 +121,18 @@ for (const file of listSourceFiles(path.join(BACKEND_ROOT, 'src'))) {
}
}
// 已发布的历史迁移必须保持字节不变,所有结构或数据修复都必须新增迁移文件。
// 基线分支中存在的所有迁移都必须保持字节不变;新增文件不受影响。
// 这样保护范围会随着 main 的发布自动扩展,不再依赖手工维护版本号。
if (!MIGRATION_BASELINE_REF) {
console.warn('[check-sql] 未找到迁移基线分支(main / origin/main),跳过迁移字节检查')
} else {
try {
const diffOutput = execFileSync(
'git',
['diff', '--name-status', MIGRATION_BASELINE_REF, '--', 'src/db/migrations'],
['diff', '--name-status', '--no-renames', MIGRATION_BASELINE_REF, '--', 'src/db/migrations'],
{ cwd: BACKEND_ROOT, encoding: 'utf8' },
)
const changedHistoricalMigrations = diffOutput
.split('\n')
.map((line) => line.trim().split(/\s+/))
.filter(([status, file]) => {
if (!status || !file?.endsWith('.sql') || status === 'A') return false
const version = Number(path.basename(file).slice(0, 3))
return Number.isFinite(version) && version <= PUBLISHED_MIGRATION_MAX_VERSION
})
.map(([status, file]) => `${status} ${file}`)
const changedHistoricalMigrations = changedPublishedMigrations(diffOutput)
if (changedHistoricalMigrations.length > 0) {
failures.push(
`已发布迁移文件不可修改(基线 ${MIGRATION_BASELINE_REF}): ${changedHistoricalMigrations.join(', ')}`,
-84
View File
@@ -24,69 +24,10 @@ const MIGRATIONS_DIR = path.join(CURRENT_DIR, 'migrations')
const MIGRATIONS_TABLE = 'schema_migrations'
const SHORT_NUMERIC_PREFIX_MESSAGE = /^Can't determine timestamp for \d+$/
const MIGRATION_FILE_PATTERN = /^(\d{3})_.+\.sql$/i
/**
* 合并 main 后寄卖迁移编号顺延,以及 142–148 收敛为单一文件:
* 129-131 -> 135-137(main 占用 129-134)
* 135-137 -> 142-144(main 又占用 135-141)
* 142-148 -> 142_consignment_platform(2026-11-06 合并为单文件)
* 2026-11-07 二次撞号:main 又占用 142-146,寄卖线整体顺延为 147-151
* 已跑过旧编号的环境要清掉旧记录,让新编号按正常顺序重跑(迁移本身幂等)。
*/
const RENAMED_MIGRATIONS: Array<{ legacyName: string; currentName: string }> = [
{ legacyName: '129_consignment_matching', currentName: '147_consignment_platform' },
{ legacyName: '135_consignment_matching', currentName: '147_consignment_platform' },
{ legacyName: '130_consignment_dynamic_specs', currentName: '147_consignment_platform' },
{ legacyName: '136_consignment_dynamic_specs', currentName: '147_consignment_platform' },
{
legacyName: '131_consignment_surnan_default_profile',
currentName: '147_consignment_platform',
},
{
legacyName: '137_consignment_surnan_default_profile',
currentName: '147_consignment_platform',
},
{ legacyName: '142_consignment_matching', currentName: '147_consignment_platform' },
{ legacyName: '143_consignment_dynamic_specs', currentName: '147_consignment_platform' },
{
legacyName: '144_consignment_surnan_default_profile',
currentName: '147_consignment_platform',
},
{
legacyName: '145_consignment_inventory_multi_binding',
currentName: '147_consignment_platform',
},
{ legacyName: '146_consignment_share_gift', currentName: '147_consignment_platform' },
{ legacyName: '147_consignment_match_and_rematch', currentName: '147_consignment_platform' },
{ legacyName: '148_consignment_remove_zone_help_text', currentName: '147_consignment_platform' },
/*
* 2026-11-07 二次撞号:合并 main 后 main 又占用 142-146(voucher/fulfillment 系列)。
* 寄卖迁移整体顺延:142→147、143→148、144→149、145/146→150、147(截止时间约束)→151。
* 旧库记录按下面的映射清掉后按新编号重跑,全部迁移均为幂等脚本。
*/
{ legacyName: '142_consignment_platform', currentName: '147_consignment_platform' },
{ legacyName: '143_consignment_sms_logs', currentName: '148_consignment_sms_logs' },
{
legacyName: '144_consignment_inventory_category_backfill',
currentName: '149_consignment_inventory_category_backfill',
},
{
legacyName: '145_work_order_consignment_original_reward',
currentName: '150_work_order_consignment_original_reward',
},
{
legacyName: '146_work_order_consignment_original_reward',
currentName: '150_work_order_consignment_original_reward',
},
{
legacyName: '147_work_order_internal_matching_deadline_guard',
currentName: '151_work_order_internal_matching_deadline_guard',
},
]
export async function runDatabaseMigrations() {
assertMigrationFilesOrdered()
await ensureNodePgMigrateMetadataCompatibility()
await dropRenamedMigrationRecords()
console.info(`[db:migrate] 迁移目录: ${MIGRATIONS_DIR}`)
@@ -239,31 +180,6 @@ async function ensureNodePgMigrateMetadataCompatibility() {
`)
}
/**
* 合并 main 后寄卖迁移编号顺延:清掉旧编号的已执行记录。
* 只在新文件存在、旧文件已不存在、且新编号尚未记录时执行,
* 之后由 runner 按新编号正常重跑(寄卖迁移均为幂等脚本)。
*/
async function dropRenamedMigrationRecords() {
for (const { legacyName, currentName } of RENAMED_MIGRATIONS) {
if (fs.existsSync(path.join(MIGRATIONS_DIR, `${legacyName}.sql`))) continue
if (!fs.existsSync(path.join(MIGRATIONS_DIR, `${currentName}.sql`))) continue
const result = await query(
`
DELETE FROM ${MIGRATIONS_TABLE}
WHERE name = $1
AND NOT EXISTS (
SELECT 1 FROM ${MIGRATIONS_TABLE} WHERE name = $2
)
`,
[legacyName, currentName],
)
if ((result.rowCount ?? 0) > 0) {
console.info(`[db:migrate] 迁移编号已顺延: ${legacyName} -> ${currentName}`)
}
}
}
async function syncLegacyMigrationColumns() {
await query(`
UPDATE ${MIGRATIONS_TABLE}
@@ -0,0 +1,27 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { changedPublishedMigrations } from './migration-integrity.js'
test('迁移完整性:新增迁移允许,已发布迁移任何修改都拒绝', () => {
assert.deepEqual(
changedPublishedMigrations(
[
'A\tsrc/db/migrations/153_new_feature.sql',
'M\tsrc/db/migrations/146_voucher_consume_fallback_channel.sql',
'D\tsrc/db/migrations/145_fulfillment_current_attempt_latest.sql',
].join('\n'),
),
[
'M src/db/migrations/146_voucher_consume_fallback_channel.sql',
'D src/db/migrations/145_fulfillment_current_attempt_latest.sql',
],
)
})
test('迁移完整性:保护范围不由固定版本号截断', () => {
assert.deepEqual(
changedPublishedMigrations('M\tsrc/db/migrations/120_fulfillment_routing_invariants.sql'),
['M src/db/migrations/120_fulfillment_routing_invariants.sql'],
)
})
@@ -0,0 +1,19 @@
import path from 'node:path'
/**
* 从 `git diff --name-status --no-renames <published-ref> -- src/db/migrations`
* 的输出中找出被修改的已发布迁移。
*
* 基线分支中不存在的新增文件状态为 A,可以继续修改;其他状态均视为历史迁移
* 被改动。这里不维护版本号,保护范围会随基线分支自动扩展。
*/
export function changedPublishedMigrations(diffOutput: string): string[] {
return diffOutput
.split('\n')
.map((line) => line.trim().split(/\s+/))
.filter(([status, file]) => {
if (!status || status === 'A' || !file?.endsWith('.sql')) return false
return path.basename(file).match(/^\d{3}_.+\.sql$/i) !== null
})
.map(([status, file]) => `${status} ${file}`)
}
@@ -338,12 +338,10 @@ ALTER TABLE cost_exceptions
ADD COLUMN IF NOT EXISTS last_seen_at TIMESTAMPTZ;
-- 存量行(断代脚本会整体清空)给一个可用的 scope_key,避免 NOT NULL 失败。
-- 旧行可能同 reason + subject_id 但 subject_type 不同,末尾拼 id 保证唯一,
-- 否则下面的 uq_cost_exceptions_scope_open 会因重复键失败。
UPDATE cost_exceptions
SET scope_key = COALESCE(
NULLIF(scope_key, ''),
reason || '|' || COALESCE(order_item_id::text, subject_id::text) || '|legacy:' || id
reason || '|' || COALESCE(order_item_id::text, subject_id::text)
),
first_seen_at = COALESCE(first_seen_at, created_at),
last_seen_at = COALESCE(last_seen_at, created_at)
@@ -406,22 +404,9 @@ COMMENT ON TABLE finance_watermarks IS
-- 先沿用旧列名承载渠道值,本迁移末尾统一改名为 channel;
-- 这里先放宽约束,允许卡券、未分类与归档状态。
-- 断代前的销售事实仍是旧口径,可能残留 legacy_platform;新约束不允许该值。
-- 这些事实随后由 finance:cutover-v2 整体清空,这里先归入 archived(未知值归入 unclassified)。
-- 必须先删除旧约束再改值:旧约束只允许四值,写 archived 会被它自己拦下。
ALTER TABLE sales_daily_facts
DROP CONSTRAINT IF EXISTS chk_sales_daily_facts_fulfillment_mode;
UPDATE sales_daily_facts
SET fulfillment_mode = CASE
WHEN fulfillment_mode = 'legacy_platform' THEN 'archived'
ELSE 'unclassified'
END
WHERE fulfillment_mode NOT IN (
'work_order', 'affiliate_dash', 'third_party_voucher',
'manual', 'unclassified', 'archived'
);
ALTER TABLE sales_daily_facts
ADD CONSTRAINT chk_sales_daily_facts_fulfillment_mode
CHECK (
@@ -12,8 +12,10 @@ const BACKEND_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)),
const REPO_ROOT = path.resolve(BACKEND_ROOT, '../..')
const CLASSIFIER = path.join(REPO_ROOT, 'deploy/lib/classify-changes.sh')
const DEPLOY_SCRIPT = path.join(REPO_ROOT, 'deploy/ubuntu-deploy.sh')
const CUTOVER_PREFLIGHT = path.join(REPO_ROOT, 'deploy/sql/prepare-113-finance-cutover.sql')
const hasDeployScripts = fs.existsSync(CLASSIFIER) && fs.existsSync(DEPLOY_SCRIPT)
const hasDeployScripts =
fs.existsSync(CLASSIFIER) && fs.existsSync(DEPLOY_SCRIPT) && fs.existsSync(CUTOVER_PREFLIGHT)
function classify(files: string[]): string {
return execFileSync('bash', [CLASSIFIER], {
@@ -60,7 +62,14 @@ test('改动分类:进得了镜像的部署文件要带上对应一端', { ski
test('改动分类:进不了镜像的改动不触发重建', { skip: !hasDeployScripts }, () => {
assert.equal(classify([]), '')
assert.equal(classify(['README.md', 'docs/部署启动与数据库重建.md']), '')
assert.equal(classify(['deploy/ubuntu-deploy.sh', 'deploy/lib/classify-changes.sh']), '')
assert.equal(
classify([
'deploy/ubuntu-deploy.sh',
'deploy/lib/classify-changes.sh',
'deploy/sql/prepare-113-finance-cutover.sql',
]),
'',
)
assert.equal(classify(['deploy/docker/backend-dev.Dockerfile']), '')
assert.equal(classify(['scripts/install-git-hooks.sh', '.gitignore']), '')
// 测试文件不进 dist、生产也不跑 npm test,只改测试不该重建镜像
@@ -107,10 +116,27 @@ test('改动分类:中文文件名不会被 git 转义成"漏判 → 两端全
test('部署脚本:基础镜像只补缺、部署状态在健康检查之后才落盘', { skip: !hasDeployScripts }, () => {
const source = fs.readFileSync(DEPLOY_SCRIPT, 'utf8')
const preflight = fs.readFileSync(CUTOVER_PREFLIGHT, 'utf8')
assert.ok(source.includes('--policy missing'), 'compose pull 应使用 --policy missing')
assert.ok(source.includes('write_deploy_state'), '缺少部署状态落盘')
assert.ok(source.includes('print_timing_summary'), '缺少阶段耗时输出')
assert.ok(source.includes('--git-diff'), '改动列表应通过 classify-changes.sh --git-diff 获取')
assert.ok(
source.indexOf('prepare_database_migrations\n') < source.indexOf('run_database_migrations\n'),
'迁移前置兼容检查必须在数据库迁移之前执行',
)
assert.ok(
preflight.includes("to_jsonb(migration)->>'filename' = '113_finance_channel_cutover.sql'"),
'迁移前置脚本必须识别 113 是否已经执行',
)
assert.ok(
preflight.includes('duplicate_scope_keys'),
'迁移前置脚本必须处理 cost_exceptions 重复 scope_key',
)
assert.ok(
preflight.includes("fulfillment_mode = 'legacy_platform'"),
'迁移前置脚本必须归一 sales_daily_facts 旧渠道',
)
const healthIndex = source.indexOf('wait_for_health\n')
const stateIndex = source.indexOf('write_deploy_state\n')
+1 -1
View File
@@ -53,7 +53,7 @@ classify_changed_paths() {
docs/* | *.md | LICENSE) ;;
.gitignore | .editorconfig | .prettierignore | .prettierrc.json) ;;
.vscode/* | scripts/* | .githooks/*) ;;
deploy/lib/* | deploy/*.sh) ;;
deploy/lib/* | deploy/*.sh | deploy/sql/*) ;;
# 其余(compose 文件、根目录杂项、未知路径)一律两端重建
*) backend=1; web=1 ;;
+150
View File
@@ -0,0 +1,150 @@
-- 113 迁移前置兼容修复。
--
-- 113 已经进入生产,文件必须保持字节不变。旧快照库中存在两类历史脏数据,
-- 会让原版 113 在“加新约束/唯一索引”时失败:
-- 1. sales_daily_facts 仍有旧约束不接受的渠道值;
-- 2. cost_exceptions 同一范围的旧行会生成重复 scope_key。
--
-- 本文件不是迁移,不写 schema_migrations。它只做幂等兼容准备:
-- - 113 已执行时立即跳过;
-- - 清理旧库数据,让随后执行的原版 113 可以通过。
--
-- 由 deploy/ubuntu-deploy.sh 在数据库迁移前自动执行。
DO $preflight$
DECLARE
v_has_113 BOOLEAN := FALSE;
v_sales_has_mode BOOLEAN := FALSE;
v_scope_duplicates BIGINT := 0;
v_scope_missing BIGINT := 0;
BEGIN
IF to_regclass('public.schema_migrations') IS NOT NULL THEN
EXECUTE $query$
SELECT EXISTS (
SELECT 1
FROM public.schema_migrations migration
WHERE to_jsonb(migration)->>'filename' = '113_finance_channel_cutover.sql'
OR to_jsonb(migration)->>'name' = '113_finance_channel_cutover'
)
$query$ INTO v_has_113;
END IF;
IF v_has_113 THEN
RAISE NOTICE '113 已执行,跳过前置兼容修复';
RETURN;
END IF;
IF to_regclass('public.sales_daily_facts') IS NOT NULL THEN
SELECT EXISTS (
SELECT 1
FROM pg_attribute
WHERE attrelid = 'public.sales_daily_facts'::regclass
AND attname = 'fulfillment_mode'
AND NOT attisdropped
) INTO v_sales_has_mode;
IF v_sales_has_mode THEN
EXECUTE $sql$
ALTER TABLE public.sales_daily_facts
DROP CONSTRAINT IF EXISTS chk_sales_daily_facts_fulfillment_mode
$sql$;
EXECUTE $sql$
UPDATE public.sales_daily_facts
SET fulfillment_mode = CASE
WHEN fulfillment_mode = 'legacy_platform' THEN 'archived'
ELSE 'unclassified'
END
WHERE fulfillment_mode NOT IN (
'work_order', 'affiliate_dash', 'third_party_voucher',
'manual', 'unclassified', 'archived'
)
$sql$;
-- 迁移前临时使用新旧并集的约束;原版 113 会立即替换成最终约束。
EXECUTE $sql$
ALTER TABLE public.sales_daily_facts
ADD CONSTRAINT chk_sales_daily_facts_fulfillment_mode
CHECK (
fulfillment_mode IN (
'work_order', 'affiliate_dash', 'third_party_voucher',
'manual', 'unclassified', 'archived'
)
)
$sql$;
RAISE NOTICE 'sales_daily_facts 历史渠道值已兼容归一';
END IF;
END IF;
IF to_regclass('public.cost_exceptions') IS NOT NULL THEN
EXECUTE $sql$
ALTER TABLE public.cost_exceptions
ADD COLUMN IF NOT EXISTS scope_key TEXT,
ADD COLUMN IF NOT EXISTS item_count INTEGER NOT NULL DEFAULT 1,
ADD COLUMN IF NOT EXISTS first_seen_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS last_seen_at TIMESTAMPTZ
$sql$;
EXECUTE $sql$
WITH duplicate_scope_keys AS (
SELECT scope_key
FROM public.cost_exceptions
WHERE status <> 'resolved'
AND NULLIF(BTRIM(scope_key), '') IS NOT NULL
GROUP BY scope_key
HAVING COUNT(*) > 1
)
UPDATE public.cost_exceptions ce
SET scope_key =
COALESCE(NULLIF(BTRIM(ce.reason), ''), 'legacy')
|| '|'
|| COALESCE(ce.order_item_id::text, ce.subject_id::text, ce.id::text)
|| '|legacy:'
|| ce.id,
first_seen_at = COALESCE(ce.first_seen_at, ce.created_at),
last_seen_at = COALESCE(ce.last_seen_at, ce.created_at)
WHERE ce.scope_key IS NULL
OR BTRIM(ce.scope_key) = ''
OR ce.first_seen_at IS NULL
OR ce.last_seen_at IS NULL
OR (
ce.status <> 'resolved'
AND ce.scope_key IN (SELECT scope_key FROM duplicate_scope_keys)
)
$sql$;
EXECUTE $sql$
SELECT COUNT(*)
FROM (
SELECT scope_key
FROM public.cost_exceptions
WHERE status <> 'resolved'
AND NULLIF(BTRIM(scope_key), '') IS NOT NULL
GROUP BY scope_key
HAVING COUNT(*) > 1
) duplicate_scope_keys
$sql$ INTO v_scope_duplicates;
EXECUTE $sql$
SELECT COUNT(*)
FROM public.cost_exceptions
WHERE scope_key IS NULL OR BTRIM(scope_key) = ''
$sql$ INTO v_scope_missing;
IF v_scope_duplicates > 0 THEN
RAISE EXCEPTION
'cost_exceptions 仍有 % 组重复 scope_key,113 无法创建唯一索引',
v_scope_duplicates;
END IF;
IF v_scope_missing > 0 THEN
RAISE EXCEPTION
'cost_exceptions 仍有 % 行 scope_key 为空,113 的唯一索引前置数据不完整',
v_scope_missing;
END IF;
RAISE NOTICE 'cost_exceptions 历史 scope_key 已保证唯一';
END IF;
END
$preflight$;
+16
View File
@@ -666,6 +666,21 @@ start_postgres() {
fail "PostgreSQL 启动失败,请根据上面的日志排查。"
}
prepare_database_migrations() {
local preflight_sql="${PROJECT_ROOT}/deploy/sql/prepare-113-finance-cutover.sql"
[[ -f "${preflight_sql}" ]] || fail "缺少迁移前置兼容脚本:${preflight_sql}"
log "检查旧库的迁移前置兼容数据"
if docker compose exec -T postgres sh -lc \
'psql -v ON_ERROR_STOP=1 -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-order_site}"' \
< "${preflight_sql}"; then
return
fi
print_diagnostics
fail "迁移前置兼容检查失败,已停止部署"
}
run_database_migrations() {
log "执行数据库迁移"
if docker compose run --rm --no-deps backend node dist/db/migrate.js; then
@@ -810,6 +825,7 @@ deploy_stack() {
reset_database
time_step "启动 PostgreSQL" start_postgres
time_step "迁移前备份" backup_before_migrate
time_step "检查迁移前置兼容数据" prepare_database_migrations
time_step "执行数据库迁移" run_database_migrations
time_step "启动后端" restart_backend
time_step "启动 Web 入口" restart_web
@@ -34,7 +34,7 @@ NFKC -> 删除全部空白 -> 转小写
## 部署后核对
部署生产时把 `PUBLISHED_MIGRATION_MAX_VERSION` 提升到 `129`。
历史迁移保护范围会自动覆盖 `main` 中已有的迁移,不需要手工提升版本号。
```sql
SELECT pipeline, last_item_id, last_success_at, last_error
+1 -1
View File
@@ -50,7 +50,7 @@
- 124:`work_orders` 增加付款金额旧价、修改时间、修改人三列(`ADD COLUMN IF NOT EXISTS`),幂等且不动存量数据。
- 125:新建 `work_order_payment_amount_history` 并回填已有 `payment_amount_updated` 事件,写入按工单级联删除。
- 126:统一中文内部 SKU 规范化函数,平台 SKU 不再作为渠道兜底;补充统一配置查询索引。
- 生产已发布到 125;部署 126 后应把 `PUBLISHED_MIGRATION_MAX_VERSION` 提升为 126。
- 历史迁移保护范围自动取 `main` 中已有的迁移文件;合并后无需手工维护发布版本号。
## 3. 部署顺序