chore(deploy): 上线脚本与验收 SQL 适配渠道模型
- post-deploy 去掉已删除的履约回填与 finance:sync,改用 finance:rebuild-cutover - 验收 SQL 的履约检查改读 fulfillment_channel/status - 相关文档同步更新
This commit is contained in:
+12
-17
@@ -7,8 +7,8 @@ set -Eeuo pipefail
|
|||||||
#
|
#
|
||||||
# 它把以前散落在文档里的脚本步骤串成一条可重复执行的流水线:
|
# 它把以前散落在文档里的脚本步骤串成一条可重复执行的流水线:
|
||||||
# ① 存量订单项收入回填 revenue:backfill
|
# ① 存量订单项收入回填 revenue:backfill
|
||||||
# ② 履约方式回填 fulfillment:backfill
|
# ② 渠道/商品由管道处理(无需回填)
|
||||||
# ③ 财务历史同步(分窗口) finance:sync --mode=all
|
# ③ 财务历史重建(分窗口) finance:rebuild-cutover
|
||||||
# ④ 上线验收(只读 SQL) scripts/verify-finance-launch.sql
|
# ④ 上线验收(只读 SQL) scripts/verify-finance-launch.sql
|
||||||
#
|
#
|
||||||
# 设计原则:
|
# 设计原则:
|
||||||
@@ -343,7 +343,7 @@ preflight() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
PENDING_REVENUE="$(psql_scalar "SELECT count(*) FROM order_items WHERE revenue_basis = 'unknown'" | tr -d '[:space:]')"
|
PENDING_REVENUE="$(psql_scalar "SELECT count(*) FROM order_items WHERE revenue_basis = 'unknown'" | tr -d '[:space:]')"
|
||||||
PENDING_FULFILLMENT="$(psql_scalar "SELECT count(*) FROM order_items WHERE fulfillment_mode IS NULL" | tr -d '[:space:]')"
|
PENDING_FULFILLMENT="$(psql_scalar "SELECT count(*) FROM order_items WHERE fulfillment_status = 'pending'" | tr -d '[:space:]')"
|
||||||
|
|
||||||
log " postgres 运行中 / 迁移 103–106 已应用"
|
log " postgres 运行中 / 迁移 103–106 已应用"
|
||||||
log " 待回填收入项:${PENDING_REVENUE} 行;待回填履约方式:${PENDING_FULFILLMENT} 行"
|
log " 待回填收入项:${PENDING_REVENUE} 行;待回填履约方式:${PENDING_FULFILLMENT} 行"
|
||||||
@@ -362,21 +362,16 @@ step_backfill() {
|
|||||||
run_backfill revenue:backfill --batchSize="$BATCH_SIZE" --maxBatches="$MAX_BATCHES"
|
run_backfill revenue:backfill --batchSize="$BATCH_SIZE" --maxBatches="$MAX_BATCHES"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
log "② 回填履约方式(fulfillment:backfill,只填 NULL)"
|
log "② 渠道判定与商品归一出管道负责,无需手工回填"
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
|
||||||
log " [dry-run] 跳过执行"
|
|
||||||
else
|
|
||||||
run_backfill fulfillment:backfill --batchSize="$BATCH_SIZE" --maxBatches="$MAX_BATCHES"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" != true ]]; then
|
if [[ "$DRY_RUN" != true ]]; then
|
||||||
refresh_stats
|
refresh_stats
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# 回填会 UPDATE 40 万行 order_items:既产生大量死元组,又给新列 fulfillment_mode
|
# 回填会 UPDATE 40 万行 order_items:既产生大量死元组,又给新列留下空统计。
|
||||||
# 留下空统计。autovacuum 追不上时,订单列表的 COUNT 会被估算成 orders 全表扫描,
|
# autovacuum 追不上时,订单列表的 COUNT 会被估算成 orders 全表扫描,冷缓存实测
|
||||||
# 冷缓存实测 20 秒级(测试库曾因此 500)。这里在回填后立刻补一次,VACUUM 只拿
|
# 20 秒级(测试库曾因此 500)。这里在回填后立刻补一次,VACUUM 只拿
|
||||||
# SHARE UPDATE EXCLUSIVE,不阻塞读写。
|
# SHARE UPDATE EXCLUSIVE,不阻塞读写。
|
||||||
refresh_stats() {
|
refresh_stats() {
|
||||||
log "②' 刷新统计信息与可见性(VACUUM ANALYZE order_items/orders)"
|
log "②' 刷新统计信息与可见性(VACUUM ANALYZE order_items/orders)"
|
||||||
@@ -391,19 +386,19 @@ step_sync() {
|
|||||||
local products
|
local products
|
||||||
products="$(psql_scalar 'SELECT count(*) FROM product_catalog' | tr -d '[:space:]')"
|
products="$(psql_scalar 'SELECT count(*) FROM product_catalog' | tr -d '[:space:]')"
|
||||||
if [[ "$products" == "0" ]]; then
|
if [[ "$products" == "0" ]]; then
|
||||||
warn "商品目录为空:finance:sync 会为未归一订单项生成大量 product_mapping_missing 异常;建议先完成商品初始化再跑 --only=sync"
|
warn "商品目录为空:财务重建会生成大量 product_mapping_missing 异常;建议先完成商品初始化再跑 --only=sync"
|
||||||
fi
|
fi
|
||||||
log "③ 财务历史同步(finance:sync --mode=all),区间 ${from} → ${to},每窗 ${WINDOW_DAYS} 天"
|
log "③ 财务历史重建(finance:rebuild-cutover),区间 ${from} → ${to},每窗 ${WINDOW_DAYS} 天"
|
||||||
while :; do
|
while :; do
|
||||||
win_to="$(date_add_days "$from" $((WINDOW_DAYS - 1)))"
|
win_to="$(date_add_days "$from" $((WINDOW_DAYS - 1)))"
|
||||||
if [[ "$win_to" > "$to" ]]; then
|
if [[ "$win_to" > "$to" ]]; then
|
||||||
win_to="$to"
|
win_to="$to"
|
||||||
fi
|
fi
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
log " [dry-run] npm run finance:sync -- --mode=all --from=${from} --to=${win_to}"
|
log " [dry-run] npm run finance:rebuild-cutover -- --date-from=${from} --date-to=${win_to}"
|
||||||
else
|
else
|
||||||
log " 窗口 ${from} → ${win_to}"
|
log " 窗口 ${from} → ${win_to}"
|
||||||
run_tool finance:sync --mode=all --from="$from" --to="$win_to"
|
run_tool finance:rebuild-cutover --date-from="$from" --date-to="$win_to"
|
||||||
fi
|
fi
|
||||||
[[ "$win_to" == "$to" ]] && break
|
[[ "$win_to" == "$to" ]] && break
|
||||||
from="$(date_add_days "$win_to" 1)"
|
from="$(date_add_days "$win_to" 1)"
|
||||||
@@ -433,7 +428,7 @@ manual_checklist() {
|
|||||||
剩余人工步骤(后台 UI,脚本无法代替):
|
剩余人工步骤(后台 UI,脚本无法代替):
|
||||||
1) 钱包管理 → 补建客服账户 → 拨付备用金
|
1) 钱包管理 → 补建客服账户 → 拨付备用金
|
||||||
2) 商品与成本 → 积分钱包 → 发放三平台积分
|
2) 商品与成本 → 积分钱包 → 发放三平台积分
|
||||||
3) 商品与成本 → 标准商品 →「从接单模板初始化」→「按工单回填商品」→「归一人工商品」
|
3) 商品与成本 → 标准商品 →「从接单模板初始化」(一次性建模板商品与映射)
|
||||||
4) 商品与成本 → 人工 SKU 配置 →「扫描」并逐项配置
|
4) 商品与成本 → 人工 SKU 配置 →「扫描」并逐项配置
|
||||||
5) 财务 → 总账与对账 →「同步销售事实到总账」→ 试算平衡 → 按需关账
|
5) 财务 → 总账与对账 →「同步销售事实到总账」→ 试算平衡 → 按需关账
|
||||||
首次上线顺序:--only=backfill → 完成 1)–4) → --only=sync → 完成 5) → --only=verify
|
首次上线顺序:--only=backfill → 完成 1)–4) → --only=sync → 完成 5) → --only=verify
|
||||||
|
|||||||
@@ -230,7 +230,7 @@ resolveCanonicalProduct(provider, shop_id, sku_code, sku_name)
|
|||||||
|
|
||||||
| 脚本 | 作用 | 幂等 |
|
| 脚本 | 作用 | 幂等 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `scripts/backfill-fulfillment-channel.ts` | 438k 行分批回填三列 | ✅ 可续跑 |
|
| ~~`scripts/backfill-fulfillment-channel.ts`~~ | 已删除:断代脚本自带归档 | — |
|
||||||
| `scripts/bootstrap-channel-rules.ts` | 从 aff 商品 / 人工配置生成规则 | ✅ |
|
| `scripts/bootstrap-channel-rules.ts` | 从 aff 商品 / 人工配置生成规则 | ✅ |
|
||||||
| `scripts/finance-cutover-v2.ts` | 断代归档 + 派生数据清零 | 一次性(有 cutover_meta 护栏) |
|
| `scripts/finance-cutover-v2.ts` | 断代归档 + 派生数据清零 | 一次性(有 cutover_meta 护栏) |
|
||||||
| `scripts/rebuild-after-cutover.ts` | 10-31 起点重建 + 期初建账 | ✅ 可重放 |
|
| `scripts/rebuild-after-cutover.ts` | 10-31 起点重建 + 期初建账 | ✅ 可重放 |
|
||||||
|
|||||||
+18
-11
@@ -399,21 +399,28 @@ WHERE oi.id = d.id
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 七、同步脚本与调度
|
## 七、管道与调度
|
||||||
|
|
||||||
|
财务链路已改成六条独立管道,由定时任务驱动,不再有手工同步脚本:
|
||||||
|
|
||||||
|
| 管道 | 作用 | 默认间隔 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `channel_assign` | 规则/证据判定履约渠道 | 60s |
|
||||||
|
| `carrier_dispatch` | 挂载人工配置、发现缺失载体 | 60s |
|
||||||
|
| `product_normalize` | 模板/人工配置/前缀直达归一 | 300s |
|
||||||
|
| `cost_snapshot` | 按渠道解析成本版本写快照 | 300s |
|
||||||
|
| `sales_facts` | 按渠道重算销售事实(含未分类桶) | 600s |
|
||||||
|
| `ledger_post` | 事实过账到总账 | 600s |
|
||||||
|
|
||||||
|
运维命令只剩断代与重建:
|
||||||
|
|
||||||
| 命令 | 作用 |
|
| 命令 | 作用 |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `npm run fulfillment:backfill` | 履约方式分批回填(只填 NULL) |
|
| `npm run finance:cutover-v2 -- --execute --confirm=2026-10-31` | 断代:归档历史 + 清空派生数据 |
|
||||||
| `npm run finance:sync -- --mode=all` | 预估快照 + 实际快照 + 销售事实重算 |
|
| `npm run finance:rebuild-cutover -- --date-from=2026-10-31` | 断代后重跑管道并生成期初分录 |
|
||||||
| `npm run finance:sync -- --mode=estimated` | 只算预估快照并写缺失成本异常 |
|
|
||||||
| `npm run finance:sync -- --mode=actual` | 只把工单快照升级为实际金额 |
|
|
||||||
| `npm run finance:sync -- --mode=facts` | 只重算销售事实 |
|
|
||||||
|
|
||||||
长历史重算要分窗口:单次 `rebuildSalesDailyFacts` 超过约 400 天会被语句超时(`DATABASE_STATEMENT_TIMEOUT_MS`,默认 15 秒)打断。本地 2026-07-10 ~ 2026-10-23 分 4 段重算,正常。
|
后台「履约渠道 → 管道状态」可查看每条管道的水位、上次成功时间与错误;
|
||||||
|
「立即执行」只用于排障补跑,日常无需干预。
|
||||||
定时任务 `services/scheduler/cost-finance-sync-job.ts` 每 5 分钟先归集待判定的订单项,再算成本与销售事实;默认只处理最近 7 天(可在 `scheduled_jobs` 配置里调大,测试/小机器不建议超过 30 天,避免单条语句超时)。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 八、迁移基线与本地库同步(2026-10-27 收敛)
|
## 八、迁移基线与本地库同步(2026-10-27 收敛)
|
||||||
|
|
||||||
|
|||||||
@@ -163,8 +163,7 @@ GROUP BY status;
|
|||||||
```bash
|
```bash
|
||||||
npm run revenue:backfill -- --dry-run # 待回填量,应为 0
|
npm run revenue:backfill -- --dry-run # 待回填量,应为 0
|
||||||
npm run revenue:backfill # 分批回填;中断可重跑
|
npm run revenue:backfill # 分批回填;中断可重跑
|
||||||
npm run fulfillment:backfill # 只填 NULL(可选,报表用)
|
npm run finance:rebuild-cutover -- --date-from=2026-07-10 --date-to=2026-10-22
|
||||||
npm run finance:sync -- --mode=all --from=2026-07-10 --to=2026-10-22
|
|
||||||
```
|
```
|
||||||
|
|
||||||
> 回填完成后必须刷新统计信息,否则订单列表的分页总数会被规划成 `orders` 全表扫描:
|
> 回填完成后必须刷新统计信息,否则订单列表的分页总数会被规划成 `orders` 全表扫描:
|
||||||
|
|||||||
+1
-1
@@ -234,7 +234,7 @@ bash deploy/post-deploy.sh --only=backfill # ① 收入回填 + ② 履约
|
|||||||
> 不要在商品目录为空时直接跑 `--only=sync`:财务同步会为每个未归一订单项写一条
|
> 不要在商品目录为空时直接跑 `--only=sync`:财务同步会为每个未归一订单项写一条
|
||||||
> `product_mapping_missing` 成本异常(生产规模 90 天实测约 36 万条),后台异常队列会被淹没。
|
> `product_mapping_missing` 成本异常(生产规模 90 天实测约 36 万条),后台异常队列会被淹没。
|
||||||
|
|
||||||
脚本包含 `revenue:backfill`、`fulfillment:backfill`、`finance:sync` 与只读验收四段;任何一步都能重跑,
|
脚本包含 `revenue:backfill`、渠道/成本管道重建(`finance:rebuild-cutover`)与只读验收;任何一步都能重跑,
|
||||||
`--only=backfill|sync|verify` 可单独执行。
|
`--only=backfill|sync|verify` 可单独执行。
|
||||||
|
|
||||||
> 生产 backend 镜像没有 `tsx` 与 `scripts/`,脚本通过 `tools` 镜像执行
|
> 生产 backend 镜像没有 `tsx` 与 `scripts/`,脚本通过 `tools` 镜像执行
|
||||||
|
|||||||
@@ -68,27 +68,29 @@ FROM (
|
|||||||
SELECT count(*) AS v FROM order_items WHERE revenue_basis = 'unknown'
|
SELECT count(*) AS v FROM order_items WHERE revenue_basis = 'unknown'
|
||||||
) s;
|
) s;
|
||||||
|
|
||||||
-- 03 履约方式回填 ----------------------------------------------------------
|
-- 03 履约渠道/状态回填 -----------------------------------------------------
|
||||||
INSERT INTO _launch_checks
|
INSERT INTO _launch_checks
|
||||||
SELECT 3, '03', '履约方式已回填(fulfillment_mode)',
|
SELECT 3, '03', '履约渠道与状态合法(fulfillment_channel/status)',
|
||||||
v::text, '0',
|
v::text, '0',
|
||||||
CASE WHEN v = 0 THEN 'PASS' ELSE 'FAIL' END,
|
CASE WHEN v = 0 THEN 'PASS' ELSE 'FAIL' END,
|
||||||
'跑 npm run fulfillment:backfill;NULL 会让订单/成本页口径缺失'
|
'存在非法状态:检查迁移 112 与渠道判定管道(channel_assign)'
|
||||||
FROM (
|
|
||||||
SELECT count(*) AS v FROM order_items WHERE fulfillment_mode IS NULL
|
|
||||||
) s;
|
|
||||||
|
|
||||||
-- 04 履约方式取值合法 ------------------------------------------------------
|
|
||||||
INSERT INTO _launch_checks
|
|
||||||
SELECT 4, '04', '履约方式取值合法',
|
|
||||||
v::text, '0',
|
|
||||||
CASE WHEN v = 0 THEN 'PASS' ELSE 'FAIL' END,
|
|
||||||
'存在未知 fulfillment_mode,检查回填脚本与历史脏数据'
|
|
||||||
FROM (
|
FROM (
|
||||||
SELECT count(*) AS v
|
SELECT count(*) AS v
|
||||||
FROM order_items
|
FROM order_items
|
||||||
WHERE fulfillment_mode IS NOT NULL
|
WHERE fulfillment_status NOT IN ('pending', 'assigned', 'archived')
|
||||||
AND fulfillment_mode NOT IN ('work_order', 'manual', 'legacy_platform', 'affiliate_dash')
|
) s;
|
||||||
|
|
||||||
|
-- 04 履约渠道取值合法 ------------------------------------------------------
|
||||||
|
INSERT INTO _launch_checks
|
||||||
|
SELECT 4, '04', '履约渠道取值合法',
|
||||||
|
v::text, '0',
|
||||||
|
CASE WHEN v = 0 THEN 'PASS' ELSE 'FAIL' END,
|
||||||
|
'存在未知 fulfillment_channel,检查渠道规则与管道写入'
|
||||||
|
FROM (
|
||||||
|
SELECT count(*) AS v
|
||||||
|
FROM order_items
|
||||||
|
WHERE fulfillment_status = 'assigned'
|
||||||
|
AND fulfillment_channel NOT IN ('work_order', 'affiliate_dash', 'third_party_voucher', 'manual')
|
||||||
) s;
|
) s;
|
||||||
|
|
||||||
-- 05 商品目录 --------------------------------------------------------------
|
-- 05 商品目录 --------------------------------------------------------------
|
||||||
@@ -139,7 +141,8 @@ SELECT 8, '08', '上线后人工项已匹配履约配置',
|
|||||||
FROM (
|
FROM (
|
||||||
SELECT count(*) AS v
|
SELECT count(*) AS v
|
||||||
FROM order_items oi
|
FROM order_items oi
|
||||||
WHERE oi.fulfillment_mode = 'manual'
|
WHERE oi.fulfillment_status = 'assigned'
|
||||||
|
AND oi.fulfillment_channel = 'manual'
|
||||||
AND oi.created_at >= :'launch_at'::timestamptz
|
AND oi.created_at >= :'launch_at'::timestamptz
|
||||||
AND oi.fulfillment_config_id IS NULL
|
AND oi.fulfillment_config_id IS NULL
|
||||||
) s;
|
) s;
|
||||||
@@ -153,7 +156,8 @@ SELECT 9, '09', '上线后人工项均已确认履约(关账门槛)',
|
|||||||
FROM (
|
FROM (
|
||||||
SELECT count(*) AS v
|
SELECT count(*) AS v
|
||||||
FROM order_items oi
|
FROM order_items oi
|
||||||
WHERE oi.fulfillment_mode = 'manual'
|
WHERE oi.fulfillment_status = 'assigned'
|
||||||
|
AND oi.fulfillment_channel = 'manual'
|
||||||
AND oi.created_at >= :'launch_at'::timestamptz
|
AND oi.created_at >= :'launch_at'::timestamptz
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1 FROM cost_snapshots cs
|
SELECT 1 FROM cost_snapshots cs
|
||||||
@@ -235,7 +239,7 @@ INSERT INTO _launch_checks
|
|||||||
SELECT 14, '14', '销售事实已生成(sales_daily_facts)',
|
SELECT 14, '14', '销售事实已生成(sales_daily_facts)',
|
||||||
v::text, '> 0',
|
v::text, '> 0',
|
||||||
CASE WHEN v > 0 THEN 'PASS' ELSE 'FAIL' END,
|
CASE WHEN v > 0 THEN 'PASS' ELSE 'FAIL' END,
|
||||||
'跑 finance:sync --mode=facts(post-deploy.sh 会按窗口执行);定时任务只覆盖最近 7 天'
|
'跑 finance:rebuild-cutover(post-deploy.sh 会按窗口执行);定时任务只回看水位窗口'
|
||||||
FROM (SELECT count(*) AS v FROM sales_daily_facts) s;
|
FROM (SELECT count(*) AS v FROM sales_daily_facts) s;
|
||||||
|
|
||||||
-- 15 近 7 天成本覆盖(仅提示,不阻断上线) ---------------------------------
|
-- 15 近 7 天成本覆盖(仅提示,不阻断上线) ---------------------------------
|
||||||
|
|||||||
Reference in New Issue
Block a user