Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4159a435d5 | ||
|
|
9ff9eda3b3 | ||
|
|
93dec72b2e |
@@ -72,12 +72,6 @@ LOG_RETENTION_DAYS=30
|
|||||||
LOG_MAX_FILE_SIZE_MB=20
|
LOG_MAX_FILE_SIZE_MB=20
|
||||||
LOG_MAX_FILES=5
|
LOG_MAX_FILES=5
|
||||||
|
|
||||||
# 数据库敏感报文/事件保留策略(天)。清理任务每天执行一次。
|
|
||||||
RAW_PAYLOAD_RETENTION_DAYS=180
|
|
||||||
TASK_EVENT_RETENTION_DAYS=180
|
|
||||||
WEBHOOK_EVENT_RETENTION_DAYS=180
|
|
||||||
AUDIT_LOG_RETENTION_DAYS=365
|
|
||||||
WORK_ORDER_EVENT_RETENTION_DAYS=180
|
|
||||||
# 已处理成本异常保留天数,超期搬入 cost_exceptions_archive(open 行不归档)
|
# 已处理成本异常保留天数,超期搬入 cost_exceptions_archive(open 行不归档)
|
||||||
COST_EXCEPTION_RETENTION_DAYS=60
|
COST_EXCEPTION_RETENTION_DAYS=60
|
||||||
|
|
||||||
|
|||||||
@@ -109,7 +109,7 @@ SKIP_CHECKS=1 git push
|
|||||||
| 文件 | 含义 |
|
| 文件 | 含义 |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `001_init.sql` | 空库基线,建立完整业务结构 |
|
| `001_init.sql` | 空库基线,建立完整业务结构 |
|
||||||
| `002_xxx.sql` ~ `121_xxx.sql` | 后续结构与数据增量;当前生产发布基线为 119,120–121 为本次履约路由升级 |
|
| `002_xxx.sql` ~ `152_xxx.sql` | 后续结构与数据增量;当前生产基线为 151,152 为本地待发布的事实保护与路由升级 |
|
||||||
|
|
||||||
约定:
|
约定:
|
||||||
|
|
||||||
|
|||||||
@@ -24,12 +24,6 @@ export function createDefaultRuntimeConfig(projectRoot: string): RuntimeConfig {
|
|||||||
},
|
},
|
||||||
|
|
||||||
retention: {
|
retention: {
|
||||||
// 原始报文只保留追溯所需窗口,清理后保留订单/事件摘要。
|
|
||||||
rawPayloadDays: 180,
|
|
||||||
taskEventDays: 180,
|
|
||||||
webhookEventDays: 180,
|
|
||||||
auditLogDays: 365,
|
|
||||||
workOrderEventDays: 180,
|
|
||||||
// 已处理的成本异常保留 60 天,之后搬进 cost_exceptions_archive;open 行不归档。
|
// 已处理的成本异常保留 60 天,之后搬进 cost_exceptions_archive;open 行不归档。
|
||||||
costExceptionDays: 60,
|
costExceptionDays: 60,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -96,10 +96,6 @@ function createRuntimeConfig(): RuntimeConfig {
|
|||||||
retentionDays: 30,
|
retentionDays: 30,
|
||||||
},
|
},
|
||||||
retention: {
|
retention: {
|
||||||
rawPayloadDays: 180,
|
|
||||||
taskEventDays: 180,
|
|
||||||
webhookEventDays: 180,
|
|
||||||
auditLogDays: 365,
|
|
||||||
costExceptionDays: 60,
|
costExceptionDays: 60,
|
||||||
},
|
},
|
||||||
database: {
|
database: {
|
||||||
|
|||||||
@@ -30,11 +30,6 @@ export const ENV_OVERRIDES: readonly EnvOverride[] = [
|
|||||||
integerEnv('LOG_RETENTION_DAYS', ['logging', 'retentionDays']),
|
integerEnv('LOG_RETENTION_DAYS', ['logging', 'retentionDays']),
|
||||||
integerEnv('LOG_MAX_FILE_SIZE_MB', ['logging', 'maxFileSizeMb']),
|
integerEnv('LOG_MAX_FILE_SIZE_MB', ['logging', 'maxFileSizeMb']),
|
||||||
integerEnv('LOG_MAX_FILES', ['logging', 'maxFiles']),
|
integerEnv('LOG_MAX_FILES', ['logging', 'maxFiles']),
|
||||||
integerEnv('RAW_PAYLOAD_RETENTION_DAYS', ['retention', 'rawPayloadDays']),
|
|
||||||
integerEnv('TASK_EVENT_RETENTION_DAYS', ['retention', 'taskEventDays']),
|
|
||||||
integerEnv('WEBHOOK_EVENT_RETENTION_DAYS', ['retention', 'webhookEventDays']),
|
|
||||||
integerEnv('AUDIT_LOG_RETENTION_DAYS', ['retention', 'auditLogDays']),
|
|
||||||
integerEnv('WORK_ORDER_EVENT_RETENTION_DAYS', ['retention', 'workOrderEventDays']),
|
|
||||||
integerEnv('COST_EXCEPTION_RETENTION_DAYS', ['retention', 'costExceptionDays']),
|
integerEnv('COST_EXCEPTION_RETENTION_DAYS', ['retention', 'costExceptionDays']),
|
||||||
stringEnv('DATABASE_URL', ['database', 'url']),
|
stringEnv('DATABASE_URL', ['database', 'url']),
|
||||||
booleanEnv('DATABASE_SSL', ['database', 'ssl']),
|
booleanEnv('DATABASE_SSL', ['database', 'ssl']),
|
||||||
|
|||||||
@@ -160,10 +160,6 @@ function createRuntimeConfig(overrides: Record<string, unknown> = {}): RuntimeCo
|
|||||||
integrationLevel: 'warn',
|
integrationLevel: 'warn',
|
||||||
},
|
},
|
||||||
retention: {
|
retention: {
|
||||||
rawPayloadDays: 180,
|
|
||||||
taskEventDays: 180,
|
|
||||||
webhookEventDays: 180,
|
|
||||||
auditLogDays: 365,
|
|
||||||
costExceptionDays: 60,
|
costExceptionDays: 60,
|
||||||
},
|
},
|
||||||
database: {
|
database: {
|
||||||
|
|||||||
@@ -82,12 +82,6 @@ export function validateRuntimeConfig(
|
|||||||
max: 100,
|
max: 100,
|
||||||
})
|
})
|
||||||
if (config.retention) {
|
if (config.retention) {
|
||||||
requireInteger(issues, 'retention.rawPayloadDays', config.retention.rawPayloadDays, { min: 1 })
|
|
||||||
requireInteger(issues, 'retention.taskEventDays', config.retention.taskEventDays, { min: 1 })
|
|
||||||
requireInteger(issues, 'retention.webhookEventDays', config.retention.webhookEventDays, {
|
|
||||||
min: 1,
|
|
||||||
})
|
|
||||||
requireInteger(issues, 'retention.auditLogDays', config.retention.auditLogDays, { min: 1 })
|
|
||||||
requireInteger(issues, 'retention.costExceptionDays', config.retention.costExceptionDays, {
|
requireInteger(issues, 'retention.costExceptionDays', config.retention.costExceptionDays, {
|
||||||
min: 1,
|
min: 1,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
-- 151_sales_daily_facts_on_account_channel.sql
|
||||||
|
--
|
||||||
|
-- 财务销售事实渠道枚举补齐统一结算渠道。
|
||||||
|
-- v_finance_order_item_scope 会原样输出 fulfillments.channel,而履约真源自
|
||||||
|
-- 132_on_account_and_manual_channels.sql 起已合法包含 on_account。
|
||||||
|
|
||||||
|
ALTER TABLE sales_daily_facts
|
||||||
|
DROP CONSTRAINT IF EXISTS chk_sales_daily_facts_fulfillment_mode;
|
||||||
|
|
||||||
|
ALTER TABLE sales_daily_facts
|
||||||
|
ADD CONSTRAINT chk_sales_daily_facts_fulfillment_mode
|
||||||
|
CHECK (
|
||||||
|
channel IN (
|
||||||
|
'work_order',
|
||||||
|
'affiliate_dash',
|
||||||
|
'third_party_voucher',
|
||||||
|
'manual',
|
||||||
|
'on_account',
|
||||||
|
'unclassified',
|
||||||
|
'archived'
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN sales_daily_facts.channel IS
|
||||||
|
'销售事实维度(渠道):work_order / affiliate_dash / third_party_voucher / manual / on_account / unclassified 未分类 / archived 归档';
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
-- 152_order_source_facts_immutable.sql
|
||||||
|
--
|
||||||
|
-- 永久保留平台订单事实:
|
||||||
|
-- 1. 每次来源同步追加完整订单快照,供审计和后续重放;
|
||||||
|
-- 2. 快照只允许插入,不允许应用更新或删除;
|
||||||
|
-- 3. orders.raw_payload_json 由应用写入口保证首次写入后不再覆盖。
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS order_source_snapshots (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
order_id BIGINT NOT NULL REFERENCES orders(id) ON DELETE RESTRICT,
|
||||||
|
provider TEXT NOT NULL,
|
||||||
|
platform TEXT NOT NULL,
|
||||||
|
shop_id TEXT NOT NULL DEFAULT '',
|
||||||
|
platform_order_id TEXT NOT NULL,
|
||||||
|
source_label TEXT NOT NULL DEFAULT '',
|
||||||
|
payload_json JSONB NOT NULL,
|
||||||
|
payload_sha256 TEXT NOT NULL,
|
||||||
|
recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
CONSTRAINT chk_order_source_snapshots_payload_sha256
|
||||||
|
CHECK (payload_sha256 ~ '^[0-9a-f]{64}$'),
|
||||||
|
CONSTRAINT uq_order_source_snapshots_payload
|
||||||
|
UNIQUE (order_id, payload_sha256)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_order_source_snapshots_order
|
||||||
|
ON order_source_snapshots(order_id, recorded_at DESC, id DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_order_source_snapshots_platform_order
|
||||||
|
ON order_source_snapshots(provider, platform, platform_order_id, recorded_at DESC);
|
||||||
|
|
||||||
|
COMMENT ON TABLE order_source_snapshots IS
|
||||||
|
'平台订单同步的不可变原始快照;重复同步只追加,不覆盖订单主表原始报文';
|
||||||
|
COMMENT ON COLUMN order_source_snapshots.payload_json IS
|
||||||
|
'平台来源事件、原始报文与订单项完整快照';
|
||||||
|
COMMENT ON COLUMN order_source_snapshots.payload_sha256 IS
|
||||||
|
'完整快照内容 SHA-256,用于同一订单重复消息幂等';
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION reject_order_source_snapshot_mutation()
|
||||||
|
RETURNS TRIGGER
|
||||||
|
LANGUAGE plpgsql
|
||||||
|
AS $$
|
||||||
|
BEGIN
|
||||||
|
IF current_setting('app.allow_order_fact_repair', TRUE) = 'on' THEN
|
||||||
|
RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
RAISE EXCEPTION 'order_source_snapshots is append-only'
|
||||||
|
USING ERRCODE = '23514';
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS trg_order_source_snapshots_append_only
|
||||||
|
ON order_source_snapshots;
|
||||||
|
CREATE TRIGGER trg_order_source_snapshots_append_only
|
||||||
|
BEFORE UPDATE OR DELETE ON order_source_snapshots
|
||||||
|
FOR EACH ROW
|
||||||
|
EXECUTE FUNCTION reject_order_source_snapshot_mutation();
|
||||||
@@ -38,7 +38,7 @@ test('resolveOrderItemSyncPlan preserves matching order item ids across repeated
|
|||||||
},
|
},
|
||||||
])
|
])
|
||||||
assert.deepEqual(plan.creates, [])
|
assert.deepEqual(plan.creates, [])
|
||||||
assert.deepEqual(plan.deletes, [])
|
assert.deepEqual(plan.surplusExistingIds, [])
|
||||||
})
|
})
|
||||||
|
|
||||||
test('resolveOrderItemSyncPlan matches by identity before falling back to position', () => {
|
test('resolveOrderItemSyncPlan matches by identity before falling back to position', () => {
|
||||||
@@ -74,10 +74,10 @@ test('resolveOrderItemSyncPlan matches by identity before falling back to positi
|
|||||||
[22, 21],
|
[22, 21],
|
||||||
)
|
)
|
||||||
assert.deepEqual(plan.creates, [])
|
assert.deepEqual(plan.creates, [])
|
||||||
assert.deepEqual(plan.deletes, [])
|
assert.deepEqual(plan.surplusExistingIds, [])
|
||||||
})
|
})
|
||||||
|
|
||||||
test('resolveOrderItemSyncPlan keeps surplus existing ids for conditional cleanup instead of recreating everything', () => {
|
test('resolveOrderItemSyncPlan reports surplus existing items without deleting them', () => {
|
||||||
const existingItems = [
|
const existingItems = [
|
||||||
{ id: 31, sku_code: 'sku-a', sku_name: '礼包 A', quantity: 1 },
|
{ id: 31, sku_code: 'sku-a', sku_name: '礼包 A', quantity: 1 },
|
||||||
{ id: 32, sku_code: 'sku-b', sku_name: '礼包 B', quantity: 1 },
|
{ id: 32, sku_code: 'sku-b', sku_name: '礼包 B', quantity: 1 },
|
||||||
@@ -101,7 +101,7 @@ test('resolveOrderItemSyncPlan keeps surplus existing ids for conditional cleanu
|
|||||||
[31],
|
[31],
|
||||||
)
|
)
|
||||||
assert.deepEqual(plan.creates, [])
|
assert.deepEqual(plan.creates, [])
|
||||||
assert.deepEqual(plan.deletes, [32])
|
assert.deepEqual(plan.surplusExistingIds, [32])
|
||||||
})
|
})
|
||||||
|
|
||||||
test('已分配订单重推时冻结原渠道决策与中文内部 SKU', () => {
|
test('已分配订单重推时冻结原渠道决策与中文内部 SKU', () => {
|
||||||
@@ -155,7 +155,7 @@ test('已分配订单重推时冻结原渠道决策与中文内部 SKU', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
test('已分配订单重推写库时不会用平台新 SKU 覆盖冻结值', async () => {
|
test('平台订单重推只追加新订单项,不更新或删除已有原始订单项', async () => {
|
||||||
const existing = {
|
const existing = {
|
||||||
id: '41',
|
id: '41',
|
||||||
order_id: 7,
|
order_id: 7,
|
||||||
@@ -184,24 +184,37 @@ test('已分配订单重推写库时不会用平台新 SKU 覆盖冻结值', asy
|
|||||||
}
|
}
|
||||||
const updateParams: unknown[][] = []
|
const updateParams: unknown[][] = []
|
||||||
let selectCount = 0
|
let selectCount = 0
|
||||||
|
let writeSql = ''
|
||||||
|
|
||||||
const executor = async (sql: string, params: unknown[] = []) => {
|
const executor = async (sql: string, params: unknown[] = []) => {
|
||||||
if (sql.includes('SELECT *') && sql.includes('FROM v_order_item_current')) {
|
if (sql.includes('SELECT *') && sql.includes('FROM v_order_item_current')) {
|
||||||
selectCount += 1
|
selectCount += 1
|
||||||
return { rows: selectCount === 1 ? [existing] : [] }
|
return { rows: selectCount === 1 ? [existing] : [{ ...existing, id: 42 }] }
|
||||||
}
|
}
|
||||||
if (sql.includes('UPDATE order_items')) {
|
if (sql.includes('INSERT INTO order_items')) {
|
||||||
updateParams.push(params)
|
updateParams.push(params)
|
||||||
|
writeSql += sql
|
||||||
return { rows: [], rowCount: 1 }
|
return { rows: [], rowCount: 1 }
|
||||||
}
|
}
|
||||||
|
if (/UPDATE\s+order_items|DELETE\s+FROM\s+order_items/i.test(sql)) {
|
||||||
|
throw new Error(`permanent source item must not be mutated: ${sql}`)
|
||||||
|
}
|
||||||
throw new Error(`unexpected SQL: ${sql}`)
|
throw new Error(`unexpected SQL: ${sql}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
await replaceOrderItemsWithExecutor(executor as never, 7, [incoming])
|
await replaceOrderItemsWithExecutor(executor as never, 7, [
|
||||||
|
incoming,
|
||||||
|
{
|
||||||
|
...incoming,
|
||||||
|
skuCode: 'platform-extra',
|
||||||
|
skuName: '平台新增商品',
|
||||||
|
},
|
||||||
|
])
|
||||||
|
|
||||||
assert.equal(updateParams.length, 1)
|
assert.equal(updateParams.length, 1)
|
||||||
assert.equal(updateParams[0]?.[0], 'lucky_coin_x2')
|
assert.ok(writeSql.includes('INSERT INTO order_items'))
|
||||||
assert.equal(updateParams[0]?.[1], '幸运币2个')
|
assert.equal(updateParams[0]?.[1], 'platform-extra')
|
||||||
|
assert.match(String(updateParams[0]?.[2]), /平台新增商品/)
|
||||||
})
|
})
|
||||||
|
|
||||||
test('所有替换订单项入口都在同一事务内分摊订单收入', async () => {
|
test('所有替换订单项入口都在同一事务内分摊订单收入', async () => {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ type OrderItemSyncPlan = {
|
|||||||
item: OrderItemReplaceInput
|
item: OrderItemReplaceInput
|
||||||
}>
|
}>
|
||||||
creates: OrderItemReplaceInput[]
|
creates: OrderItemReplaceInput[]
|
||||||
deletes: number[]
|
surplusExistingIds: number[]
|
||||||
}
|
}
|
||||||
|
|
||||||
type OrderItemIdentityInput = Partial<OrderItemReplaceInput> & Partial<OrderItemRow>
|
type OrderItemIdentityInput = Partial<OrderItemReplaceInput> & Partial<OrderItemRow>
|
||||||
@@ -100,36 +100,6 @@ export async function replaceOrderItemsWithExecutor(
|
|||||||
): Promise<OrderItemRow[]> {
|
): Promise<OrderItemRow[]> {
|
||||||
const existingItems = await listOrderItemsByOrderIdWithExecutor(executor, orderId)
|
const existingItems = await listOrderItemsByOrderIdWithExecutor(executor, orderId)
|
||||||
const plan = resolveOrderItemSyncPlan(existingItems, items)
|
const plan = resolveOrderItemSyncPlan(existingItems, items)
|
||||||
const existingById = new Map(existingItems.map((item) => [Number(item.id), item]))
|
|
||||||
|
|
||||||
for (const update of plan.updates) {
|
|
||||||
const effectiveItem = preserveAssignedOrderItemState(
|
|
||||||
existingById.get(Number(update.orderItemId)),
|
|
||||||
update.item,
|
|
||||||
)
|
|
||||||
await executor(
|
|
||||||
`
|
|
||||||
UPDATE order_items
|
|
||||||
SET
|
|
||||||
sku_code = $1,
|
|
||||||
sku_name = $2,
|
|
||||||
quantity = $3,
|
|
||||||
spec_json = $4::jsonb,
|
|
||||||
item_snapshot_json = $5::jsonb,
|
|
||||||
updated_at = $6
|
|
||||||
WHERE id = $7
|
|
||||||
`,
|
|
||||||
[
|
|
||||||
effectiveItem.skuCode,
|
|
||||||
effectiveItem.skuName,
|
|
||||||
update.item.quantity,
|
|
||||||
safeJsonStringify(effectiveItem.specJson || '{}'),
|
|
||||||
safeJsonStringify(effectiveItem.itemSnapshotJson || effectiveItem.specJson || '{}'),
|
|
||||||
effectiveItem.updatedAt,
|
|
||||||
update.orderItemId,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const create of plan.creates) {
|
for (const create of plan.creates) {
|
||||||
await executor(
|
await executor(
|
||||||
@@ -158,14 +128,6 @@ export async function replaceOrderItemsWithExecutor(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (plan.deletes.length > 0) {
|
|
||||||
const deletableIds = await listDeletableOrderItemIdsWithExecutor(executor, plan.deletes)
|
|
||||||
|
|
||||||
if (deletableIds.length > 0) {
|
|
||||||
await executor('DELETE FROM order_items WHERE id = ANY($1::bigint[])', [deletableIds])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return listOrderItemsByOrderIdWithExecutor(executor, orderId)
|
return listOrderItemsByOrderIdWithExecutor(executor, orderId)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,7 +242,9 @@ export function resolveOrderItemSyncPlan(
|
|||||||
return {
|
return {
|
||||||
updates,
|
updates,
|
||||||
creates,
|
creates,
|
||||||
deletes: unmatchedExisting.map((item) => Number(item.id)).filter((value) => value > 0),
|
surplusExistingIds: unmatchedExisting
|
||||||
|
.map((item) => Number(item.id))
|
||||||
|
.filter((value) => value > 0),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,34 +265,6 @@ function buildOrderItemIdentity(item: OrderItemIdentityInput): string {
|
|||||||
].join('::')
|
].join('::')
|
||||||
}
|
}
|
||||||
|
|
||||||
async function listDeletableOrderItemIdsWithExecutor(
|
|
||||||
executor: QueryExecutor,
|
|
||||||
orderItemIds: number[],
|
|
||||||
): Promise<number[]> {
|
|
||||||
const normalizedIds = Array.isArray(orderItemIds)
|
|
||||||
? orderItemIds.map((value) => Number(value)).filter((value) => value > 0)
|
|
||||||
: []
|
|
||||||
|
|
||||||
if (normalizedIds.length === 0) {
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await executor(
|
|
||||||
`
|
|
||||||
SELECT oi.id
|
|
||||||
FROM v_order_item_current oi
|
|
||||||
LEFT JOIN fulfillment_tasks ft ON ft.order_item_id = oi.id
|
|
||||||
WHERE oi.id = ANY($1::bigint[])
|
|
||||||
GROUP BY oi.id
|
|
||||||
HAVING COUNT(ft.id) = 0
|
|
||||||
ORDER BY oi.id ASC
|
|
||||||
`,
|
|
||||||
[normalizedIds],
|
|
||||||
)
|
|
||||||
|
|
||||||
return result.rows.map((row) => Number(row.id)).filter((value) => value > 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
async function listOrderItemsByOrderIdWithExecutor(
|
async function listOrderItemsByOrderIdWithExecutor(
|
||||||
executor: QueryExecutor,
|
executor: QueryExecutor,
|
||||||
orderId: number | string,
|
orderId: number | string,
|
||||||
|
|||||||
@@ -1,8 +1,18 @@
|
|||||||
import assert from 'node:assert/strict'
|
import assert from 'node:assert/strict'
|
||||||
|
import { readFile } from 'node:fs/promises'
|
||||||
import test from 'node:test'
|
import test from 'node:test'
|
||||||
|
|
||||||
import { listOrdersWithExecutor, type OrderQueryExecutor } from './order-repo.js'
|
import { listOrdersWithExecutor, type OrderQueryExecutor } from './order-repo.js'
|
||||||
|
|
||||||
|
test('订单更新永不覆盖首次写入的平台原始报文', async () => {
|
||||||
|
const source = await readFile(new URL('./order-repo.ts', import.meta.url), 'utf8')
|
||||||
|
const updateSource = source.slice(
|
||||||
|
source.indexOf('export async function updateOrderWithExecutor'),
|
||||||
|
source.indexOf('export async function getOrderById'),
|
||||||
|
)
|
||||||
|
assert.doesNotMatch(updateSource, /raw_payload_json\s*=/)
|
||||||
|
})
|
||||||
|
|
||||||
test('订单列表综合筛选下推到总数和分页查询', async () => {
|
test('订单列表综合筛选下推到总数和分页查询', async () => {
|
||||||
const calls: Array<{ sql: string; params: unknown[] }> = []
|
const calls: Array<{ sql: string; params: unknown[] }> = []
|
||||||
const executor = createExecutor(calls, 3)
|
const executor = createExecutor(calls, 3)
|
||||||
|
|||||||
@@ -191,10 +191,9 @@ export async function updateOrderWithExecutor(
|
|||||||
receiver_contact = $9,
|
receiver_contact = $9,
|
||||||
total_amount = $10,
|
total_amount = $10,
|
||||||
currency = $11,
|
currency = $11,
|
||||||
raw_payload_json = $12::jsonb,
|
paid_at = $12,
|
||||||
paid_at = $13,
|
updated_at = $13
|
||||||
updated_at = $14
|
WHERE id = $14
|
||||||
WHERE id = $15
|
|
||||||
RETURNING *
|
RETURNING *
|
||||||
`,
|
`,
|
||||||
[
|
[
|
||||||
@@ -209,7 +208,6 @@ export async function updateOrderWithExecutor(
|
|||||||
sanitizeWellFormedUnicode(input.receiverContact),
|
sanitizeWellFormedUnicode(input.receiverContact),
|
||||||
input.totalAmount,
|
input.totalAmount,
|
||||||
input.currency,
|
input.currency,
|
||||||
safeJsonStringify(input.rawPayloadJson || '{}'),
|
|
||||||
input.paidAt || null,
|
input.paidAt || null,
|
||||||
input.updatedAt,
|
input.updatedAt,
|
||||||
Number(orderId),
|
Number(orderId),
|
||||||
|
|||||||
@@ -0,0 +1,137 @@
|
|||||||
|
/**
|
||||||
|
* 平台订单原始事实不可变集成测试(需要真实 PostgreSQL)。
|
||||||
|
*
|
||||||
|
* ORDER_SOURCE_FACTS_TEST_DATABASE_URL=postgres://... \
|
||||||
|
* npx tsx --test src/repositories/order-source-facts.integration.test.ts
|
||||||
|
*/
|
||||||
|
import assert from 'node:assert/strict'
|
||||||
|
import test from 'node:test'
|
||||||
|
|
||||||
|
const TEST_DATABASE_URL = String(process.env.ORDER_SOURCE_FACTS_TEST_DATABASE_URL || '').trim()
|
||||||
|
|
||||||
|
test(
|
||||||
|
'平台重推只追加原始事实,不覆盖订单报文、不删除已有订单项',
|
||||||
|
{ skip: !TEST_DATABASE_URL ? '未设置 ORDER_SOURCE_FACTS_TEST_DATABASE_URL' : false },
|
||||||
|
async () => {
|
||||||
|
process.env.DATABASE_URL = TEST_DATABASE_URL
|
||||||
|
const { closeDb, query, withTransaction } = await import('../db/client.js')
|
||||||
|
const { appendOrderSourceSnapshotWithExecutor } =
|
||||||
|
await import('./order-source-snapshot-repo.js')
|
||||||
|
const { replaceOrderItemsWithExecutor } = await import('./order-item-repo.js')
|
||||||
|
const suffix = `${Date.now()}_${Math.floor(Math.random() * 1_000_000)}`
|
||||||
|
let orderId = 0
|
||||||
|
|
||||||
|
try {
|
||||||
|
const order = await query<{ id: number }>(
|
||||||
|
`
|
||||||
|
INSERT INTO orders (
|
||||||
|
provider, platform, shop_id, shop_name, platform_order_id,
|
||||||
|
order_status, pay_status, total_amount, raw_payload_json, created_at, updated_at
|
||||||
|
) VALUES (
|
||||||
|
'91kaquan', 'kuaishou-91', $1, '原始事实测试店', $2,
|
||||||
|
'paid', 'paid', 1000, $3::jsonb, NOW(), NOW()
|
||||||
|
) RETURNING id
|
||||||
|
`,
|
||||||
|
[
|
||||||
|
`source-facts-shop-${suffix}`,
|
||||||
|
`SOURCE-FACTS-${suffix}`,
|
||||||
|
JSON.stringify({ orderNo: `SOURCE-FACTS-${suffix}`, version: 1 }),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
orderId = Number(order.rows[0]?.id || 0)
|
||||||
|
assert.ok(orderId > 0)
|
||||||
|
|
||||||
|
const item = await query<{ id: number }>(
|
||||||
|
`
|
||||||
|
INSERT INTO order_items (
|
||||||
|
order_id, sku_code, sku_name, quantity, spec_json, item_snapshot_json,
|
||||||
|
revenue_amount_fen, revenue_basis, created_at, updated_at
|
||||||
|
) VALUES (
|
||||||
|
$1, 'sku-old', '原始商品', 1, '{}'::jsonb, '{}'::jsonb,
|
||||||
|
1000, 'order_total', NOW(), NOW()
|
||||||
|
) RETURNING id
|
||||||
|
`,
|
||||||
|
[orderId],
|
||||||
|
)
|
||||||
|
const originalItemId = Number(item.rows[0]?.id || 0)
|
||||||
|
assert.ok(originalItemId > 0)
|
||||||
|
|
||||||
|
const snapshotInput = {
|
||||||
|
orderId,
|
||||||
|
provider: '91kaquan',
|
||||||
|
platform: 'kuaishou-91',
|
||||||
|
shopId: `source-facts-shop-${suffix}`,
|
||||||
|
platformOrderId: `SOURCE-FACTS-${suffix}`,
|
||||||
|
sourceLabel: 'integration-test',
|
||||||
|
payload: { orderNo: `SOURCE-FACTS-${suffix}`, version: 1, items: ['sku-old'] },
|
||||||
|
recordedAt: new Date().toISOString(),
|
||||||
|
}
|
||||||
|
assert.equal(await appendOrderSourceSnapshotWithExecutor(query, snapshotInput), true)
|
||||||
|
assert.equal(await appendOrderSourceSnapshotWithExecutor(query, snapshotInput), false)
|
||||||
|
|
||||||
|
const snapshots = await query<{ count: number }>(
|
||||||
|
'SELECT COUNT(*)::int AS count FROM order_source_snapshots WHERE order_id = $1',
|
||||||
|
[orderId],
|
||||||
|
)
|
||||||
|
assert.equal(Number(snapshots.rows[0]?.count), 1)
|
||||||
|
|
||||||
|
await replaceOrderItemsWithExecutor(query, orderId, [
|
||||||
|
{
|
||||||
|
skuCode: 'sku-new',
|
||||||
|
skuName: '平台新商品',
|
||||||
|
quantity: 1,
|
||||||
|
specJson: '{}',
|
||||||
|
itemSnapshotJson: '{}',
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
skuCode: 'sku-extra',
|
||||||
|
skuName: '平台新增商品',
|
||||||
|
quantity: 1,
|
||||||
|
specJson: '{}',
|
||||||
|
itemSnapshotJson: '{}',
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
])
|
||||||
|
|
||||||
|
const items = await query<{ original_exists: boolean; total: number }>(
|
||||||
|
`
|
||||||
|
SELECT
|
||||||
|
EXISTS (SELECT 1 FROM order_items WHERE id = $1) AS original_exists,
|
||||||
|
(SELECT COUNT(*)::int FROM order_items WHERE order_id = $2) AS total
|
||||||
|
`,
|
||||||
|
[originalItemId, orderId],
|
||||||
|
)
|
||||||
|
assert.equal(items.rows[0]?.original_exists, true)
|
||||||
|
assert.equal(Number(items.rows[0]?.total), 2)
|
||||||
|
|
||||||
|
await replaceOrderItemsWithExecutor(query, orderId, [
|
||||||
|
{
|
||||||
|
skuCode: 'sku-new',
|
||||||
|
skuName: '平台新商品',
|
||||||
|
quantity: 1,
|
||||||
|
specJson: '{}',
|
||||||
|
itemSnapshotJson: '{}',
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
])
|
||||||
|
const afterShrink = await query<{ total: number }>(
|
||||||
|
'SELECT COUNT(*)::int AS total FROM order_items WHERE order_id = $1',
|
||||||
|
[orderId],
|
||||||
|
)
|
||||||
|
assert.equal(Number(afterShrink.rows[0]?.total), 2)
|
||||||
|
} finally {
|
||||||
|
if (orderId > 0) {
|
||||||
|
await withTransaction(async (client) => {
|
||||||
|
await client.query(`SET LOCAL app.allow_order_fact_repair = 'on'`)
|
||||||
|
await client.query('DELETE FROM order_source_snapshots WHERE order_id = $1', [orderId])
|
||||||
|
await client.query('DELETE FROM orders WHERE id = $1', [orderId])
|
||||||
|
})
|
||||||
|
}
|
||||||
|
await closeDb()
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
import assert from 'node:assert/strict'
|
||||||
|
import test from 'node:test'
|
||||||
|
|
||||||
|
import {
|
||||||
|
appendOrderSourceSnapshotWithExecutor,
|
||||||
|
orderSourceSnapshotHash,
|
||||||
|
} from './order-source-snapshot-repo.js'
|
||||||
|
|
||||||
|
test('订单来源快照哈希稳定且能区分不同报文', () => {
|
||||||
|
const first = orderSourceSnapshotHash({ orderNo: 'A-1', status: 'paid' })
|
||||||
|
assert.equal(first, orderSourceSnapshotHash({ orderNo: 'A-1', status: 'paid' }))
|
||||||
|
assert.notEqual(first, orderSourceSnapshotHash({ orderNo: 'A-1', status: 'refunded' }))
|
||||||
|
assert.match(first, /^[0-9a-f]{64}$/)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('同订单相同来源快照只追加一次', async () => {
|
||||||
|
const calls: unknown[][] = []
|
||||||
|
const executor = async (sql: string, params: unknown[] = []) => {
|
||||||
|
calls.push(params)
|
||||||
|
assert.ok(sql.includes('INSERT INTO order_source_snapshots'))
|
||||||
|
assert.ok(sql.includes('ON CONFLICT (order_id, payload_sha256) DO NOTHING'))
|
||||||
|
return { rows: [{ id: 1 }] }
|
||||||
|
}
|
||||||
|
|
||||||
|
const inserted = await appendOrderSourceSnapshotWithExecutor(executor as never, {
|
||||||
|
orderId: 7,
|
||||||
|
provider: 'kuaishou-91',
|
||||||
|
platform: 'kuaishou-91',
|
||||||
|
shopId: 'shop-1',
|
||||||
|
platformOrderId: 'ORDER-1',
|
||||||
|
sourceLabel: 'source-order-upsert',
|
||||||
|
payload: { orderNo: 'ORDER-1', status: 'paid' },
|
||||||
|
recordedAt: '2026-11-09T00:00:00.000Z',
|
||||||
|
})
|
||||||
|
|
||||||
|
assert.equal(inserted, true)
|
||||||
|
assert.equal(calls.length, 1)
|
||||||
|
assert.equal(calls[0]?.[0], 7)
|
||||||
|
assert.equal(calls[0]?.[7], orderSourceSnapshotHash({ orderNo: 'ORDER-1', status: 'paid' }))
|
||||||
|
})
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
import { createHash } from 'node:crypto'
|
||||||
|
import type { QueryResult, QueryResultRow } from 'pg'
|
||||||
|
|
||||||
|
import { safeJsonStringify } from '../utils/json.js'
|
||||||
|
|
||||||
|
type QueryExecutor = <T extends QueryResultRow = QueryResultRow>(
|
||||||
|
text: string,
|
||||||
|
params?: unknown[],
|
||||||
|
) => Promise<QueryResult<T>>
|
||||||
|
|
||||||
|
export type OrderSourceSnapshotInput = {
|
||||||
|
orderId: number
|
||||||
|
provider: string
|
||||||
|
platform: string
|
||||||
|
shopId: string
|
||||||
|
platformOrderId: string
|
||||||
|
sourceLabel: string
|
||||||
|
payload: unknown
|
||||||
|
recordedAt: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export function orderSourceSnapshotHash(payload: unknown): string {
|
||||||
|
return createHash('sha256')
|
||||||
|
.update(safeJsonStringify(payload || {}))
|
||||||
|
.digest('hex')
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 追加平台订单原始快照。
|
||||||
|
* 同一订单的相同内容只写一次,不同内容永远追加,不覆盖订单主表原始报文。
|
||||||
|
*/
|
||||||
|
export async function appendOrderSourceSnapshotWithExecutor(
|
||||||
|
executor: QueryExecutor,
|
||||||
|
input: OrderSourceSnapshotInput,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const payloadJson = safeJsonStringify(input.payload || {})
|
||||||
|
const payloadHash = orderSourceSnapshotHash(input.payload)
|
||||||
|
const result = await executor<{ id: number }>(
|
||||||
|
`
|
||||||
|
INSERT INTO order_source_snapshots (
|
||||||
|
order_id,
|
||||||
|
provider,
|
||||||
|
platform,
|
||||||
|
shop_id,
|
||||||
|
platform_order_id,
|
||||||
|
source_label,
|
||||||
|
payload_json,
|
||||||
|
payload_sha256,
|
||||||
|
recorded_at
|
||||||
|
) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9)
|
||||||
|
ON CONFLICT (order_id, payload_sha256) DO NOTHING
|
||||||
|
RETURNING id
|
||||||
|
`,
|
||||||
|
[
|
||||||
|
input.orderId,
|
||||||
|
input.provider,
|
||||||
|
input.platform,
|
||||||
|
input.shopId,
|
||||||
|
input.platformOrderId,
|
||||||
|
input.sourceLabel,
|
||||||
|
payloadJson,
|
||||||
|
payloadHash,
|
||||||
|
input.recordedAt,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
return result.rows.length > 0
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import {
|
|||||||
applyOrderItemRevenueAllocationWithExecutor,
|
applyOrderItemRevenueAllocationWithExecutor,
|
||||||
replaceOrderItemsWithExecutor,
|
replaceOrderItemsWithExecutor,
|
||||||
} from '../../repositories/order-item-repo.js'
|
} from '../../repositories/order-item-repo.js'
|
||||||
|
import { appendOrderSourceSnapshotWithExecutor } from '../../repositories/order-source-snapshot-repo.js'
|
||||||
import { withTransaction } from '../../db/client.js'
|
import { withTransaction } from '../../db/client.js'
|
||||||
import { syncDeliveryTasksForOrder } from './delivery-task-service.js'
|
import { syncDeliveryTasksForOrder } from './delivery-task-service.js'
|
||||||
import { resolveOrderItemForFulfillment } from '../fulfillment/product-resolution-service.js'
|
import { resolveOrderItemForFulfillment } from '../fulfillment/product-resolution-service.js'
|
||||||
@@ -189,6 +190,34 @@ export async function upsertOrderFromSource(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await appendOrderSourceSnapshotWithExecutor(executor, {
|
||||||
|
orderId: Number(order.id),
|
||||||
|
provider: event.provider,
|
||||||
|
platform: event.platform,
|
||||||
|
shopId: String(order.shop_id || event.shopId || '').trim(),
|
||||||
|
platformOrderId: event.platformOrderId,
|
||||||
|
sourceLabel,
|
||||||
|
payload: {
|
||||||
|
sourceLabel,
|
||||||
|
provider: event.provider,
|
||||||
|
platform: event.platform,
|
||||||
|
shopId: event.shopId,
|
||||||
|
shopName: event.shopName,
|
||||||
|
platformOrderId: event.platformOrderId,
|
||||||
|
orderStatus: event.orderStatus,
|
||||||
|
payStatus: event.payStatus,
|
||||||
|
buyerId: event.buyerId,
|
||||||
|
buyerName: event.buyerName,
|
||||||
|
receiverContact: event.receiverContact,
|
||||||
|
totalAmount: event.totalAmount,
|
||||||
|
currency: event.currency,
|
||||||
|
paidAt: event.paidAt || null,
|
||||||
|
rawPayload: event.rawPayload,
|
||||||
|
items: event.items,
|
||||||
|
},
|
||||||
|
recordedAt: now,
|
||||||
|
})
|
||||||
|
|
||||||
const orderItems = await replaceOrderItemsWithExecutor(
|
const orderItems = await replaceOrderItemsWithExecutor(
|
||||||
executor,
|
executor,
|
||||||
order.id,
|
order.id,
|
||||||
|
|||||||
@@ -25,9 +25,8 @@ import type { OrderItemRow, OrderRow } from '../../../types/repository/rows.js'
|
|||||||
export const OPEN_91_PENDING_CONFIG_STATUS = 'pending_config'
|
export const OPEN_91_PENDING_CONFIG_STATUS = 'pending_config'
|
||||||
export const OPEN_91_MANUAL_FAILED_STATUS = 'manual_failed'
|
export const OPEN_91_MANUAL_FAILED_STATUS = 'manual_failed'
|
||||||
|
|
||||||
export async function markOpen91OrderPendingConfig(order: OrderRow, reason = '') {
|
export async function markOpen91OrderPendingConfig(order: OrderRow, _reason = '') {
|
||||||
const now = nowIso()
|
const now = nowIso()
|
||||||
const rawPayload = parseJsonObject(order.raw_payload_json)
|
|
||||||
return updateOrder(order.id, {
|
return updateOrder(order.id, {
|
||||||
provider: order.provider,
|
provider: order.provider,
|
||||||
platform: order.platform,
|
platform: order.platform,
|
||||||
@@ -41,11 +40,6 @@ export async function markOpen91OrderPendingConfig(order: OrderRow, reason = '')
|
|||||||
receiverContact: order.receiver_contact,
|
receiverContact: order.receiver_contact,
|
||||||
totalAmount: Number(order.total_amount) || 0,
|
totalAmount: Number(order.total_amount) || 0,
|
||||||
currency: order.currency,
|
currency: order.currency,
|
||||||
rawPayloadJson: JSON.stringify({
|
|
||||||
...rawPayload,
|
|
||||||
pendingReason: String(reason || '').trim() || 'fulfillment_not_ready',
|
|
||||||
pendingAt: now,
|
|
||||||
}),
|
|
||||||
paidAt: order.paid_at,
|
paidAt: order.paid_at,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
})
|
})
|
||||||
@@ -154,11 +148,6 @@ export async function upsertOpen91PendingOrder(payload: JsonObject = {}, config:
|
|||||||
platformOrderId: event.platformOrderId,
|
platformOrderId: event.platformOrderId,
|
||||||
})
|
})
|
||||||
|
|
||||||
const rawPayloadJson = JSON.stringify({
|
|
||||||
...event.rawPayload,
|
|
||||||
pendingReason: 'unconfigured_items',
|
|
||||||
pendingAt: now,
|
|
||||||
})
|
|
||||||
const orderPayload = {
|
const orderPayload = {
|
||||||
provider: event.provider,
|
provider: event.provider,
|
||||||
platform: event.platform,
|
platform: event.platform,
|
||||||
@@ -172,7 +161,6 @@ export async function upsertOpen91PendingOrder(payload: JsonObject = {}, config:
|
|||||||
receiverContact: '',
|
receiverContact: '',
|
||||||
totalAmount: event.totalAmount,
|
totalAmount: event.totalAmount,
|
||||||
currency: 'CNY',
|
currency: 'CNY',
|
||||||
rawPayloadJson,
|
|
||||||
paidAt: existing?.paid_at || now,
|
paidAt: existing?.paid_at || now,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -245,10 +233,9 @@ export async function retryOpen91Order(orderId: string | number) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function failOpen91Order(orderId: string | number, reason = '') {
|
export async function failOpen91Order(orderId: string | number, _reason = '') {
|
||||||
const order = await getRequiredOpen91Order(orderId)
|
const order = await getRequiredOpen91Order(orderId)
|
||||||
const now = nowIso()
|
const now = nowIso()
|
||||||
const rawPayload = parseJsonObject(order.raw_payload_json)
|
|
||||||
const updated = await updateOrder(order.id, {
|
const updated = await updateOrder(order.id, {
|
||||||
provider: order.provider,
|
provider: order.provider,
|
||||||
platform: order.platform,
|
platform: order.platform,
|
||||||
@@ -262,11 +249,6 @@ export async function failOpen91Order(orderId: string | number, reason = '') {
|
|||||||
receiverContact: order.receiver_contact,
|
receiverContact: order.receiver_contact,
|
||||||
totalAmount: order.total_amount,
|
totalAmount: order.total_amount,
|
||||||
currency: order.currency,
|
currency: order.currency,
|
||||||
rawPayloadJson: JSON.stringify({
|
|
||||||
...rawPayload,
|
|
||||||
manualFailedAt: now,
|
|
||||||
manualFailedReason: String(reason || '').trim() || '商家手动标记无法履约',
|
|
||||||
}),
|
|
||||||
paidAt: order.paid_at,
|
paidAt: order.paid_at,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -6,21 +6,25 @@ async function readJobSource() {
|
|||||||
return readFile(new URL('./data-retention-job.ts', import.meta.url), 'utf8')
|
return readFile(new URL('./data-retention-job.ts', import.meta.url), 'utf8')
|
||||||
}
|
}
|
||||||
|
|
||||||
test('保留清理必须覆盖 work_order_events,否则这张只增不减的表会一直拖慢历史单检索', async () => {
|
test('数据维护不得覆盖原始报文或删除永久业务事实', async () => {
|
||||||
const src = await readJobSource()
|
const src = await readJobSource()
|
||||||
assert.ok(
|
assert.equal(src.includes('raw_payload_json ='), false)
|
||||||
src.includes("deleteOldRowsInBatches('work_order_events'"),
|
assert.equal(src.includes('DELETE FROM'), false)
|
||||||
'work_order_events 未纳入保留清理(生产上 71 万行 / 269MB 且只增不减)',
|
for (const table of [
|
||||||
)
|
'orders',
|
||||||
assert.ok(
|
'kuaishou_industry_vouchers',
|
||||||
src.includes("'task_events' | 'webhook_events' | 'admin_audit_logs' | 'work_order_events'"),
|
'task_events',
|
||||||
'删除助手需要显式允许 work_order_events(表名是白名单联合类型,避免拼接注入)',
|
'webhook_events',
|
||||||
)
|
'admin_audit_logs',
|
||||||
assert.ok(src.includes('retention.workOrderEventDays'), '保留天数必须走配置项,便于按环境调整')
|
'work_order_events',
|
||||||
assert.ok(
|
]) {
|
||||||
src.includes('workOrderEventRows: number') && src.includes('workOrderEventRows,'),
|
assert.equal(
|
||||||
'清理结果要带上工单事件的删除行数,方便定时任务面板核对',
|
new RegExp(`(?:UPDATE|DELETE)\\s+${table}\\b`, 'i').test(src),
|
||||||
)
|
false,
|
||||||
|
`${table} 属于永久业务事实,不能在维护任务中清理或覆盖`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
assert.ok(src.includes('永久业务事实未清理'))
|
||||||
})
|
})
|
||||||
|
|
||||||
test('成本异常只归档已处理的行,open 队列不能被藏起来', async () => {
|
test('成本异常只归档已处理的行,open 队列不能被藏起来', async () => {
|
||||||
@@ -64,29 +68,28 @@ test('保留任务要回填漏写的付款派生列,并在有命中时打日
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
test('保留清理默认窗口:敏感报文/事件 180 天,审计 365 天,工单事件 180 天', async () => {
|
test('数据维护只保留可派生数据的成本异常归档窗口', async () => {
|
||||||
const defaults = await readFile(new URL('../../config/defaults.ts', import.meta.url), 'utf8')
|
const defaults = await readFile(new URL('../../config/defaults.ts', import.meta.url), 'utf8')
|
||||||
|
assert.ok(defaults.includes('costExceptionDays: 60'))
|
||||||
for (const line of [
|
for (const line of [
|
||||||
'rawPayloadDays: 180',
|
'rawPayloadDays:',
|
||||||
'taskEventDays: 180',
|
'taskEventDays:',
|
||||||
'webhookEventDays: 180',
|
'webhookEventDays:',
|
||||||
'auditLogDays: 365',
|
'auditLogDays:',
|
||||||
'workOrderEventDays: 180',
|
'workOrderEventDays:',
|
||||||
'costExceptionDays: 60',
|
|
||||||
]) {
|
]) {
|
||||||
assert.ok(defaults.includes(line), `默认保留策略缺少 ${line}`)
|
assert.equal(defaults.includes(line), false, `永久事实不能存在清理窗口 ${line}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
const overrides = await readFile(
|
const overrides = await readFile(
|
||||||
new URL('../../config/env-overrides.ts', import.meta.url),
|
new URL('../../config/env-overrides.ts', import.meta.url),
|
||||||
'utf8',
|
'utf8',
|
||||||
)
|
)
|
||||||
assert.ok(
|
assert.equal(overrides.includes('RAW_PAYLOAD_RETENTION_DAYS'), false)
|
||||||
overrides.includes(
|
assert.equal(overrides.includes('TASK_EVENT_RETENTION_DAYS'), false)
|
||||||
"integerEnv('WORK_ORDER_EVENT_RETENTION_DAYS', ['retention', 'workOrderEventDays'])",
|
assert.equal(overrides.includes('WEBHOOK_EVENT_RETENTION_DAYS'), false)
|
||||||
),
|
assert.equal(overrides.includes('AUDIT_LOG_RETENTION_DAYS'), false)
|
||||||
'WORK_ORDER_EVENT_RETENTION_DAYS 需要环境变量出口,否则生产无法调整',
|
assert.equal(overrides.includes('WORK_ORDER_EVENT_RETENTION_DAYS'), false)
|
||||||
)
|
|
||||||
assert.ok(
|
assert.ok(
|
||||||
overrides.includes(
|
overrides.includes(
|
||||||
"integerEnv('COST_EXCEPTION_RETENTION_DAYS', ['retention', 'costExceptionDays'])",
|
"integerEnv('COST_EXCEPTION_RETENTION_DAYS', ['retention', 'costExceptionDays'])",
|
||||||
|
|||||||
@@ -8,49 +8,29 @@ import { pruneWorkProductMatchLogs } from '../../repositories/worker-platform/pr
|
|||||||
export type DataRetentionResult = {
|
export type DataRetentionResult = {
|
||||||
status: 'ok'
|
status: 'ok'
|
||||||
message: string
|
message: string
|
||||||
rawPayloadRows: number
|
|
||||||
taskEventRows: number
|
|
||||||
webhookEventRows: number
|
|
||||||
auditLogRows: number
|
|
||||||
workOrderEventRows: number
|
|
||||||
matchLogRows: number
|
|
||||||
matchLogDeletedRows: number
|
matchLogDeletedRows: number
|
||||||
sourceOrderAmountBackfillRows: number
|
sourceOrderAmountBackfillRows: number
|
||||||
costExceptionArchivedRows: number
|
costExceptionArchivedRows: number
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 清理敏感原始报文并删除已过保留期的事件/审计记录。 */
|
/**
|
||||||
|
* 维护可重建的派生数据。
|
||||||
|
*
|
||||||
|
* 原始订单、平台报文、工单/任务事件、审计日志和资金记录都是永久事实,
|
||||||
|
* 不在这里按时间清理或覆盖。
|
||||||
|
*/
|
||||||
export async function runDataRetentionJob(): Promise<DataRetentionResult> {
|
export async function runDataRetentionJob(): Promise<DataRetentionResult> {
|
||||||
const retention = runtimeConfig.retention || {}
|
const retention = runtimeConfig.retention || {}
|
||||||
const rawPayloadDays = positiveDays(retention.rawPayloadDays, 180)
|
|
||||||
const taskEventDays = positiveDays(retention.taskEventDays, 180)
|
|
||||||
const webhookEventDays = positiveDays(retention.webhookEventDays, 180)
|
|
||||||
const auditLogDays = positiveDays(retention.auditLogDays, 365)
|
|
||||||
// 工单事件表以前完全没纳入保留:717k 行 / 269MB 且只增不减,会一直拖慢历史单检索。
|
|
||||||
const workOrderEventDays = positiveDays(retention.workOrderEventDays, 180)
|
|
||||||
// 成本异常只归档「已处理」的行;open 行是待办队列,不能因为超期就藏起来。
|
// 成本异常只归档「已处理」的行;open 行是待办队列,不能因为超期就藏起来。
|
||||||
const costExceptionDays = positiveDays(retention.costExceptionDays, 60)
|
const costExceptionDays = positiveDays(retention.costExceptionDays, 60)
|
||||||
|
|
||||||
// 1. 按配置清理超出上限或过期的打手商品匹配日志
|
// 商品匹配日志是诊断数据,不是订单或履约事实。
|
||||||
const matchConfig = getWorkerProductMatchConfig()
|
const matchConfig = getWorkerProductMatchConfig()
|
||||||
const matchLogPruneResult = await pruneWorkProductMatchLogs({
|
const matchLogPruneResult = await pruneWorkProductMatchLogs({
|
||||||
maxRows: matchConfig.maxMatchLogs,
|
maxRows: matchConfig.maxMatchLogs,
|
||||||
retentionDays: matchConfig.matchLogRetentionDays,
|
retentionDays: matchConfig.matchLogRetentionDays,
|
||||||
})
|
})
|
||||||
|
|
||||||
// 2. 清理其他报文与日志
|
|
||||||
const rawPayloadRows =
|
|
||||||
(await scrubJsonColumnInBatches('orders', 'updated_at', rawPayloadDays)) +
|
|
||||||
(await scrubJsonColumnInBatches('kuaishou_industry_vouchers', 'updated_at', rawPayloadDays))
|
|
||||||
const matchLogRows = await scrubJsonColumnInBatches(
|
|
||||||
'work_product_match_logs',
|
|
||||||
'created_at',
|
|
||||||
rawPayloadDays,
|
|
||||||
)
|
|
||||||
const taskEventRows = await deleteOldRowsInBatches('task_events', taskEventDays)
|
|
||||||
const webhookEventRows = await deleteOldRowsInBatches('webhook_events', webhookEventDays)
|
|
||||||
const auditLogRows = await deleteOldRowsInBatches('admin_audit_logs', auditLogDays)
|
|
||||||
const workOrderEventRows = await deleteOldRowsInBatches('work_order_events', workOrderEventDays)
|
|
||||||
// 统计热路径已不再读 material_json。若将来有第三个写入点忘了维护派生列,
|
// 统计热路径已不再读 material_json。若将来有第三个写入点忘了维护派生列,
|
||||||
// 金额会静默回退 orders.total_amount。这里把漏写补上并打日志,让口径变化可见。
|
// 金额会静默回退 orders.total_amount。这里把漏写补上并打日志,让口径变化可见。
|
||||||
const costExceptionArchive = await archiveResolvedCostExceptions({
|
const costExceptionArchive = await archiveResolvedCostExceptions({
|
||||||
@@ -66,14 +46,8 @@ export async function runDataRetentionJob(): Promise<DataRetentionResult> {
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
status: 'ok',
|
status: 'ok',
|
||||||
message: '数据保留清理完成',
|
message: '派生数据维护完成;永久业务事实未清理',
|
||||||
rawPayloadRows,
|
|
||||||
matchLogRows,
|
|
||||||
matchLogDeletedRows: matchLogPruneResult.totalDeleted,
|
matchLogDeletedRows: matchLogPruneResult.totalDeleted,
|
||||||
taskEventRows,
|
|
||||||
webhookEventRows,
|
|
||||||
auditLogRows,
|
|
||||||
workOrderEventRows,
|
|
||||||
sourceOrderAmountBackfillRows,
|
sourceOrderAmountBackfillRows,
|
||||||
costExceptionArchivedRows,
|
costExceptionArchivedRows,
|
||||||
}
|
}
|
||||||
@@ -92,59 +66,6 @@ async function backfillMissingSourceOrderAmount() {
|
|||||||
return result.rowCount || 0
|
return result.rowCount || 0
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrubJsonColumnInBatches(
|
|
||||||
table: 'orders' | 'kuaishou_industry_vouchers' | 'work_product_match_logs',
|
|
||||||
dateColumn: 'created_at' | 'updated_at',
|
|
||||||
retentionDays: number,
|
|
||||||
) {
|
|
||||||
let total = 0
|
|
||||||
while (true) {
|
|
||||||
const result = await query(
|
|
||||||
`
|
|
||||||
WITH candidates AS (
|
|
||||||
SELECT ctid
|
|
||||||
FROM ${table}
|
|
||||||
WHERE ${dateColumn} < NOW() - make_interval(days => $1::int)
|
|
||||||
AND raw_payload_json <> '{}'::jsonb
|
|
||||||
LIMIT 1000
|
|
||||||
)
|
|
||||||
UPDATE ${table} AS target
|
|
||||||
SET raw_payload_json = '{}'::jsonb
|
|
||||||
FROM candidates
|
|
||||||
WHERE target.ctid = candidates.ctid
|
|
||||||
`,
|
|
||||||
[retentionDays],
|
|
||||||
)
|
|
||||||
total += result.rowCount || 0
|
|
||||||
if ((result.rowCount || 0) < 1000) return total
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function deleteOldRowsInBatches(
|
|
||||||
table: 'task_events' | 'webhook_events' | 'admin_audit_logs' | 'work_order_events',
|
|
||||||
retentionDays: number,
|
|
||||||
) {
|
|
||||||
let total = 0
|
|
||||||
while (true) {
|
|
||||||
const result = await query(
|
|
||||||
`
|
|
||||||
WITH candidates AS (
|
|
||||||
SELECT ctid
|
|
||||||
FROM ${table}
|
|
||||||
WHERE created_at < NOW() - make_interval(days => $1::int)
|
|
||||||
LIMIT 1000
|
|
||||||
)
|
|
||||||
DELETE FROM ${table} AS target
|
|
||||||
USING candidates
|
|
||||||
WHERE target.ctid = candidates.ctid
|
|
||||||
`,
|
|
||||||
[retentionDays],
|
|
||||||
)
|
|
||||||
total += result.rowCount || 0
|
|
||||||
if ((result.rowCount || 0) < 1000) return total
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function positiveDays(value: unknown, fallback: number) {
|
function positiveDays(value: unknown, fallback: number) {
|
||||||
const days = Number(value)
|
const days = Number(value)
|
||||||
return Number.isInteger(days) && days > 0 ? days : fallback
|
return Number.isInteger(days) && days > 0 ? days : fallback
|
||||||
|
|||||||
@@ -169,12 +169,9 @@ async function runJob(job: JsonObject, { manual = false }: { manual?: boolean }
|
|||||||
status,
|
status,
|
||||||
...(job.type === 'data_retention'
|
...(job.type === 'data_retention'
|
||||||
? {
|
? {
|
||||||
rawPayloadRows: Number(summary.rawPayloadRows || 0),
|
matchLogDeletedRows: Number(summary.matchLogDeletedRows || 0),
|
||||||
matchLogRows: Number(summary.matchLogRows || 0),
|
|
||||||
taskEventRows: Number(summary.taskEventRows || 0),
|
|
||||||
webhookEventRows: Number(summary.webhookEventRows || 0),
|
|
||||||
auditLogRows: Number(summary.auditLogRows || 0),
|
|
||||||
sourceOrderAmountBackfillRows: Number(summary.sourceOrderAmountBackfillRows || 0),
|
sourceOrderAmountBackfillRows: Number(summary.sourceOrderAmountBackfillRows || 0),
|
||||||
|
costExceptionArchivedRows: Number(summary.costExceptionArchivedRows || 0),
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -28,12 +28,6 @@ export type RuntimeConfig = {
|
|||||||
maxFiles?: number
|
maxFiles?: number
|
||||||
}
|
}
|
||||||
retention: {
|
retention: {
|
||||||
rawPayloadDays: number
|
|
||||||
taskEventDays: number
|
|
||||||
webhookEventDays: number
|
|
||||||
auditLogDays: number
|
|
||||||
/** 工单事件(work_order_events)保留天数:这张表只增不减,必须纳入清理。 */
|
|
||||||
workOrderEventDays: number
|
|
||||||
/** 已处理成本异常(cost_exceptions)保留天数:超期归档,open 行不归档。 */
|
/** 已处理成本异常(cost_exceptions)保留天数:超期归档,open 行不归档。 */
|
||||||
costExceptionDays: number
|
costExceptionDays: number
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-6
@@ -3,16 +3,16 @@ type: 权威架构与实施计划
|
|||||||
description: 订单、内部 SKU、履约路由、履约执行、商品、成本与财务的唯一权威口径及重构计划。
|
description: 订单、内部 SKU、履约路由、履约执行、商品、成本与财务的唯一权威口径及重构计划。
|
||||||
tags: [订单, 内部SKU, 履约, 商品, 成本, 财务, 迁移]
|
tags: [订单, 内部SKU, 履约, 商品, 成本, 财务, 迁移]
|
||||||
status: authoritative
|
status: authoritative
|
||||||
version: 2.4
|
version: 2.5
|
||||||
updated_at: 2026-11-08
|
updated_at: 2026-11-09
|
||||||
---
|
---
|
||||||
|
|
||||||
# 统一订单、履约与财务模型
|
# 统一订单、履约与财务模型
|
||||||
|
|
||||||
> 状态:**唯一权威文档**。
|
> 状态:**唯一权威文档**。
|
||||||
> 最后校准:2026-11-08。
|
> 最后校准:2026-11-09。
|
||||||
> 生产基线:迁移 `146`。
|
> 生产基线:迁移 `151`。
|
||||||
> 开发基线:迁移 `150`(仅本地,尚未部署生产)。
|
> 开发基线:迁移 `152`(仅本地,尚未部署生产)。
|
||||||
>
|
>
|
||||||
> 从本文件生效起,订单、内部 SKU、履约渠道、履约载体、标准商品、成本和财务相关实现,
|
> 从本文件生效起,订单、内部 SKU、履约渠道、履约载体、标准商品、成本和财务相关实现,
|
||||||
> 只能以本文件为业务口径。其他旧分析只作为历史资料,不再具备设计或验收效力。
|
> 只能以本文件为业务口径。其他旧分析只作为历史资料,不再具备设计或验收效力。
|
||||||
@@ -313,7 +313,7 @@ finance_journal_* -- 总账凭证,可重建
|
|||||||
| 信息 | 唯一写入方 | 唯一真源 | 明确禁止 |
|
| 信息 | 唯一写入方 | 唯一真源 | 明确禁止 |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| 平台订单与订单项 | 订单接入服务 | `orders` / `order_items` | 财务或履约服务改写平台事实 |
|
| 平台订单与订单项 | 订单接入服务 | `orders` / `order_items` | 财务或履约服务改写平台事实 |
|
||||||
| 平台原始报文 | 各平台适配器 | 原始表 / `raw_payload_json` | 覆盖历史报文 |
|
| 平台原始报文 | 各平台适配器 | `order_source_snapshots` / 首次 `raw_payload_json` | 覆盖历史报文 |
|
||||||
| 中文内部 SKU | 商品身份服务 | `internal_skus` | 各业务表各自发明同义 SKU |
|
| 中文内部 SKU | 商品身份服务 | `internal_skus` | 各业务表各自发明同义 SKU |
|
||||||
| 平台字段到内部 SKU | 映射服务 | `internal_sku_mappings` | 按名称猜测渠道 |
|
| 平台字段到内部 SKU | 映射服务 | `internal_sku_mappings` | 按名称猜测渠道 |
|
||||||
| 主渠道与兜底 | 路由服务 | `sku_channel_routes` | UI 根据商品前缀筛选 |
|
| 主渠道与兜底 | 路由服务 | `sku_channel_routes` | UI 根据商品前缀筛选 |
|
||||||
@@ -366,6 +366,7 @@ finance_journal_* -- 总账凭证,可重建
|
|||||||
- `orders`
|
- `orders`
|
||||||
- `order_items` 的原始平台字段
|
- `order_items` 的原始平台字段
|
||||||
- 平台原始订单表与原始报文字段
|
- 平台原始订单表与原始报文字段
|
||||||
|
- `order_source_snapshots` 平台订单不可变事件快照
|
||||||
- 平台售后、退款、支付事实
|
- 平台售后、退款、支付事实
|
||||||
- `work_orders`、`work_order_shares`、`work_order_events`
|
- `work_orders`、`work_order_shares`、`work_order_events`
|
||||||
- `fulfillment_tasks` 中真实执行与结果记录
|
- `fulfillment_tasks` 中真实执行与结果记录
|
||||||
@@ -377,6 +378,14 @@ finance_journal_* -- 总账凭证,可重建
|
|||||||
|
|
||||||
A 级数据只能追加新事件或做字段兼容迁移,不能为了“模型整齐”重写历史事实。
|
A 级数据只能追加新事件或做字段兼容迁移,不能为了“模型整齐”重写历史事实。
|
||||||
|
|
||||||
|
订单重复同步必须遵守:
|
||||||
|
|
||||||
|
- 首次写入的 `orders.raw_payload_json` 不再覆盖;
|
||||||
|
- 每次来源同步追加 `order_source_snapshots`,相同内容幂等;
|
||||||
|
- 已有 `order_items` 的平台 SKU、名称、数量、规格和快照不因重推更新;
|
||||||
|
- 重推中缺少的订单项只记录为来源差异,不得物理删除订单项或级联删除履约记录;
|
||||||
|
- 数据保留任务不得清空或删除 A 级事实,只允许维护可重建的派生数据。
|
||||||
|
|
||||||
### B 级:业务输入,必须迁移而不是重建
|
### B 级:业务输入,必须迁移而不是重建
|
||||||
|
|
||||||
- `internal_sku_mappings`
|
- `internal_sku_mappings`
|
||||||
|
|||||||
Reference in New Issue
Block a user