feat(pdd): 引入拼多多原始订单暂存仓 pdd_raw_orders 与两段式解耦流水线
1. 数据库新增 081_pdd_raw_orders.sql 迁移，创建 pdd_raw_orders 原始暂存表及索引； 2. 新增 pdd-raw-order-repo 仓储，支持原汁原味批量暂存、pending 单据消费、状态回填与重置； 3. 重构 pdd-sync-service 为采集暂存（Staging）与业务进单（Ingestion）两段式解耦流水线，实现断点续传； 4. 管理后台路由接入 raw-orders 原始流水查询及 replay 一键历史单重放 API。
This commit is contained in:
yml2213
@@ -0,0 +1,67 @@
|
||||
import test from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import { batchUpsertPddRawOrders, type QueryExecutor } from './pdd-raw-order-repo.js'
|
||||
import type { PddRawOrderItem } from '../../services/platforms/pdd/pdd-order-mapper.js'
|
||||
|
||||
test('batchUpsertPddRawOrders formats and passes parameters safely to query executor', async () => {
|
||||
const capturedQueries: Array<{ text: string; params: unknown[] }> = []
|
||||
|
||||
const mockExecutor: QueryExecutor = async (text: string, params?: unknown[]) => {
|
||||
capturedQueries.push({ text, params: params || [] })
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
id: 1,
|
||||
order_sn: '260919-999999',
|
||||
shop_id: 'test_shop',
|
||||
shop_name: '测试店铺',
|
||||
sync_status: 'pending',
|
||||
is_inserted: true,
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
command: 'INSERT',
|
||||
oid: 0,
|
||||
fields: [],
|
||||
}
|
||||
}
|
||||
|
||||
const rawOrder: PddRawOrderItem = {
|
||||
order_sn: '260919-999999',
|
||||
order_status: 1,
|
||||
order_status_str: '待发货',
|
||||
order_amount: 5000,
|
||||
goods_id: 12345,
|
||||
goods_name: '测试商品',
|
||||
out_sku_sn: '安卓血宴3钻',
|
||||
spec: '暗潮1钻',
|
||||
goods_number: 1,
|
||||
nickname: '买家\ud835*test', // Lone surrogate
|
||||
buyer_memo: '游戏ID: 8888\0null', // Null byte
|
||||
order_time: 1726718000,
|
||||
confirm_time: 1726718005,
|
||||
}
|
||||
|
||||
const result = await batchUpsertPddRawOrders(
|
||||
[rawOrder],
|
||||
{ shopId: 'test_shop', shopName: '测试店铺' },
|
||||
mockExecutor,
|
||||
)
|
||||
|
||||
assert.equal(result.totalCount, 1)
|
||||
assert.equal(result.insertedCount, 1)
|
||||
assert.equal(capturedQueries.length, 1)
|
||||
|
||||
const [q] = capturedQueries
|
||||
assert.ok(q.text.includes('INSERT INTO pdd_raw_orders'))
|
||||
assert.ok(q.text.includes('ON CONFLICT (order_sn) DO UPDATE'))
|
||||
|
||||
// Verify parameters are properly sanitized
|
||||
const params = q.params
|
||||
assert.equal(params[0], '260919-999999') // orderSn
|
||||
assert.equal(params[1], 'test_shop') // shopId
|
||||
assert.equal(params[2], '测试店铺') // shopName
|
||||
assert.equal(params[11], '买家\uFFFD*test') // buyerNickname (cleaned surrogate)
|
||||
assert.equal(params[12], '游戏ID: 8888null') // buyerMemo (removed null byte)
|
||||
})
|
||||
@@ -0,0 +1,441 @@
|
||||
/**
|
||||
* Pinduoduo Raw Orders Staging Repository
|
||||
* 拼多多原始订单暂存仓数据库仓储
|
||||
*/
|
||||
|
||||
import { query, withTransaction } from '../../db/client.js'
|
||||
import type { QueryResult } from 'pg'
|
||||
import { safeJsonStringify, sanitizeWellFormedUnicode } from '../../utils/json.js'
|
||||
import type { PddRawOrderItem } from '../../services/platforms/pdd/pdd-order-mapper.js'
|
||||
import {
|
||||
extractPddGoodsLine,
|
||||
extractPddOutSkuSn,
|
||||
} from '../../services/platforms/pdd/pdd-order-mapper.js'
|
||||
|
||||
export type QueryExecutor = <T extends Record<string, any> = any>(
|
||||
text: string,
|
||||
params?: unknown[],
|
||||
) => Promise<QueryResult<T>>
|
||||
|
||||
export interface PddRawOrderRow {
|
||||
id: number
|
||||
order_sn: string
|
||||
shop_id: string
|
||||
shop_name: string
|
||||
order_status: number
|
||||
order_status_str: string
|
||||
goods_id: string
|
||||
goods_name: string
|
||||
out_sku_sn: string
|
||||
spec: string
|
||||
quantity: number
|
||||
order_amount_fen: number
|
||||
buyer_nickname: string
|
||||
buyer_memo: string
|
||||
mall_remark: string
|
||||
order_time: string | null
|
||||
confirm_time: string | null
|
||||
raw_payload_json: Record<string, unknown>
|
||||
sync_status: 'pending' | 'synced' | 'ignored' | 'failed'
|
||||
sync_error: string
|
||||
synced_order_id: number | null
|
||||
synced_at: string | null
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
|
||||
export interface PddRawOrderUpsertResult {
|
||||
insertedCount: number
|
||||
updatedCount: number
|
||||
totalCount: number
|
||||
rows: PddRawOrderRow[]
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量暂存拼多多原始订单数据（ON CONFLICT DO UPDATE）
|
||||
*/
|
||||
export async function batchUpsertPddRawOrders(
|
||||
rawOrders: PddRawOrderItem[],
|
||||
options: { shopId: string; shopName: string },
|
||||
executor: QueryExecutor = query,
|
||||
): Promise<PddRawOrderUpsertResult> {
|
||||
if (!rawOrders || rawOrders.length === 0) {
|
||||
return { insertedCount: 0, updatedCount: 0, totalCount: 0, rows: [] }
|
||||
}
|
||||
|
||||
const shopId = options.shopId.trim() || 'pdd_default'
|
||||
const shopName = options.shopName.trim() || '拼多多店铺'
|
||||
const rows: PddRawOrderRow[] = []
|
||||
let insertedCount = 0
|
||||
let updatedCount = 0
|
||||
|
||||
for (const rawItem of rawOrders) {
|
||||
const orderSn = String(rawItem.order_sn || '').trim()
|
||||
if (!orderSn) continue
|
||||
|
||||
const goodsLine = extractPddGoodsLine(rawItem)
|
||||
const outSkuSn = extractPddOutSkuSn(rawItem)
|
||||
const orderStatus = Number(rawItem.order_status || 1)
|
||||
const orderStatusStr = sanitizeWellFormedUnicode(rawItem.order_status_str || '')
|
||||
const goodsId = sanitizeWellFormedUnicode(goodsLine.goodsId)
|
||||
const goodsName = sanitizeWellFormedUnicode(goodsLine.goodsName)
|
||||
const spec = sanitizeWellFormedUnicode(goodsLine.spec)
|
||||
const quantity = Math.max(1, Number(goodsLine.quantity || 1))
|
||||
const orderAmountFen = Math.max(0, Number(rawItem.order_amount || 0))
|
||||
const buyerNickname = sanitizeWellFormedUnicode(rawItem.nickname || '')
|
||||
const buyerMemo = sanitizeWellFormedUnicode(rawItem.buyer_memo || '').trim()
|
||||
const mallRemark = sanitizeWellFormedUnicode(rawItem.mall_remark || '').trim()
|
||||
|
||||
const orderTime = rawItem.order_time
|
||||
? new Date(Number(rawItem.order_time) * 1000).toISOString()
|
||||
: null
|
||||
const confirmTime = rawItem.confirm_time
|
||||
? new Date(Number(rawItem.confirm_time) * 1000).toISOString()
|
||||
: null
|
||||
|
||||
const rawPayloadJson = safeJsonStringify(rawItem)
|
||||
|
||||
const result = await executor<PddRawOrderRow>(
|
||||
`
|
||||
INSERT INTO pdd_raw_orders (
|
||||
order_sn,
|
||||
shop_id,
|
||||
shop_name,
|
||||
order_status,
|
||||
order_status_str,
|
||||
goods_id,
|
||||
goods_name,
|
||||
out_sku_sn,
|
||||
spec,
|
||||
quantity,
|
||||
order_amount_fen,
|
||||
buyer_nickname,
|
||||
buyer_memo,
|
||||
mall_remark,
|
||||
order_time,
|
||||
confirm_time,
|
||||
raw_payload_json,
|
||||
sync_status,
|
||||
created_at,
|
||||
updated_at
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
||||
$11, $12, $13, $14, $15, $16, $17::jsonb, 'pending', NOW(), NOW()
|
||||
)
|
||||
ON CONFLICT (order_sn) DO UPDATE SET
|
||||
shop_id = EXCLUDED.shop_id,
|
||||
shop_name = EXCLUDED.shop_name,
|
||||
order_status = EXCLUDED.order_status,
|
||||
order_status_str = EXCLUDED.order_status_str,
|
||||
goods_id = EXCLUDED.goods_id,
|
||||
goods_name = EXCLUDED.goods_name,
|
||||
out_sku_sn = EXCLUDED.out_sku_sn,
|
||||
spec = EXCLUDED.spec,
|
||||
quantity = EXCLUDED.quantity,
|
||||
order_amount_fen = EXCLUDED.order_amount_fen,
|
||||
buyer_nickname = EXCLUDED.buyer_nickname,
|
||||
buyer_memo = EXCLUDED.buyer_memo,
|
||||
mall_remark = EXCLUDED.mall_remark,
|
||||
order_time = COALESCE(EXCLUDED.order_time, pdd_raw_orders.order_time),
|
||||
confirm_time = COALESCE(EXCLUDED.confirm_time, pdd_raw_orders.confirm_time),
|
||||
raw_payload_json = EXCLUDED.raw_payload_json,
|
||||
updated_at = NOW()
|
||||
RETURNING *, (xmax = 0) AS is_inserted
|
||||
`,
|
||||
[
|
||||
orderSn,
|
||||
shopId,
|
||||
shopName,
|
||||
orderStatus,
|
||||
orderStatusStr,
|
||||
goodsId,
|
||||
goodsName,
|
||||
outSkuSn,
|
||||
spec,
|
||||
quantity,
|
||||
orderAmountFen,
|
||||
buyerNickname,
|
||||
buyerMemo,
|
||||
mallRemark,
|
||||
orderTime,
|
||||
confirmTime,
|
||||
rawPayloadJson,
|
||||
],
|
||||
)
|
||||
|
||||
const row = result.rows[0]
|
||||
if (row) {
|
||||
rows.push(row)
|
||||
if ((row as any).is_inserted) {
|
||||
insertedCount++
|
||||
} else {
|
||||
updatedCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
insertedCount,
|
||||
updatedCount,
|
||||
totalCount: rows.length,
|
||||
rows,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取指定店铺最近一次订单成团/支付时间，为增量同步提供断点
|
||||
*/
|
||||
export async function getLatestPddOrderTimeByShopId(
|
||||
shopId: string,
|
||||
executor: QueryExecutor = query,
|
||||
): Promise<{ latestOrderTime: string | null; latestConfirmTime: string | null }> {
|
||||
const result = await executor<{
|
||||
latest_order_time: string | null
|
||||
latest_confirm_time: string | null
|
||||
}>(
|
||||
`
|
||||
SELECT
|
||||
order_time AS latest_order_time,
|
||||
confirm_time AS latest_confirm_time
|
||||
FROM pdd_raw_orders
|
||||
WHERE shop_id = $1
|
||||
ORDER BY COALESCE(confirm_time, order_time, created_at) DESC NULLS LAST
|
||||
LIMIT 1
|
||||
`,
|
||||
[shopId],
|
||||
)
|
||||
|
||||
const row = result.rows[0]
|
||||
return {
|
||||
latestOrderTime: row?.latest_order_time || null,
|
||||
latestConfirmTime: row?.latest_confirm_time || null,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 扫描待转化的原始拼多多订单
|
||||
*/
|
||||
export async function findPendingPddRawOrders(
|
||||
options: { shopId?: string; limit?: number } = {},
|
||||
executor: QueryExecutor = query,
|
||||
): Promise<PddRawOrderRow[]> {
|
||||
const limit = Math.min(100, Math.max(1, options.limit || 50))
|
||||
const params: unknown[] = [limit]
|
||||
let whereClause = "WHERE sync_status = 'pending'"
|
||||
|
||||
if (options.shopId) {
|
||||
params.push(options.shopId)
|
||||
whereClause += ` AND shop_id = $${params.length}`
|
||||
}
|
||||
|
||||
const result = await executor<PddRawOrderRow>(
|
||||
`
|
||||
SELECT *
|
||||
FROM pdd_raw_orders
|
||||
${whereClause}
|
||||
ORDER BY COALESCE(confirm_time, order_time, created_at) ASC, id ASC
|
||||
LIMIT $1
|
||||
`,
|
||||
params,
|
||||
)
|
||||
|
||||
return result.rows
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记拼多多原始订单已成功进单落库
|
||||
*/
|
||||
export async function markPddRawOrderSynced(
|
||||
orderSn: string,
|
||||
syncedOrderId: number,
|
||||
executor: QueryExecutor = query,
|
||||
): Promise<boolean> {
|
||||
const result = await executor(
|
||||
`
|
||||
UPDATE pdd_raw_orders
|
||||
SET
|
||||
sync_status = 'synced',
|
||||
synced_order_id = $2,
|
||||
synced_at = NOW(),
|
||||
sync_error = '',
|
||||
updated_at = NOW()
|
||||
WHERE order_sn = $1
|
||||
`,
|
||||
[orderSn, syncedOrderId],
|
||||
)
|
||||
return (result.rowCount ?? 0) > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记拼多多原始订单因规则未命中等原因被忽略
|
||||
*/
|
||||
export async function markPddRawOrderIgnored(
|
||||
orderSn: string,
|
||||
reason: string,
|
||||
executor: QueryExecutor = query,
|
||||
): Promise<boolean> {
|
||||
const result = await executor(
|
||||
`
|
||||
UPDATE pdd_raw_orders
|
||||
SET
|
||||
sync_status = 'ignored',
|
||||
sync_error = $2,
|
||||
updated_at = NOW()
|
||||
WHERE order_sn = $1
|
||||
`,
|
||||
[orderSn, sanitizeWellFormedUnicode(reason)],
|
||||
)
|
||||
return (result.rowCount ?? 0) > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记拼多多原始订单进单异常失败
|
||||
*/
|
||||
export async function markPddRawOrderFailed(
|
||||
orderSn: string,
|
||||
error: string,
|
||||
executor: QueryExecutor = query,
|
||||
): Promise<boolean> {
|
||||
const result = await executor(
|
||||
`
|
||||
UPDATE pdd_raw_orders
|
||||
SET
|
||||
sync_status = 'failed',
|
||||
sync_error = $2,
|
||||
updated_at = NOW()
|
||||
WHERE order_sn = $1
|
||||
`,
|
||||
[orderSn, sanitizeWellFormedUnicode(error)],
|
||||
)
|
||||
return (result.rowCount ?? 0) > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* 重置指定订单为 pending 状态，用于一键重新扫描派单
|
||||
*/
|
||||
export async function resetPddRawOrdersToPending(
|
||||
options: { shopId?: string; orderSns?: string[] } = {},
|
||||
executor: QueryExecutor = query,
|
||||
): Promise<number> {
|
||||
const params: unknown[] = []
|
||||
const filters: string[] = ["sync_status IN ('ignored', 'failed')"]
|
||||
|
||||
if (options.shopId) {
|
||||
params.push(options.shopId)
|
||||
filters.push(`shop_id = $${params.length}`)
|
||||
}
|
||||
|
||||
if (options.orderSns && options.orderSns.length > 0) {
|
||||
params.push(options.orderSns)
|
||||
filters.push(`order_sn = ANY($${params.length}::text[])`)
|
||||
}
|
||||
|
||||
const whereClause = `WHERE ${filters.join(' AND ')}`
|
||||
const result = await executor(
|
||||
`
|
||||
UPDATE pdd_raw_orders
|
||||
SET
|
||||
sync_status = 'pending',
|
||||
sync_error = '',
|
||||
updated_at = NOW()
|
||||
${whereClause}
|
||||
`,
|
||||
params,
|
||||
)
|
||||
|
||||
return result.rowCount ?? 0
|
||||
}
|
||||
|
||||
export interface ListPddRawOrdersQuery {
|
||||
page?: number | undefined
|
||||
pageSize?: number | undefined
|
||||
shopId?: string | undefined
|
||||
orderSn?: string | undefined
|
||||
syncStatus?: string | undefined
|
||||
outSkuSn?: string | undefined
|
||||
dateFrom?: string | undefined
|
||||
dateTo?: string | undefined
|
||||
}
|
||||
|
||||
export interface ListPddRawOrdersResult {
|
||||
items: PddRawOrderRow[]
|
||||
total: number
|
||||
page: number
|
||||
pageSize: number
|
||||
}
|
||||
|
||||
/**
|
||||
* 管理后台查询拼多多原始订单流水列表
|
||||
*/
|
||||
export async function listPddRawOrders(
|
||||
input: ListPddRawOrdersQuery = {},
|
||||
executor: QueryExecutor = query,
|
||||
): Promise<ListPddRawOrdersResult> {
|
||||
const page = Math.max(1, Number(input.page || 1))
|
||||
const pageSize = Math.min(100, Math.max(1, Number(input.pageSize || 20)))
|
||||
const offset = (page - 1) * pageSize
|
||||
|
||||
const filters: string[] = []
|
||||
const params: unknown[] = []
|
||||
|
||||
if (input.shopId) {
|
||||
params.push(input.shopId.trim())
|
||||
filters.push(`shop_id = $${params.length}`)
|
||||
}
|
||||
|
||||
if (input.orderSn) {
|
||||
params.push(`%${input.orderSn.trim()}%`)
|
||||
filters.push(`order_sn ILIKE $${params.length}`)
|
||||
}
|
||||
|
||||
if (input.syncStatus) {
|
||||
params.push(input.syncStatus.trim())
|
||||
filters.push(`sync_status = $${params.length}`)
|
||||
}
|
||||
|
||||
if (input.outSkuSn) {
|
||||
params.push(`%${input.outSkuSn.trim()}%`)
|
||||
filters.push(`out_sku_sn ILIKE $${params.length}`)
|
||||
}
|
||||
|
||||
if (input.dateFrom) {
|
||||
params.push(input.dateFrom.trim().slice(0, 10))
|
||||
filters.push(`COALESCE(confirm_time, order_time, created_at) >= ($${params.length}::date)`)
|
||||
}
|
||||
|
||||
if (input.dateTo) {
|
||||
params.push(input.dateTo.trim().slice(0, 10))
|
||||
filters.push(
|
||||
`COALESCE(confirm_time, order_time, created_at) < (($${params.length}::date + 1)::timestamp)`,
|
||||
)
|
||||
}
|
||||
|
||||
const whereClause = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : ''
|
||||
|
||||
const countResult = await executor<{ total: number }>(
|
||||
`SELECT COUNT(*)::int AS total FROM pdd_raw_orders ${whereClause}`,
|
||||
params,
|
||||
)
|
||||
const total = Number(countResult.rows[0]?.total || 0)
|
||||
|
||||
params.push(pageSize)
|
||||
params.push(offset)
|
||||
|
||||
const itemsResult = await executor<PddRawOrderRow>(
|
||||
`
|
||||
SELECT *
|
||||
FROM pdd_raw_orders
|
||||
${whereClause}
|
||||
ORDER BY COALESCE(confirm_time, order_time, created_at) DESC, id DESC
|
||||
LIMIT $${params.length - 1} OFFSET $${params.length}
|
||||
`,
|
||||
params,
|
||||
)
|
||||
|
||||
return {
|
||||
items: itemsResult.rows,
|
||||
total,
|
||||
page,
|
||||
pageSize,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user