feat(fulfillment): 增加事务 Outbox 同步履约状态

- 用事务 Outbox 和签名 Webhook 替换直接状态回调
- 增加 Outbox 投递 worker、重试与低频对账兜底
- 缓存客户领取页绑定状态并补充签名与迁移测试
This commit is contained in:
yml
2026-11-18 11:00:28 +08:00
parent 54b9f3251d
commit 9b2fcf1d1f
23 changed files with 1324 additions and 130 deletions
@@ -1,10 +1,10 @@
import type { PoolClient } from 'pg'
import { query, withTransaction } from '../../db/client.js'
import { publishRealtimeEvent } from '../realtime/realtime-event-service.js'
import { nowIso } from '../../utils/time.js'
import { createHttpError } from '../../utils/http.js'
import { logWarn } from '../../utils/logger.js'
import { notifyLiveHubOrderSyncAsync } from './live-hub-notifier.js'
type DouyuTaskRow = {
id: number
@@ -30,6 +30,9 @@ type DouyuTaskRow = {
bound_at: string | null
completed_at: string | null
last_synced_at: string | null
qr_url: string
qr_generated_at: string | null
qr_expires_at: string | null
created_at: string
updated_at: string
platform_order_id: string
@@ -230,9 +233,6 @@ export async function ensureDouyuFulfillmentTasks(orderNo = '') {
}
}
})
if (created > 0) {
notifyLiveHubOrderSyncAsync({ created, updated: candidates.rows.length - created })
}
return { created, updated: candidates.rows.length - created }
}
@@ -374,8 +374,11 @@ async function updateTaskFromLiveState(
WHEN $14 THEN COALESCE(completed_at, NOW())
ELSE completed_at
END,
qr_url = $15,
qr_generated_at = NULLIF($16, '')::timestamptz,
qr_expires_at = NULLIF($17, '')::timestamptz,
last_synced_at = NOW(),
updated_at = $15
updated_at = $18
WHERE id = $1
`,
[
@@ -393,6 +396,9 @@ async function updateTaskFromLiveState(
live.qrUrl,
live.bindConfirmed,
live.executionStatus === 'completed',
live.qrUrl,
live.qrGeneratedAt,
live.qrExpiresAt,
now,
],
)
@@ -437,29 +443,121 @@ function isLiveBindingState(value: unknown): value is LiveBindingState {
)
}
export async function applyLiveBindingStateSnapshot(input: unknown) {
function liveBindingStateFromTask(task: DouyuTaskRow): LiveBindingState {
return {
sessionId: Number(task.live_session_id),
platformOrderId: task.platform_order_id,
sourceOrderItemId: String(task.order_item_id),
productName: task.product_name,
quantity: Number(task.quantity || 1),
executionStatus: task.live_execution_status || 'preparing',
bindingStatus: task.live_binding_status || 'not_started',
exchangeStatus: '',
issueCode: task.issue_code || '',
message: task.last_error || '',
accountReady: !['waiting_profile', 'preparing'].includes(task.status),
bindTaskId: null,
bindTaskStatus: '',
bindPhase: task.binding_phase || '',
qrUrl: task.qr_url || '',
qrGeneratedAt: task.qr_generated_at || '',
qrExpiresAt: task.qr_expires_at || '',
roleName: task.bound_role_name || '',
areaName: task.bound_area_name || '',
platName: task.bound_plat_name || '',
bindReadyForConfirm: task.status === 'role_ready',
bindConfirmed: ['bound', 'waiting_operator', 'processing', 'completed'].includes(task.status),
completedAt: task.completed_at,
updatedAt: task.updated_at,
}
}
export async function applyLiveBindingStateSnapshot(
input: unknown,
options: { eventId?: string; eventType?: string } = {},
) {
const rawItems =
input && typeof input === 'object' && Array.isArray((input as { items?: unknown }).items)
? (input as { items: unknown[] }).items
: []
const items = rawItems.filter(isLiveBindingState)
if (items.length === 0) {
return { received: rawItems.length, updated: 0 }
}
const sessionIds = Array.from(
new Set(items.map((item) => Number(item.sessionId)).filter((id) => id > 0)),
)
if (sessionIds.length === 0) return { received: rawItems.length, updated: 0 }
const tasks = await query<DouyuTaskRow>(
`${TASK_SELECT}
WHERE task.live_session_id = ANY($1::bigint[])`,
[sessionIds],
const orderNos = Array.from(
new Set(items.map((item) => String(item.platformOrderId || '').trim()).filter(Boolean)),
).slice(0, 50)
const orderItemIds = Array.from(
new Set(items.map((item) => String(item.sourceOrderItemId || '').trim()).filter(Boolean)),
)
const taskBySessionId = new Map(tasks.rows.map((task) => [Number(task.live_session_id), task]))
const entries = items.flatMap((live) => {
const task = taskBySessionId.get(Number(live.sessionId))
return task ? [{ task, live }] : []
if (sessionIds.length === 0 && orderNos.length === 0 && orderItemIds.length === 0) {
return { received: rawItems.length, updated: 0 }
}
// Webhook 可能先于客户首次打开领取页到达;先幂等补齐本地订单投影。
for (const orderNo of orderNos) {
await ensureDouyuFulfillmentTasks(orderNo)
}
const result = await withTransaction(async (client) => {
const eventId = String(options.eventId || '').trim()
if (eventId) {
const claimed = await client.query(
`
INSERT INTO integration_inbox (
event_id, source, event_type, received_at, processed_at
) VALUES ($1, 'live_hub', $2, NOW(), NOW())
ON CONFLICT (event_id) DO NOTHING
RETURNING event_id
`,
[eventId, String(options.eventType || '')],
)
if (claimed.rowCount === 0) {
return { received: rawItems.length, updated: 0 }
}
}
const tasks = await client.query<DouyuTaskRow>(
`${TASK_SELECT}
WHERE task.live_session_id = ANY($1::bigint[])
OR CAST(task.order_item_id AS text) = ANY($2::text[])
OR o.platform_order_id = ANY($3::text[])`,
[sessionIds, orderItemIds, orderNos],
)
const taskBySessionId = new Map(
tasks.rows
.filter((task) => task.live_session_id)
.map((task) => [Number(task.live_session_id), task]),
)
const taskByOrderItemId = new Map(tasks.rows.map((task) => [String(task.order_item_id), task]))
const now = nowIso()
const entries = items.flatMap((live) => {
const task =
taskBySessionId.get(Number(live.sessionId)) ||
taskByOrderItemId.get(String(live.sourceOrderItemId || ''))
return task ? [{ task, live }] : []
})
for (const entry of entries) {
await updateTaskFromLiveState(client, entry.task, entry.live, now)
}
return { received: rawItems.length, updated: entries.length }
})
await applyLiveStates(entries)
return { received: rawItems.length, updated: entries.length }
if (result.updated > 0) {
publishRealtimeEvent({
type: 'order.changed',
entityId: 0,
scopes: ['admin_douyu_fulfillment'],
operation: 'refresh',
data: { douyuFulfillmentUpdated: true },
admin: true,
})
}
return result
}
async function ensureLiveSession(task: DouyuTaskRow) {
@@ -572,6 +670,21 @@ export async function refreshDouyuCustomerBinding(orderNo: string) {
return applyLiveState(task, live)
}
export async function getDouyuCustomerFulfillmentCached(orderNo: string) {
let task = await loadTaskByOrderNo(orderNo)
if (!task.live_session_id) {
return ensureLiveSession(task)
}
task = await loadTask(task.id)
const live = liveBindingStateFromTask(task)
return {
task,
live,
comparison: compareCustomerRole(task, live),
linkPath: `/coll/d/${encodeURIComponent(task.platform_order_id)}`,
}
}
export async function confirmDouyuCustomerBinding(orderNo: string) {
const refreshed = await refreshDouyuCustomerBinding(orderNo)
if (!refreshed.comparison.matched) {
@@ -594,10 +707,11 @@ export async function confirmDouyuCustomerBinding(orderNo: string) {
return applyLiveState(task, live)
}
async function syncLiveStatesForTasks(tasks: DouyuTaskRow[]) {
async function syncLiveStatesForTasks(tasks: DouyuTaskRow[], force = false) {
const now = Date.now()
const candidates = tasks.filter((task) => {
if (!task.live_session_id || TERMINAL_TASK_STATUSES.has(task.status)) return false
if (force) return true
const lastSyncedAt = Date.parse(task.last_synced_at || '')
return !lastSyncedAt || now - lastSyncedAt >= LIVE_STATE_SYNC_INTERVAL_MS
})
@@ -619,11 +733,33 @@ async function syncLiveStatesForTasks(tasks: DouyuTaskRow[]) {
return task ? [{ task, live }] : []
})
await applyLiveStates(entries)
if (entries.length > 0) {
publishRealtimeEvent({
type: 'order.changed',
entityId: 0,
scopes: ['admin_douyu_fulfillment'],
operation: 'refresh',
data: { douyuFulfillmentUpdated: true },
admin: true,
})
}
} catch (error) {
logWarn('[douyu-fulfillment]', '自动同步直播账号履约状态失败', { error })
}
}
export async function reconcileDouyuFulfillmentStates() {
const rows = await query<DouyuTaskRow>(
`${TASK_SELECT}
WHERE task.live_session_id IS NOT NULL
AND task.status NOT IN ('completed', 'cancelled')
ORDER BY task.updated_at ASC, task.id ASC
LIMIT 100`,
)
await syncLiveStatesForTasks(rows.rows, true)
return { checked: rows.rows.length }
}
export async function listDouyuFulfillmentTasks(
input: {
keyword?: unknown
@@ -661,15 +797,7 @@ export async function listDouyuFulfillmentTasks(
params,
)
params.push(pageSize, (page - 1) * pageSize)
let rows = await query<DouyuTaskRow>(
`${TASK_SELECT}
${where}
ORDER BY task.updated_at DESC, task.id DESC
LIMIT $${params.length - 1} OFFSET $${params.length}`,
params,
)
await syncLiveStatesForTasks(rows.rows)
rows = await query<DouyuTaskRow>(
const rows = await query<DouyuTaskRow>(
`${TASK_SELECT}
${where}
ORDER BY task.updated_at DESC, task.id DESC
@@ -0,0 +1,51 @@
import { logWarn } from '../../utils/logger.js'
import { reconcileDouyuFulfillmentStates } from './douyu-fulfillment-service.js'
const DEFAULT_RECONCILE_SECONDS = 900
let timer: NodeJS.Timeout | null = null
let stopped = true
function intervalMs() {
const value = Number(process.env.PLATFORM_FULFILLMENT_RECONCILE_SECONDS)
const seconds =
Number.isFinite(value) && value > 0
? Math.max(60, Math.trunc(value))
: DEFAULT_RECONCILE_SECONDS
return seconds * 1000
}
async function run() {
if (stopped) return
try {
await reconcileDouyuFulfillmentStates()
} catch (error) {
logWarn('[douyu-fulfillment]', '低频对账失败', { error })
} finally {
if (!stopped) {
timer = setTimeout(() => {
void run()
}, intervalMs())
}
}
}
export function startDouyuFulfillmentReconciler() {
if (timer || !stopped) return
stopped = false
// 启动后先让 Webhook 工作一分钟,再执行首轮兜底对账。
timer = setTimeout(
() => {
void run()
},
Math.min(60_000, intervalMs()),
)
}
export function stopDouyuFulfillmentReconciler() {
stopped = true
if (timer) {
clearTimeout(timer)
timer = null
}
}
@@ -1,54 +0,0 @@
import { randomUUID } from 'node:crypto'
import { logWarn } from '../../utils/logger.js'
const PUBLISH_ATTEMPTS = 3
const PUBLISH_TIMEOUT_MS = 5000
function liveHubBaseUrl() {
return String(process.env.LIVE_HUB_PLATFORM_FULFILLMENT_URL || '')
.trim()
.replace(/\/+$/, '')
}
export function notifyLiveHubOrderSyncAsync(detail: Record<string, unknown> = {}) {
const baseUrl = liveHubBaseUrl()
const integrationKey = String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim()
if (!baseUrl || !integrationKey) return
const event = {
eventId: randomUUID(),
occurredAt: new Date().toISOString(),
...detail,
}
void publish(`${baseUrl}/api/integration/platform-fulfillment/events/order-sync`, event)
}
async function publish(url: string, event: Record<string, unknown>) {
const integrationKey = String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim()
for (let attempt = 1; attempt <= PUBLISH_ATTEMPTS; attempt += 1) {
try {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Integration-Key': integrationKey,
},
body: JSON.stringify(event),
signal: AbortSignal.timeout(PUBLISH_TIMEOUT_MS),
})
if (response.ok) return
throw new Error(`live-hub 返回 HTTP ${response.status}`)
} catch (error) {
if (attempt >= PUBLISH_ATTEMPTS) {
logWarn('[douyu-fulfillment]', '推送 live-hub 订单同步事件失败', {
eventId: event.eventId,
attempts: attempt,
error,
})
return
}
await new Promise((resolve) => setTimeout(resolve, attempt * 500))
}
}
}
@@ -0,0 +1,15 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
test('Outbox migration keeps header event id and payload eventId identical', () => {
const sql = readFileSync(
new URL('../../db/migrations/172_platform_fulfillment_outbox.sql', import.meta.url),
'utf8',
)
assert.match(sql, /'eventId', event_id::text/)
assert.doesNotMatch(sql, /'eventId', gen_random_uuid\(\)::text/)
assert.match(sql, /CREATE TABLE IF NOT EXISTS integration_outbox/)
assert.match(sql, /CREATE TABLE IF NOT EXISTS integration_inbox/)
})
@@ -0,0 +1,209 @@
import { query, withTransaction } from '../../db/client.js'
import { logError, logInfo, logWarn } from '../../utils/logger.js'
import { buildWebhookSignature } from './webhook-signature.js'
const DEFAULT_INTERVAL_MS = 1000
const DEFAULT_BATCH_SIZE = 20
const DEFAULT_MAX_ATTEMPTS = 12
const REQUEST_TIMEOUT_MS = 5000
const PROCESSING_STALE_SECONDS = 120
type OutboxRow = {
id: string
event_id: string
payload_json: unknown
attempts: number
}
let timer: NodeJS.Timeout | null = null
let running = false
let stopped = true
function envInt(name: string, fallback: number, min: number, max: number) {
const parsed = Number(process.env[name])
if (!Number.isFinite(parsed)) return fallback
return Math.max(min, Math.min(max, Math.trunc(parsed)))
}
export function integrationOutboxConfigured() {
return Boolean(
String(process.env.LIVE_HUB_PLATFORM_FULFILLMENT_URL || '').trim() &&
String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim(),
)
}
export function integrationOutboxBackoffSeconds(attempts: number) {
const normalized = Math.max(1, Math.trunc(Number(attempts) || 1))
return Math.min(3600, 5 * 2 ** Math.min(normalized - 1, 10))
}
async function claimDueEvents(limit: number): Promise<OutboxRow[]> {
return withTransaction(async (client) => {
await client.query(
`
UPDATE integration_outbox
SET status = 'pending',
locked_at = NULL,
next_attempt_at = NOW()
WHERE status = 'processing'
AND locked_at < NOW() - ($1 * INTERVAL '1 second')
`,
[PROCESSING_STALE_SECONDS],
)
const claimed = await client.query<OutboxRow>(
`
SELECT id, event_id, payload_json, attempts
FROM integration_outbox
WHERE status = 'pending'
AND next_attempt_at <= NOW()
ORDER BY id ASC
LIMIT $1
FOR UPDATE SKIP LOCKED
`,
[limit],
)
const ids = claimed.rows.map((row) => Number(row.id)).filter((id) => id > 0)
if (ids.length === 0) return []
await client.query(
`
UPDATE integration_outbox
SET status = 'processing',
locked_at = NOW()
WHERE id = ANY($1::bigint[])
`,
[ids],
)
return claimed.rows
})
}
async function deliverEvent(row: OutboxRow) {
const baseUrl = String(process.env.LIVE_HUB_PLATFORM_FULFILLMENT_URL || '')
.trim()
.replace(/\/+$/, '')
const integrationKey = String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim()
const endpoint = `${baseUrl}/api/integration/platform-fulfillment/events/order-sync`
const rawBody =
typeof row.payload_json === 'string' ? row.payload_json : JSON.stringify(row.payload_json)
const timestamp = String(Math.floor(Date.now() / 1000))
const signature = buildWebhookSignature(integrationKey, timestamp, rawBody)
const response = await fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Integration-Key': integrationKey,
'X-Integration-Event-Id': row.event_id,
'X-Integration-Timestamp': timestamp,
'X-Integration-Signature': signature,
},
body: rawBody,
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
})
if (!response.ok) {
throw new Error(`live-hub 返回 HTTP ${response.status}`)
}
}
async function markSent(eventId: string) {
await query(
`
UPDATE integration_outbox
SET status = 'sent',
sent_at = NOW(),
locked_at = NULL,
last_error = ''
WHERE event_id = $1
`,
[eventId],
)
}
async function markFailed(row: OutboxRow, error: unknown) {
const attempts = Number(row.attempts || 0) + 1
const maxAttempts = envInt('INTEGRATION_OUTBOX_MAX_ATTEMPTS', DEFAULT_MAX_ATTEMPTS, 1, 100)
const dead = attempts >= maxAttempts
const message = (error instanceof Error ? error.message : String(error)).slice(0, 2000)
await query(
`
UPDATE integration_outbox
SET status = $2,
attempts = $3,
locked_at = NULL,
next_attempt_at = NOW() + ($4 * INTERVAL '1 second'),
last_error = $5
WHERE event_id = $1
`,
[
row.event_id,
dead ? 'dead' : 'pending',
attempts,
dead ? 0 : integrationOutboxBackoffSeconds(attempts),
message,
],
)
if (dead) {
logError('[integration/outbox]', '事件超过最大重试次数,已进入 dead', {
eventId: row.event_id,
attempts,
error: message,
})
}
}
async function publishDueEvents() {
if (!integrationOutboxConfigured()) return
const batchSize = envInt('INTEGRATION_OUTBOX_BATCH_SIZE', DEFAULT_BATCH_SIZE, 1, 100)
const rows = await claimDueEvents(batchSize)
for (const row of rows) {
try {
await deliverEvent(row)
await markSent(row.event_id)
} catch (error) {
await markFailed(row, error)
}
}
}
async function runLoop() {
if (stopped || running) return
running = true
try {
await publishDueEvents()
} catch (error) {
logWarn('[integration/outbox]', '投递轮次失败', { error })
} finally {
running = false
if (!stopped) {
timer = setTimeout(
() => {
void runLoop()
},
envInt('INTEGRATION_OUTBOX_INTERVAL_MS', DEFAULT_INTERVAL_MS, 250, 60_000),
)
}
}
}
export function startIntegrationOutboxWorker() {
if (timer || !stopped) return
stopped = false
if (!integrationOutboxConfigured()) {
logWarn('[integration/outbox]', '未配置 live-hub 地址或集成密钥,worker 仅保持待命')
} else {
logInfo('[integration/outbox]', '事务 Outbox worker 已启动')
}
timer = setTimeout(() => {
void runLoop()
}, 0)
}
export function stopIntegrationOutboxWorker() {
stopped = true
if (timer) {
clearTimeout(timer)
timer = null
}
}
@@ -0,0 +1,49 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { buildWebhookSignature, verifyWebhookSignature } from './webhook-signature.js'
test('webhook signature binds timestamp and raw body', () => {
const rawBody = '{"eventId":"evt-1","items":[]}'
const timestamp = '1783394219'
const signature = buildWebhookSignature('secret-key', timestamp, rawBody)
assert.equal(
verifyWebhookSignature({
secret: 'secret-key',
timestamp,
rawBody,
signature,
nowSeconds: Number(timestamp),
}),
true,
)
assert.equal(
verifyWebhookSignature({
secret: 'secret-key',
timestamp,
rawBody: `${rawBody} `,
signature,
nowSeconds: Number(timestamp),
}),
false,
)
})
test('webhook signature rejects expired timestamps', () => {
const timestamp = '1783394219'
const rawBody = '{}'
const signature = buildWebhookSignature('secret-key', timestamp, rawBody)
assert.equal(
verifyWebhookSignature({
secret: 'secret-key',
timestamp,
rawBody,
signature,
toleranceSeconds: 300,
nowSeconds: Number(timestamp) + 301,
}),
false,
)
})
@@ -0,0 +1,45 @@
import { createHmac, timingSafeEqual } from 'node:crypto'
const SIGNATURE_PREFIX = 'sha256='
const DEFAULT_TOLERANCE_SECONDS = 300
export function buildWebhookSignature(secret: string, timestamp: string, rawBody: string) {
return (
SIGNATURE_PREFIX + createHmac('sha256', secret).update(`${timestamp}.${rawBody}`).digest('hex')
)
}
export function verifyWebhookSignature(input: {
secret: string
timestamp: string
rawBody: string
signature: string
toleranceSeconds?: number
nowSeconds?: number
}) {
const secret = String(input.secret || '').trim()
const timestamp = String(input.timestamp || '').trim()
const signature = String(input.signature || '').trim()
if (!secret || !timestamp || !signature) return false
const timestampSeconds = Number(timestamp)
if (!Number.isInteger(timestampSeconds) || timestampSeconds <= 0) return false
const tolerance = Math.max(
1,
Math.trunc(Number(input.toleranceSeconds) || DEFAULT_TOLERANCE_SECONDS),
)
const nowSeconds = Math.trunc(Number(input.nowSeconds) || Date.now() / 1000)
if (Math.abs(nowSeconds - timestampSeconds) > tolerance) return false
const expected = buildWebhookSignature(secret, timestamp, String(input.rawBody || ''))
const expectedBuffer = Buffer.from(expected)
const providedBuffer = Buffer.from(signature)
if (expectedBuffer.length !== providedBuffer.length) return false
return timingSafeEqual(expectedBuffer, providedBuffer)
}
export function webhookToleranceSeconds() {
const value = Number(process.env.PLATFORM_FULFILLMENT_WEBHOOK_TOLERANCE_SECONDS || 300)
return Number.isFinite(value) && value > 0 ? Math.trunc(value) : DEFAULT_TOLERANCE_SECONDS
}