feat(fulfillment): 增加事务 Outbox 同步履约状态

- 用事务 Outbox 和签名 Webhook 替换直接状态回调
- 增加 Outbox 投递 worker、重试与低频对账兜底
- 缓存客户领取页绑定状态并补充签名与迁移测试
This commit is contained in:
yml
2026-11-18 11:00:28 +08:00
parent 54b9f3251d
commit 9b2fcf1d1f
23 changed files with 1324 additions and 130 deletions
+15 -1
View File
@@ -4,6 +4,7 @@ import test from 'node:test'
import { createApp } from './app.js'
import { createDefaultRuntimeConfig } from './config/defaults.js'
import { buildWebhookSignature } from './services/integration/webhook-signature.js'
import { createStartupState } from './startup/state.js'
test('HTTP health route and explicit CORS preflight', async () => {
@@ -64,13 +65,26 @@ test('履约状态集成接口要求密钥并接受空快照', async () => {
})
assert.equal(unauthorized.status, 401)
const rawBody = JSON.stringify({
eventId: '8a8da35c-46c3-45a8-a914-3d7f28ee91ca',
eventType: 'platform_fulfillment.binding_state_changed',
items: [],
})
const timestamp = String(Math.floor(Date.now() / 1000))
const accepted = await fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Integration-Key': 'integration-test-key',
'X-Integration-Event-Id': '8a8da35c-46c3-45a8-a914-3d7f28ee91ca',
'X-Integration-Timestamp': timestamp,
'X-Integration-Signature': buildWebhookSignature(
'integration-test-key',
timestamp,
rawBody,
),
},
body: JSON.stringify({ items: [] }),
body: rawBody,
})
assert.equal(accepted.status, 200)
assert.deepEqual((await accepted.json()).data, { received: 0, updated: 0 })
@@ -13,8 +13,13 @@ const WORKSPACE_ROOT = process.env.WORKSPACE_ROOT || path.resolve(BACKEND_ROOT,
const SPECIAL_RUNTIME_ENV_NAMES = [
'APP_BASE_URL',
'CLAIM_BASE_URL',
'INTEGRATION_OUTBOX_BATCH_SIZE',
'INTEGRATION_OUTBOX_INTERVAL_MS',
'INTEGRATION_OUTBOX_MAX_ATTEMPTS',
'LIVE_HUB_PLATFORM_FULFILLMENT_URL',
'PLATFORM_FULFILLMENT_INTEGRATION_KEY',
'PLATFORM_FULFILLMENT_RECONCILE_SECONDS',
'PLATFORM_FULFILLMENT_WEBHOOK_TOLERANCE_SECONDS',
]
const DEPLOYMENT_ONLY_ENV_NAMES = [
@@ -0,0 +1,254 @@
-- 172_platform_fulfillment_outbox.sql
--
-- 事务 Outbox:
-- 业务表变更与待发送事件同事务落库,进程崩溃后仍可由 worker 继续投递。
-- live-hub 接收端通过 integration_inbox 的 event_id 做幂等。
CREATE TABLE IF NOT EXISTS integration_outbox (
id BIGSERIAL PRIMARY KEY,
event_id UUID NOT NULL UNIQUE,
destination TEXT NOT NULL DEFAULT 'live_hub',
event_type TEXT NOT NULL,
dedupe_key TEXT UNIQUE,
payload_json JSONB NOT NULL,
status TEXT NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending', 'processing', 'sent', 'dead')),
attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
locked_at TIMESTAMPTZ,
sent_at TIMESTAMPTZ,
last_error TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_integration_outbox_due
ON integration_outbox (status, next_attempt_at, id);
CREATE INDEX IF NOT EXISTS idx_integration_outbox_destination
ON integration_outbox (destination, status, id);
CREATE TABLE IF NOT EXISTS integration_inbox (
event_id UUID PRIMARY KEY,
source TEXT NOT NULL DEFAULT '',
event_type TEXT NOT NULL DEFAULT '',
received_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
processed_at TIMESTAMPTZ
);
CREATE OR REPLACE FUNCTION enqueue_live_hub_platform_fulfillment_event(
p_order_id BIGINT,
p_order_item_id BIGINT,
p_fulfillment_id BIGINT,
p_reason TEXT
) RETURNS VOID
LANGUAGE plpgsql
AS $$
DECLARE
v_event_id UUID := gen_random_uuid();
v_platform_order_id TEXT := '';
v_dedupe_key TEXT;
BEGIN
SELECT platform_order_id
INTO v_platform_order_id
FROM orders
WHERE id = p_order_id;
IF COALESCE(v_platform_order_id, '') = '' THEN
RETURN;
END IF;
v_dedupe_key := CONCAT(
'platform-fulfillment-order:',
COALESCE(p_order_item_id, 0),
':',
p_reason
);
INSERT INTO integration_outbox (
event_id, destination, event_type, dedupe_key, payload_json, status, created_at
) VALUES (
v_event_id,
'live_hub',
'platform_fulfillment.order_changed',
v_dedupe_key,
jsonb_build_object(
'eventId', v_event_id::text,
'eventType', 'platform_fulfillment.order_changed',
'occurredAt', to_char(
clock_timestamp() AT TIME ZONE 'UTC',
'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'
),
'data', jsonb_build_object(
'orderId', p_order_id,
'orderItemId', p_order_item_id,
'fulfillmentId', p_fulfillment_id,
'platformOrderId', v_platform_order_id,
'reason', p_reason
)
),
'pending',
NOW()
)
ON CONFLICT (dedupe_key) DO NOTHING;
END;
$$;
CREATE OR REPLACE FUNCTION enqueue_platform_fulfillment_from_order()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
DECLARE
v_row RECORD;
BEGIN
FOR v_row IN
SELECT oi.id AS order_item_id, f.id AS fulfillment_id
FROM order_items oi
JOIN fulfillments f ON f.order_item_id = oi.id
JOIN manual_fulfillment_configs cfg ON cfg.id = f.fulfillment_config_id
WHERE oi.order_id = NEW.id
AND cfg.douyu_fulfillment_enabled = TRUE
AND f.channel = 'manual'
AND f.status IN ('assigned', 'processing', 'completed')
LOOP
PERFORM enqueue_live_hub_platform_fulfillment_event(
NEW.id,
v_row.order_item_id,
v_row.fulfillment_id,
CONCAT('order:', NEW.pay_status, ':', NEW.order_status)
);
END LOOP;
RETURN NEW;
END;
$$;
CREATE OR REPLACE FUNCTION enqueue_platform_fulfillment_from_fulfillment()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
DECLARE
v_order_id BIGINT;
v_enabled BOOLEAN := FALSE;
BEGIN
SELECT oi.order_id
INTO v_order_id
FROM order_items oi
WHERE oi.id = NEW.order_item_id;
SELECT cfg.douyu_fulfillment_enabled
INTO v_enabled
FROM manual_fulfillment_configs cfg
WHERE cfg.id = NEW.fulfillment_config_id;
IF v_order_id IS NOT NULL
AND v_enabled = TRUE
AND NEW.channel = 'manual'
AND NEW.status IN ('assigned', 'processing', 'completed') THEN
PERFORM enqueue_live_hub_platform_fulfillment_event(
v_order_id,
NEW.order_item_id,
NEW.id,
CONCAT('fulfillment:', NEW.status)
);
END IF;
RETURN NEW;
END;
$$;
CREATE OR REPLACE FUNCTION enqueue_platform_fulfillment_from_config()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
DECLARE
v_row RECORD;
BEGIN
IF NEW.douyu_fulfillment_enabled = TRUE THEN
FOR v_row IN
SELECT o.id AS order_id, oi.id AS order_item_id, f.id AS fulfillment_id
FROM order_items oi
JOIN orders o ON o.id = oi.order_id
JOIN fulfillments f ON f.order_item_id = oi.id
WHERE f.fulfillment_config_id = NEW.id
AND f.channel = 'manual'
AND f.status IN ('assigned', 'processing', 'completed')
LOOP
PERFORM enqueue_live_hub_platform_fulfillment_event(
v_row.order_id,
v_row.order_item_id,
v_row.fulfillment_id,
'config_enabled'
);
END LOOP;
END IF;
RETURN NEW;
END;
$$;
DROP TRIGGER IF EXISTS trg_platform_fulfillment_order_sync ON orders;
CREATE TRIGGER trg_platform_fulfillment_order_sync
AFTER INSERT OR UPDATE OF pay_status, order_status ON orders
FOR EACH ROW
EXECUTE FUNCTION enqueue_platform_fulfillment_from_order();
DROP TRIGGER IF EXISTS trg_platform_fulfillment_fulfillment_sync ON fulfillments;
CREATE TRIGGER trg_platform_fulfillment_fulfillment_sync
AFTER INSERT OR UPDATE OF channel, status, fulfillment_config_id ON fulfillments
FOR EACH ROW
EXECUTE FUNCTION enqueue_platform_fulfillment_from_fulfillment();
DROP TRIGGER IF EXISTS trg_platform_fulfillment_config_sync
ON manual_fulfillment_configs;
CREATE TRIGGER trg_platform_fulfillment_config_sync
AFTER UPDATE OF douyu_fulfillment_enabled ON manual_fulfillment_configs
FOR EACH ROW
WHEN (OLD.douyu_fulfillment_enabled IS DISTINCT FROM NEW.douyu_fulfillment_enabled)
EXECUTE FUNCTION enqueue_platform_fulfillment_from_config();
-- 上线时补一次存量待履约订单;之后全部由业务写入同事务触发。
INSERT INTO integration_outbox (
event_id, destination, event_type, dedupe_key, payload_json, status, created_at
)
SELECT
event_id,
'live_hub',
'platform_fulfillment.order_changed',
CONCAT('platform-fulfillment-backfill:', candidate.order_item_id),
jsonb_build_object(
'eventId', event_id::text,
'eventType', 'platform_fulfillment.order_changed',
'occurredAt', to_char(
clock_timestamp() AT TIME ZONE 'UTC',
'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'
),
'data', jsonb_build_object(
'orderId', candidate.order_id,
'orderItemId', candidate.order_item_id,
'fulfillmentId', candidate.fulfillment_id,
'platformOrderId', candidate.platform_order_id,
'reason', 'migration_backfill'
)
),
'pending',
NOW()
FROM (
SELECT
gen_random_uuid() AS event_id,
o.id AS order_id,
oi.id AS order_item_id,
f.id AS fulfillment_id,
o.platform_order_id
FROM orders o
JOIN order_items oi ON oi.order_id = o.id
JOIN fulfillments f ON f.order_item_id = oi.id
JOIN manual_fulfillment_configs cfg ON cfg.id = f.fulfillment_config_id
WHERE cfg.douyu_fulfillment_enabled = TRUE
AND f.channel = 'manual'
AND f.status = 'assigned'
AND o.pay_status = 'paid'
AND o.order_status NOT IN ('refunded', 'cancelled', 'closed')
AND oi.quantity = 1
) candidate
ON CONFLICT (dedupe_key) DO NOTHING;
COMMENT ON TABLE integration_outbox IS
'order_site 事务 Outbox;业务提交与待发送事件同事务,worker 负责签名投递';
COMMENT ON TABLE integration_inbox IS
'order_site 入站 Webhook 幂等表;event_id 全局唯一';
@@ -0,0 +1,12 @@
-- 173_douyu_fulfillment_cached_binding.sql
--
-- 客户领取页的绑定状态改由 live-hub 签名 Webhook 投影到本地,
-- 页面刷新只读这些缓存字段,不再服务端反向轮询 live-hub。
ALTER TABLE douyu_fulfillment_tasks
ADD COLUMN IF NOT EXISTS qr_url TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS qr_generated_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS qr_expires_at TIMESTAMPTZ;
COMMENT ON COLUMN douyu_fulfillment_tasks.qr_url IS
'live-hub 绑定二维码的客户页投影;仅用于展示,不包含内部账号或 CK';
+3 -3
View File
@@ -2,8 +2,8 @@ import { Router } from 'express'
import {
confirmDouyuCustomerBinding,
getDouyuCustomerFulfillmentCached,
getCustomerDouyuFulfillment,
refreshDouyuCustomerBinding,
saveDouyuCustomerProfile,
startDouyuCustomerBinding,
} from '../services/douyu-fulfillment/douyu-fulfillment-service.js'
@@ -46,8 +46,8 @@ router.post(
router.get(
'/:orderNo/binding/status',
createRouteHandler((req) => refreshDouyuCustomerBinding(String(req.params.orderNo || '')), {
errorMessage: '刷新绑定状态失败',
createRouteHandler((req) => getDouyuCustomerFulfillmentCached(String(req.params.orderNo || '')), {
errorMessage: '读取绑定状态失败',
scope: '[douyu-fulfillment/:orderNo/binding/status]',
}),
)
@@ -3,7 +3,11 @@ import { timingSafeEqual } from 'node:crypto'
import { Router } from 'express'
import { applyLiveBindingStateSnapshot } from '../services/douyu-fulfillment/douyu-fulfillment-service.js'
import { createRouteHandler } from '../utils/http.js'
import {
verifyWebhookSignature,
webhookToleranceSeconds,
} from '../services/integration/webhook-signature.js'
import { createHttpError, createRouteHandler } from '../utils/http.js'
const router = Router()
@@ -41,11 +45,44 @@ router.use('/platform-fulfillment', (req, res, next) => {
router.post(
'/platform-fulfillment/binding-state',
createRouteHandler((req) => applyLiveBindingStateSnapshot(req.body), {
successMessage: '履约状态已同步',
errorMessage: '同步履约状态失败',
scope: '[integration/platform-fulfillment/binding-state]',
}),
createRouteHandler(
(req) => {
const integrationKey = String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim()
const eventId = String(req.header('x-integration-event-id') || '').trim()
const timestamp = String(req.header('x-integration-timestamp') || '').trim()
const signature = String(req.header('x-integration-signature') || '').trim()
const body = req.body as Record<string, unknown> | undefined
if (
!verifyWebhookSignature({
secret: integrationKey,
timestamp,
rawBody: req.rawBody || '',
signature,
toleranceSeconds: webhookToleranceSeconds(),
})
) {
throw createHttpError('履约 Webhook 签名无效', {
statusCode: 401,
errorCode: 'integration_signature_invalid',
})
}
if (!body || String(body.eventId || '').trim() !== eventId) {
throw createHttpError('履约 Webhook 事件编号不一致', {
statusCode: 400,
errorCode: 'integration_event_id_mismatch',
})
}
return applyLiveBindingStateSnapshot(body, {
eventId,
eventType: String(body.eventType || ''),
})
},
{
successMessage: '履约状态已同步',
errorMessage: '同步履约状态失败',
scope: '[integration/platform-fulfillment/binding-state]',
},
),
)
export default router
@@ -1,10 +1,10 @@
import type { PoolClient } from 'pg'
import { query, withTransaction } from '../../db/client.js'
import { publishRealtimeEvent } from '../realtime/realtime-event-service.js'
import { nowIso } from '../../utils/time.js'
import { createHttpError } from '../../utils/http.js'
import { logWarn } from '../../utils/logger.js'
import { notifyLiveHubOrderSyncAsync } from './live-hub-notifier.js'
type DouyuTaskRow = {
id: number
@@ -30,6 +30,9 @@ type DouyuTaskRow = {
bound_at: string | null
completed_at: string | null
last_synced_at: string | null
qr_url: string
qr_generated_at: string | null
qr_expires_at: string | null
created_at: string
updated_at: string
platform_order_id: string
@@ -230,9 +233,6 @@ export async function ensureDouyuFulfillmentTasks(orderNo = '') {
}
}
})
if (created > 0) {
notifyLiveHubOrderSyncAsync({ created, updated: candidates.rows.length - created })
}
return { created, updated: candidates.rows.length - created }
}
@@ -374,8 +374,11 @@ async function updateTaskFromLiveState(
WHEN $14 THEN COALESCE(completed_at, NOW())
ELSE completed_at
END,
qr_url = $15,
qr_generated_at = NULLIF($16, '')::timestamptz,
qr_expires_at = NULLIF($17, '')::timestamptz,
last_synced_at = NOW(),
updated_at = $15
updated_at = $18
WHERE id = $1
`,
[
@@ -393,6 +396,9 @@ async function updateTaskFromLiveState(
live.qrUrl,
live.bindConfirmed,
live.executionStatus === 'completed',
live.qrUrl,
live.qrGeneratedAt,
live.qrExpiresAt,
now,
],
)
@@ -437,29 +443,121 @@ function isLiveBindingState(value: unknown): value is LiveBindingState {
)
}
export async function applyLiveBindingStateSnapshot(input: unknown) {
function liveBindingStateFromTask(task: DouyuTaskRow): LiveBindingState {
return {
sessionId: Number(task.live_session_id),
platformOrderId: task.platform_order_id,
sourceOrderItemId: String(task.order_item_id),
productName: task.product_name,
quantity: Number(task.quantity || 1),
executionStatus: task.live_execution_status || 'preparing',
bindingStatus: task.live_binding_status || 'not_started',
exchangeStatus: '',
issueCode: task.issue_code || '',
message: task.last_error || '',
accountReady: !['waiting_profile', 'preparing'].includes(task.status),
bindTaskId: null,
bindTaskStatus: '',
bindPhase: task.binding_phase || '',
qrUrl: task.qr_url || '',
qrGeneratedAt: task.qr_generated_at || '',
qrExpiresAt: task.qr_expires_at || '',
roleName: task.bound_role_name || '',
areaName: task.bound_area_name || '',
platName: task.bound_plat_name || '',
bindReadyForConfirm: task.status === 'role_ready',
bindConfirmed: ['bound', 'waiting_operator', 'processing', 'completed'].includes(task.status),
completedAt: task.completed_at,
updatedAt: task.updated_at,
}
}
export async function applyLiveBindingStateSnapshot(
input: unknown,
options: { eventId?: string; eventType?: string } = {},
) {
const rawItems =
input && typeof input === 'object' && Array.isArray((input as { items?: unknown }).items)
? (input as { items: unknown[] }).items
: []
const items = rawItems.filter(isLiveBindingState)
if (items.length === 0) {
return { received: rawItems.length, updated: 0 }
}
const sessionIds = Array.from(
new Set(items.map((item) => Number(item.sessionId)).filter((id) => id > 0)),
)
if (sessionIds.length === 0) return { received: rawItems.length, updated: 0 }
const tasks = await query<DouyuTaskRow>(
`${TASK_SELECT}
WHERE task.live_session_id = ANY($1::bigint[])`,
[sessionIds],
const orderNos = Array.from(
new Set(items.map((item) => String(item.platformOrderId || '').trim()).filter(Boolean)),
).slice(0, 50)
const orderItemIds = Array.from(
new Set(items.map((item) => String(item.sourceOrderItemId || '').trim()).filter(Boolean)),
)
const taskBySessionId = new Map(tasks.rows.map((task) => [Number(task.live_session_id), task]))
const entries = items.flatMap((live) => {
const task = taskBySessionId.get(Number(live.sessionId))
return task ? [{ task, live }] : []
if (sessionIds.length === 0 && orderNos.length === 0 && orderItemIds.length === 0) {
return { received: rawItems.length, updated: 0 }
}
// Webhook 可能先于客户首次打开领取页到达;先幂等补齐本地订单投影。
for (const orderNo of orderNos) {
await ensureDouyuFulfillmentTasks(orderNo)
}
const result = await withTransaction(async (client) => {
const eventId = String(options.eventId || '').trim()
if (eventId) {
const claimed = await client.query(
`
INSERT INTO integration_inbox (
event_id, source, event_type, received_at, processed_at
) VALUES ($1, 'live_hub', $2, NOW(), NOW())
ON CONFLICT (event_id) DO NOTHING
RETURNING event_id
`,
[eventId, String(options.eventType || '')],
)
if (claimed.rowCount === 0) {
return { received: rawItems.length, updated: 0 }
}
}
const tasks = await client.query<DouyuTaskRow>(
`${TASK_SELECT}
WHERE task.live_session_id = ANY($1::bigint[])
OR CAST(task.order_item_id AS text) = ANY($2::text[])
OR o.platform_order_id = ANY($3::text[])`,
[sessionIds, orderItemIds, orderNos],
)
const taskBySessionId = new Map(
tasks.rows
.filter((task) => task.live_session_id)
.map((task) => [Number(task.live_session_id), task]),
)
const taskByOrderItemId = new Map(tasks.rows.map((task) => [String(task.order_item_id), task]))
const now = nowIso()
const entries = items.flatMap((live) => {
const task =
taskBySessionId.get(Number(live.sessionId)) ||
taskByOrderItemId.get(String(live.sourceOrderItemId || ''))
return task ? [{ task, live }] : []
})
for (const entry of entries) {
await updateTaskFromLiveState(client, entry.task, entry.live, now)
}
return { received: rawItems.length, updated: entries.length }
})
await applyLiveStates(entries)
return { received: rawItems.length, updated: entries.length }
if (result.updated > 0) {
publishRealtimeEvent({
type: 'order.changed',
entityId: 0,
scopes: ['admin_douyu_fulfillment'],
operation: 'refresh',
data: { douyuFulfillmentUpdated: true },
admin: true,
})
}
return result
}
async function ensureLiveSession(task: DouyuTaskRow) {
@@ -572,6 +670,21 @@ export async function refreshDouyuCustomerBinding(orderNo: string) {
return applyLiveState(task, live)
}
export async function getDouyuCustomerFulfillmentCached(orderNo: string) {
let task = await loadTaskByOrderNo(orderNo)
if (!task.live_session_id) {
return ensureLiveSession(task)
}
task = await loadTask(task.id)
const live = liveBindingStateFromTask(task)
return {
task,
live,
comparison: compareCustomerRole(task, live),
linkPath: `/coll/d/${encodeURIComponent(task.platform_order_id)}`,
}
}
export async function confirmDouyuCustomerBinding(orderNo: string) {
const refreshed = await refreshDouyuCustomerBinding(orderNo)
if (!refreshed.comparison.matched) {
@@ -594,10 +707,11 @@ export async function confirmDouyuCustomerBinding(orderNo: string) {
return applyLiveState(task, live)
}
async function syncLiveStatesForTasks(tasks: DouyuTaskRow[]) {
async function syncLiveStatesForTasks(tasks: DouyuTaskRow[], force = false) {
const now = Date.now()
const candidates = tasks.filter((task) => {
if (!task.live_session_id || TERMINAL_TASK_STATUSES.has(task.status)) return false
if (force) return true
const lastSyncedAt = Date.parse(task.last_synced_at || '')
return !lastSyncedAt || now - lastSyncedAt >= LIVE_STATE_SYNC_INTERVAL_MS
})
@@ -619,11 +733,33 @@ async function syncLiveStatesForTasks(tasks: DouyuTaskRow[]) {
return task ? [{ task, live }] : []
})
await applyLiveStates(entries)
if (entries.length > 0) {
publishRealtimeEvent({
type: 'order.changed',
entityId: 0,
scopes: ['admin_douyu_fulfillment'],
operation: 'refresh',
data: { douyuFulfillmentUpdated: true },
admin: true,
})
}
} catch (error) {
logWarn('[douyu-fulfillment]', '自动同步直播账号履约状态失败', { error })
}
}
export async function reconcileDouyuFulfillmentStates() {
const rows = await query<DouyuTaskRow>(
`${TASK_SELECT}
WHERE task.live_session_id IS NOT NULL
AND task.status NOT IN ('completed', 'cancelled')
ORDER BY task.updated_at ASC, task.id ASC
LIMIT 100`,
)
await syncLiveStatesForTasks(rows.rows, true)
return { checked: rows.rows.length }
}
export async function listDouyuFulfillmentTasks(
input: {
keyword?: unknown
@@ -661,15 +797,7 @@ export async function listDouyuFulfillmentTasks(
params,
)
params.push(pageSize, (page - 1) * pageSize)
let rows = await query<DouyuTaskRow>(
`${TASK_SELECT}
${where}
ORDER BY task.updated_at DESC, task.id DESC
LIMIT $${params.length - 1} OFFSET $${params.length}`,
params,
)
await syncLiveStatesForTasks(rows.rows)
rows = await query<DouyuTaskRow>(
const rows = await query<DouyuTaskRow>(
`${TASK_SELECT}
${where}
ORDER BY task.updated_at DESC, task.id DESC
@@ -0,0 +1,51 @@
import { logWarn } from '../../utils/logger.js'
import { reconcileDouyuFulfillmentStates } from './douyu-fulfillment-service.js'
const DEFAULT_RECONCILE_SECONDS = 900
let timer: NodeJS.Timeout | null = null
let stopped = true
function intervalMs() {
const value = Number(process.env.PLATFORM_FULFILLMENT_RECONCILE_SECONDS)
const seconds =
Number.isFinite(value) && value > 0
? Math.max(60, Math.trunc(value))
: DEFAULT_RECONCILE_SECONDS
return seconds * 1000
}
async function run() {
if (stopped) return
try {
await reconcileDouyuFulfillmentStates()
} catch (error) {
logWarn('[douyu-fulfillment]', '低频对账失败', { error })
} finally {
if (!stopped) {
timer = setTimeout(() => {
void run()
}, intervalMs())
}
}
}
export function startDouyuFulfillmentReconciler() {
if (timer || !stopped) return
stopped = false
// 启动后先让 Webhook 工作一分钟,再执行首轮兜底对账。
timer = setTimeout(
() => {
void run()
},
Math.min(60_000, intervalMs()),
)
}
export function stopDouyuFulfillmentReconciler() {
stopped = true
if (timer) {
clearTimeout(timer)
timer = null
}
}
@@ -1,54 +0,0 @@
import { randomUUID } from 'node:crypto'
import { logWarn } from '../../utils/logger.js'
const PUBLISH_ATTEMPTS = 3
const PUBLISH_TIMEOUT_MS = 5000
function liveHubBaseUrl() {
return String(process.env.LIVE_HUB_PLATFORM_FULFILLMENT_URL || '')
.trim()
.replace(/\/+$/, '')
}
export function notifyLiveHubOrderSyncAsync(detail: Record<string, unknown> = {}) {
const baseUrl = liveHubBaseUrl()
const integrationKey = String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim()
if (!baseUrl || !integrationKey) return
const event = {
eventId: randomUUID(),
occurredAt: new Date().toISOString(),
...detail,
}
void publish(`${baseUrl}/api/integration/platform-fulfillment/events/order-sync`, event)
}
async function publish(url: string, event: Record<string, unknown>) {
const integrationKey = String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim()
for (let attempt = 1; attempt <= PUBLISH_ATTEMPTS; attempt += 1) {
try {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Integration-Key': integrationKey,
},
body: JSON.stringify(event),
signal: AbortSignal.timeout(PUBLISH_TIMEOUT_MS),
})
if (response.ok) return
throw new Error(`live-hub 返回 HTTP ${response.status}`)
} catch (error) {
if (attempt >= PUBLISH_ATTEMPTS) {
logWarn('[douyu-fulfillment]', '推送 live-hub 订单同步事件失败', {
eventId: event.eventId,
attempts: attempt,
error,
})
return
}
await new Promise((resolve) => setTimeout(resolve, attempt * 500))
}
}
}
@@ -0,0 +1,15 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
test('Outbox migration keeps header event id and payload eventId identical', () => {
const sql = readFileSync(
new URL('../../db/migrations/172_platform_fulfillment_outbox.sql', import.meta.url),
'utf8',
)
assert.match(sql, /'eventId', event_id::text/)
assert.doesNotMatch(sql, /'eventId', gen_random_uuid\(\)::text/)
assert.match(sql, /CREATE TABLE IF NOT EXISTS integration_outbox/)
assert.match(sql, /CREATE TABLE IF NOT EXISTS integration_inbox/)
})
@@ -0,0 +1,209 @@
import { query, withTransaction } from '../../db/client.js'
import { logError, logInfo, logWarn } from '../../utils/logger.js'
import { buildWebhookSignature } from './webhook-signature.js'
const DEFAULT_INTERVAL_MS = 1000
const DEFAULT_BATCH_SIZE = 20
const DEFAULT_MAX_ATTEMPTS = 12
const REQUEST_TIMEOUT_MS = 5000
const PROCESSING_STALE_SECONDS = 120
type OutboxRow = {
id: string
event_id: string
payload_json: unknown
attempts: number
}
let timer: NodeJS.Timeout | null = null
let running = false
let stopped = true
function envInt(name: string, fallback: number, min: number, max: number) {
const parsed = Number(process.env[name])
if (!Number.isFinite(parsed)) return fallback
return Math.max(min, Math.min(max, Math.trunc(parsed)))
}
export function integrationOutboxConfigured() {
return Boolean(
String(process.env.LIVE_HUB_PLATFORM_FULFILLMENT_URL || '').trim() &&
String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim(),
)
}
export function integrationOutboxBackoffSeconds(attempts: number) {
const normalized = Math.max(1, Math.trunc(Number(attempts) || 1))
return Math.min(3600, 5 * 2 ** Math.min(normalized - 1, 10))
}
async function claimDueEvents(limit: number): Promise<OutboxRow[]> {
return withTransaction(async (client) => {
await client.query(
`
UPDATE integration_outbox
SET status = 'pending',
locked_at = NULL,
next_attempt_at = NOW()
WHERE status = 'processing'
AND locked_at < NOW() - ($1 * INTERVAL '1 second')
`,
[PROCESSING_STALE_SECONDS],
)
const claimed = await client.query<OutboxRow>(
`
SELECT id, event_id, payload_json, attempts
FROM integration_outbox
WHERE status = 'pending'
AND next_attempt_at <= NOW()
ORDER BY id ASC
LIMIT $1
FOR UPDATE SKIP LOCKED
`,
[limit],
)
const ids = claimed.rows.map((row) => Number(row.id)).filter((id) => id > 0)
if (ids.length === 0) return []
await client.query(
`
UPDATE integration_outbox
SET status = 'processing',
locked_at = NOW()
WHERE id = ANY($1::bigint[])
`,
[ids],
)
return claimed.rows
})
}
async function deliverEvent(row: OutboxRow) {
const baseUrl = String(process.env.LIVE_HUB_PLATFORM_FULFILLMENT_URL || '')
.trim()
.replace(/\/+$/, '')
const integrationKey = String(process.env.PLATFORM_FULFILLMENT_INTEGRATION_KEY || '').trim()
const endpoint = `${baseUrl}/api/integration/platform-fulfillment/events/order-sync`
const rawBody =
typeof row.payload_json === 'string' ? row.payload_json : JSON.stringify(row.payload_json)
const timestamp = String(Math.floor(Date.now() / 1000))
const signature = buildWebhookSignature(integrationKey, timestamp, rawBody)
const response = await fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Integration-Key': integrationKey,
'X-Integration-Event-Id': row.event_id,
'X-Integration-Timestamp': timestamp,
'X-Integration-Signature': signature,
},
body: rawBody,
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
})
if (!response.ok) {
throw new Error(`live-hub 返回 HTTP ${response.status}`)
}
}
async function markSent(eventId: string) {
await query(
`
UPDATE integration_outbox
SET status = 'sent',
sent_at = NOW(),
locked_at = NULL,
last_error = ''
WHERE event_id = $1
`,
[eventId],
)
}
async function markFailed(row: OutboxRow, error: unknown) {
const attempts = Number(row.attempts || 0) + 1
const maxAttempts = envInt('INTEGRATION_OUTBOX_MAX_ATTEMPTS', DEFAULT_MAX_ATTEMPTS, 1, 100)
const dead = attempts >= maxAttempts
const message = (error instanceof Error ? error.message : String(error)).slice(0, 2000)
await query(
`
UPDATE integration_outbox
SET status = $2,
attempts = $3,
locked_at = NULL,
next_attempt_at = NOW() + ($4 * INTERVAL '1 second'),
last_error = $5
WHERE event_id = $1
`,
[
row.event_id,
dead ? 'dead' : 'pending',
attempts,
dead ? 0 : integrationOutboxBackoffSeconds(attempts),
message,
],
)
if (dead) {
logError('[integration/outbox]', '事件超过最大重试次数,已进入 dead', {
eventId: row.event_id,
attempts,
error: message,
})
}
}
async function publishDueEvents() {
if (!integrationOutboxConfigured()) return
const batchSize = envInt('INTEGRATION_OUTBOX_BATCH_SIZE', DEFAULT_BATCH_SIZE, 1, 100)
const rows = await claimDueEvents(batchSize)
for (const row of rows) {
try {
await deliverEvent(row)
await markSent(row.event_id)
} catch (error) {
await markFailed(row, error)
}
}
}
async function runLoop() {
if (stopped || running) return
running = true
try {
await publishDueEvents()
} catch (error) {
logWarn('[integration/outbox]', '投递轮次失败', { error })
} finally {
running = false
if (!stopped) {
timer = setTimeout(
() => {
void runLoop()
},
envInt('INTEGRATION_OUTBOX_INTERVAL_MS', DEFAULT_INTERVAL_MS, 250, 60_000),
)
}
}
}
export function startIntegrationOutboxWorker() {
if (timer || !stopped) return
stopped = false
if (!integrationOutboxConfigured()) {
logWarn('[integration/outbox]', '未配置 live-hub 地址或集成密钥,worker 仅保持待命')
} else {
logInfo('[integration/outbox]', '事务 Outbox worker 已启动')
}
timer = setTimeout(() => {
void runLoop()
}, 0)
}
export function stopIntegrationOutboxWorker() {
stopped = true
if (timer) {
clearTimeout(timer)
timer = null
}
}
@@ -0,0 +1,49 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { buildWebhookSignature, verifyWebhookSignature } from './webhook-signature.js'
test('webhook signature binds timestamp and raw body', () => {
const rawBody = '{"eventId":"evt-1","items":[]}'
const timestamp = '1783394219'
const signature = buildWebhookSignature('secret-key', timestamp, rawBody)
assert.equal(
verifyWebhookSignature({
secret: 'secret-key',
timestamp,
rawBody,
signature,
nowSeconds: Number(timestamp),
}),
true,
)
assert.equal(
verifyWebhookSignature({
secret: 'secret-key',
timestamp,
rawBody: `${rawBody} `,
signature,
nowSeconds: Number(timestamp),
}),
false,
)
})
test('webhook signature rejects expired timestamps', () => {
const timestamp = '1783394219'
const rawBody = '{}'
const signature = buildWebhookSignature('secret-key', timestamp, rawBody)
assert.equal(
verifyWebhookSignature({
secret: 'secret-key',
timestamp,
rawBody,
signature,
toleranceSeconds: 300,
nowSeconds: Number(timestamp) + 301,
}),
false,
)
})
@@ -0,0 +1,45 @@
import { createHmac, timingSafeEqual } from 'node:crypto'
const SIGNATURE_PREFIX = 'sha256='
const DEFAULT_TOLERANCE_SECONDS = 300
export function buildWebhookSignature(secret: string, timestamp: string, rawBody: string) {
return (
SIGNATURE_PREFIX + createHmac('sha256', secret).update(`${timestamp}.${rawBody}`).digest('hex')
)
}
export function verifyWebhookSignature(input: {
secret: string
timestamp: string
rawBody: string
signature: string
toleranceSeconds?: number
nowSeconds?: number
}) {
const secret = String(input.secret || '').trim()
const timestamp = String(input.timestamp || '').trim()
const signature = String(input.signature || '').trim()
if (!secret || !timestamp || !signature) return false
const timestampSeconds = Number(timestamp)
if (!Number.isInteger(timestampSeconds) || timestampSeconds <= 0) return false
const tolerance = Math.max(
1,
Math.trunc(Number(input.toleranceSeconds) || DEFAULT_TOLERANCE_SECONDS),
)
const nowSeconds = Math.trunc(Number(input.nowSeconds) || Date.now() / 1000)
if (Math.abs(nowSeconds - timestampSeconds) > tolerance) return false
const expected = buildWebhookSignature(secret, timestamp, String(input.rawBody || ''))
const expectedBuffer = Buffer.from(expected)
const providedBuffer = Buffer.from(signature)
if (expectedBuffer.length !== providedBuffer.length) return false
return timingSafeEqual(expectedBuffer, providedBuffer)
}
export function webhookToleranceSeconds() {
const value = Number(process.env.PLATFORM_FULFILLMENT_WEBHOOK_TOLERANCE_SECONDS || 300)
return Number.isFinite(value) && value > 0 ? Math.trunc(value) : DEFAULT_TOLERANCE_SECONDS
}
+4
View File
@@ -1,8 +1,10 @@
import { runDatabaseMigrations } from '../db/migrate.js'
import { startIntegrationOutboxWorker } from '../services/integration/outbox-worker.js'
import { ensureAdminUsersBootstrapped } from '../services/admin/admin-auth-service.js'
import { ensureFulfillmentCatalogBootstrapped } from '../services/bootstrap/fulfillment-bootstrap-service.js'
import { migrateJsonConfigFilesToDatabase } from '../services/config/json-config-migration-service.js'
import { startKuaishouIndustrySendCallbackRetryWorker } from '../services/platforms/kuaishou-industry/send-code-service.js'
import { startDouyuFulfillmentReconciler } from '../services/douyu-fulfillment/fulfillment-reconciler.js'
import { startScheduledJobs } from '../services/scheduler/scheduler-service.js'
import { logError, logInfo } from '../utils/logger.js'
import { startMemoryReporter } from './memory-reporter.js'
@@ -46,6 +48,8 @@ export async function bootstrapCoreServices(
startScheduledJobs()
startKuaishouIndustrySendCallbackRetryWorker()
startIntegrationOutboxWorker()
startDouyuFulfillmentReconciler()
startMemoryReporter()
break
} catch (error) {
+4
View File
@@ -1,6 +1,8 @@
import type { Server } from 'node:http'
import { stopKuaishouIndustrySendCallbackRetryWorker } from '../services/platforms/kuaishou-industry/send-code-service.js'
import { stopIntegrationOutboxWorker } from '../services/integration/outbox-worker.js'
import { stopDouyuFulfillmentReconciler } from '../services/douyu-fulfillment/fulfillment-reconciler.js'
import { stopScheduledJobs } from '../services/scheduler/scheduler-service.js'
import { closeDb } from '../db/client.js'
import { logError, logInfo } from '../utils/logger.js'
@@ -28,6 +30,8 @@ export function createShutdownController(
stopScheduledJobs()
stopKuaishouIndustrySendCallbackRetryWorker()
stopIntegrationOutboxWorker()
stopDouyuFulfillmentReconciler()
if (server.listening) {
await new Promise<void>((resolve) => {