From 5e59a5fdb10dbd9ffaaab60d3af5d0a00c7dcf96 Mon Sep 17 00:00:00 2001 From: yml2213 Date: Sat, 19 Sep 2026 11:39:30 +0800 Subject: [PATCH] =?UTF-8?q?fix(order-service):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E5=8C=85=E5=90=AB=20Emoji=20=E5=92=8C=E5=AD=A4=E7=AB=8B?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E5=AD=97=E7=AC=A6=E7=9A=84=E8=AE=A2=E5=8D=95?= =?UTF-8?q?=E5=9C=A8=20PostgreSQL=20jsonb=20=E4=B8=8B=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=AD=BB=E5=BE=AA=E7=8E=AF=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E8=BF=9B=E5=8D=95=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 在 utils/json.ts 新增 sanitizeWellFormedUnicode、deepSanitizeUnicode 和 safeJsonStringify，自动清理空字节 (\0) 并将畸变孤立 UTF-16 代理字符 (Lone Surrogates) 安全转换为规范字符，杜绝 PostgreSQL jsonb 解析错误； 2. 拼多多订单映射器与通用进单服务 (order-service) 全面接入 Unicode 深度清洗与安全序列化，确保买家昵称/留言中的各类特殊 Emoji 和生僻字符平滑落库； 3. 数据库仓储层 (order-repo / order-item-repo) 针对 raw_payload_json、spec_json、item_snapshot_json 等 jsonb 字段增加兜底安全序列化保护； 4. 补充 utils/json.test.ts 与 pdd-order-mapper.test.ts 单元测试，覆盖畸变字符与空字节场景。 --- .../src/repositories/order-item-repo.ts | 9 +-- apps/backend/src/repositories/order-repo.ts | 17 +++--- .../src/services/order/order-service.ts | 17 +++--- .../platforms/pdd/pdd-order-mapper.test.ts | 27 +++++++++ .../platforms/pdd/pdd-order-mapper.ts | 21 ++++--- apps/backend/src/utils/json.test.ts | 55 +++++++++++++++++ apps/backend/src/utils/json.ts | 60 +++++++++++++++++++ 7 files changed, 178 insertions(+), 28 deletions(-) create mode 100644 apps/backend/src/utils/json.test.ts diff --git a/apps/backend/src/repositories/order-item-repo.ts b/apps/backend/src/repositories/order-item-repo.ts index 3834fab1..2fa902a7 100644 --- a/apps/backend/src/repositories/order-item-repo.ts +++ b/apps/backend/src/repositories/order-item-repo.ts @@ -3,6 +3,7 @@ import type { PoolClient, QueryResult } from 'pg' import { query, withTransaction } from '../db/client.js' import type { OrderItemReplaceInput } from '../types/repository/inputs.js' import type { OrderItemRow } from '../types/repository/rows.js' +import { safeJsonStringify } from '../utils/json.js' export type QueryExecutor = (text: string, params?: unknown[]) => Promise> @@ -71,8 +72,8 @@ export async function replaceOrderItemsWithExecutor( update.item.skuCode, update.item.skuName, update.item.quantity, - update.item.specJson || '{}', - update.item.itemSnapshotJson || update.item.specJson || '{}', + safeJsonStringify(update.item.specJson || '{}'), + safeJsonStringify(update.item.itemSnapshotJson || update.item.specJson || '{}'), update.item.updatedAt, update.orderItemId, ], @@ -98,8 +99,8 @@ export async function replaceOrderItemsWithExecutor( create.skuCode, create.skuName, create.quantity, - create.specJson || '{}', - create.itemSnapshotJson || create.specJson || '{}', + safeJsonStringify(create.specJson || '{}'), + safeJsonStringify(create.itemSnapshotJson || create.specJson || '{}'), create.createdAt, create.updatedAt, ], diff --git a/apps/backend/src/repositories/order-repo.ts b/apps/backend/src/repositories/order-repo.ts index f6976f07..3a8b5de9 100644 --- a/apps/backend/src/repositories/order-repo.ts +++ b/apps/backend/src/repositories/order-repo.ts @@ -8,6 +8,7 @@ import type { } from '../types/repository/inputs.js' import type { OrderListQueryResult, OrderListRow, OrderRow } from '../types/repository/rows.js' import { isPlatformOrderIdLookupKeyword } from '../utils/platform-order-id.js' +import { safeJsonStringify, sanitizeWellFormedUnicode } from '../utils/json.js' export type OrderQueryExecutor = ( text: string, @@ -147,12 +148,12 @@ export async function createOrderWithExecutor( input.platformOrderId, input.orderStatus, input.payStatus, - input.buyerId, - input.buyerName, - input.receiverContact, + sanitizeWellFormedUnicode(input.buyerId), + sanitizeWellFormedUnicode(input.buyerName), + sanitizeWellFormedUnicode(input.receiverContact), input.totalAmount, input.currency, - input.rawPayloadJson || '{}', + safeJsonStringify(input.rawPayloadJson || '{}'), input.paidAt || null, input.createdAt, input.updatedAt, @@ -202,12 +203,12 @@ export async function updateOrderWithExecutor( input.shopName, input.orderStatus, input.payStatus, - input.buyerId, - input.buyerName, - input.receiverContact, + sanitizeWellFormedUnicode(input.buyerId), + sanitizeWellFormedUnicode(input.buyerName), + sanitizeWellFormedUnicode(input.receiverContact), input.totalAmount, input.currency, - input.rawPayloadJson || '{}', + safeJsonStringify(input.rawPayloadJson || '{}'), input.paidAt || null, input.updatedAt, Number(orderId), diff --git a/apps/backend/src/services/order/order-service.ts b/apps/backend/src/services/order/order-service.ts index 376ce79a..1c30dc9c 100644 --- a/apps/backend/src/services/order/order-service.ts +++ b/apps/backend/src/services/order/order-service.ts @@ -19,6 +19,7 @@ import { import { nowIso } from '../../utils/time.js' import { logDebug, logInfo, logIntegration, logWarn } from '../../utils/logger.js' import { createHttpError } from '../../utils/http.js' +import { safeJsonStringify, sanitizeWellFormedUnicode } from '../../utils/json.js' import type { OrderItemRow, OrderRow, TaskRow } from '../../types/repository/rows.js' type SourceOrderItem = { @@ -138,12 +139,12 @@ export async function upsertOrderFromSource( platformOrderId: event.platformOrderId, orderStatus: event.orderStatus, payStatus: event.payStatus, - buyerId: event.buyerId, - buyerName: event.buyerName, - receiverContact: event.receiverContact, + buyerId: sanitizeWellFormedUnicode(event.buyerId), + buyerName: sanitizeWellFormedUnicode(event.buyerName), + receiverContact: sanitizeWellFormedUnicode(event.receiverContact), totalAmount: event.totalAmount, currency: event.currency, - rawPayloadJson: JSON.stringify(event.rawPayload), + rawPayloadJson: safeJsonStringify(event.rawPayload), ...(event.paidAt !== undefined ? { paidAt: event.paidAt } : {}), } const mergedPayload = mergeSourceOrderState(existing, basePayload) @@ -172,11 +173,11 @@ export async function upsertOrderFromSource( executor, order.id, resolvedFulfillmentItems.map((item) => ({ - skuCode: item.skuCode, - skuName: item.skuName, + skuCode: sanitizeWellFormedUnicode(item.skuCode), + skuName: sanitizeWellFormedUnicode(item.skuName), quantity: item.quantity, - specJson: JSON.stringify(item.spec || {}), - itemSnapshotJson: JSON.stringify(item.snapshot || item.spec || {}), + specJson: safeJsonStringify(item.spec || {}), + itemSnapshotJson: safeJsonStringify(item.snapshot || item.spec || {}), createdAt: now, updatedAt: now, })), diff --git a/apps/backend/src/services/platforms/pdd/pdd-order-mapper.test.ts b/apps/backend/src/services/platforms/pdd/pdd-order-mapper.test.ts index be2a510a..3e391ad7 100644 --- a/apps/backend/src/services/platforms/pdd/pdd-order-mapper.test.ts +++ b/apps/backend/src/services/platforms/pdd/pdd-order-mapper.test.ts @@ -100,6 +100,33 @@ test('mapPddOrderToSourceEvent prioritizes out_sku_sn as primary matching field' assert.equal((item.spec as Record).spec, '暗潮1钻') }) +test('mapPddOrderToSourceEvent sanitizes lone unicode surrogates and null bytes in buyer nickname/memo', () => { + const pddOrderWithSurrogates: PddRawOrderItem = { + order_sn: '260919-616772441623292', + order_status: 2, + order_status_str: '已发货', + order_amount: 10000, + goods_id: 888, + goods_name: '测试商品', + nickname: '\ud835*VIP\ud83c', // Unpaired lone surrogates + buyer_memo: '游戏ID: 12345\0null', // Null byte + } + + const event = mapPddOrderToSourceEvent(pddOrderWithSurrogates, { + shopId: 'xiangrikui', + shopName: '向日葵网络游戏专营店', + }) + + assert.equal(event.buyerName, '\uFFFD*VIP\uFFFD') + assert.equal(event.buyerId, '\uFFFD*VIP\uFFFD') + assert.equal(event.receiverContact, '游戏ID: 12345null') + + // JSON stringification should be 100% valid without lone surrogates or null bytes + const serialized = JSON.stringify(event.rawPayload) + assert.ok(!serialized.includes('\0')) + assert.doesNotThrow(() => JSON.parse(serialized)) +}) + test('mapPddOrderToSourceEvent reads nested order_goods when top-level goods fields missing', () => { const nestedOnly: PddRawOrderItem = { order_sn: '260915-111', diff --git a/apps/backend/src/services/platforms/pdd/pdd-order-mapper.ts b/apps/backend/src/services/platforms/pdd/pdd-order-mapper.ts index 45aaa6aa..571984b1 100644 --- a/apps/backend/src/services/platforms/pdd/pdd-order-mapper.ts +++ b/apps/backend/src/services/platforms/pdd/pdd-order-mapper.ts @@ -5,6 +5,7 @@ import type { SourceOrderEvent } from '../../order/order-service.js' import type { JsonObject } from '../../../types/json.js' +import { deepSanitizeUnicode, sanitizeWellFormedUnicode } from '../../../utils/json.js' export interface PddRawOrderItem { order_sn: string @@ -180,9 +181,13 @@ export function mapPddOrderToSourceEvent( const externalSkuCode = skuCode const externalSkuName = skuName + const cleanRawOrder = deepSanitizeUnicode(rawOrder) + const cleanNickname = sanitizeWellFormedUnicode(rawOrder.nickname || '') + const cleanBuyerMemo = sanitizeWellFormedUnicode(rawOrder.buyer_memo || '').trim() + // Retain the entire raw snapshot dictionary as requested / 保留原始报文完整快照 const itemSnapshot: JsonObject = { - ...(rawOrder as unknown as JsonObject), + ...(cleanRawOrder as unknown as JsonObject), goodsId: goodsIdStr, goodsName, spec, @@ -191,9 +196,9 @@ export function mapPddOrderToSourceEvent( quantity, orderAmount: Number(rawOrder.order_amount || 0), goodsAmount: Number(rawOrder.goods_amount || 0), - buyerMemo: String(rawOrder.buyer_memo || ''), - mallRemark: String(rawOrder.mall_remark || ''), - nickname: String(rawOrder.nickname || ''), + buyerMemo: cleanBuyerMemo, + mallRemark: sanitizeWellFormedUnicode(rawOrder.mall_remark || ''), + nickname: cleanNickname, orderTime: rawOrder.order_time, confirmTime: rawOrder.confirm_time, } @@ -206,14 +211,14 @@ export function mapPddOrderToSourceEvent( platformOrderId: orderSn, orderStatus, payStatus, - buyerId: String(rawOrder.nickname || ''), - buyerName: String(rawOrder.nickname || ''), + buyerId: cleanNickname, + buyerName: cleanNickname, // Receiver contact is often buyer's recharge game ID / 买家留言通常为充值游戏账号 - receiverContact: String(rawOrder.buyer_memo || '').trim(), + receiverContact: cleanBuyerMemo, totalAmount: Number(rawOrder.order_amount || 0), currency: 'CNY', paidAt, - rawPayload: rawOrder, + rawPayload: cleanRawOrder, items: [ { itemId: goodsIdStr, diff --git a/apps/backend/src/utils/json.test.ts b/apps/backend/src/utils/json.test.ts new file mode 100644 index 00000000..e05837b6 --- /dev/null +++ b/apps/backend/src/utils/json.test.ts @@ -0,0 +1,55 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { + parseJsonObject, + sanitizeWellFormedUnicode, + deepSanitizeUnicode, + safeJsonStringify, +} from './json.js' + +test('sanitizeWellFormedUnicode handles null bytes and lone surrogates', () => { + // Null byte + assert.equal(sanitizeWellFormedUnicode('hello\0world'), 'helloworld') + + // Lone high surrogate \ud835 + const loneSurrogate = 'nickname\ud835test' + const cleaned = sanitizeWellFormedUnicode(loneSurrogate) + assert.equal(cleaned, 'nickname\uFFFDtest') + + // Valid emoji surrogate pair \uD83D\uDE00 (😀) should remain intact + const validEmoji = 'Hello 😀' + assert.equal(sanitizeWellFormedUnicode(validEmoji), 'Hello 😀') +}) + +test('deepSanitizeUnicode recursively cleans objects and arrays', () => { + const input = { + user: 'buyer\ud83c', + nested: { + comment: 'good\0deal', + tags: ['cool\ud835', 'ok'], + }, + } + + const output = deepSanitizeUnicode(input) + assert.deepEqual(output, { + user: 'buyer\uFFFD', + nested: { + comment: 'gooddeal', + tags: ['cool\uFFFD', 'ok'], + }, + }) +}) + +test('safeJsonStringify produces valid JSON safe for postgres jsonb', () => { + const input = { + orderSn: '260919-123456', + nickname: '\ud835\ud83c*buyer', + nullByte: 'abc\0def', + } + + const json = safeJsonStringify(input) + assert.ok(typeof json === 'string') + const parsed = JSON.parse(json) + assert.equal(parsed.nickname, '\uFFFD\uFFFD*buyer') + assert.equal(parsed.nullByte, 'abcdef') +}) diff --git a/apps/backend/src/utils/json.ts b/apps/backend/src/utils/json.ts index 4738ca5d..030a0707 100644 --- a/apps/backend/src/utils/json.ts +++ b/apps/backend/src/utils/json.ts @@ -64,6 +64,66 @@ function shouldPreserveIntegerToken(rawDigits: unknown): boolean { return unsigned > '9007199254740991' } +export function sanitizeWellFormedUnicode(value: unknown): string { + if (value === null || value === undefined) return '' + const str = String(value) + if (!str) return '' + // 1. Remove null bytes (\u0000) which PostgreSQL text/jsonb columns strictly forbid + const withoutNull = str.replace(/\0/g, '') + // 2. Convert lone surrogates into replacement character U+FFFD + const toWellFormedFn = (withoutNull as { toWellFormed?: () => string }).toWellFormed + return typeof toWellFormedFn === 'function' + ? toWellFormedFn.call(withoutNull) + : withoutNull.replace( + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?(value: T): T { + if (value === null || value === undefined) { + return value + } + if (typeof value === 'string') { + return sanitizeWellFormedUnicode(value) as unknown as T + } + if (Array.isArray(value)) { + return value.map((item) => deepSanitizeUnicode(item)) as unknown as T + } + if (isPlainObject(value)) { + const res: Record = {} + for (const [k, v] of Object.entries(value)) { + const cleanKey = sanitizeWellFormedUnicode(k) + res[cleanKey] = deepSanitizeUnicode(v) + } + return res as unknown as T + } + return value +} + +export function safeJsonStringify(value: unknown, fallback = '{}'): string { + if (value === undefined || value === null) { + return fallback + } + if (typeof value === 'string') { + const trimmed = value.trim() + if (!trimmed) return fallback + try { + const parsed = JSON.parse(trimmed) + const sanitized = deepSanitizeUnicode(parsed) + return JSON.stringify(sanitized) + } catch { + return JSON.stringify(sanitizeWellFormedUnicode(value)) + } + } + try { + const sanitized = deepSanitizeUnicode(value) + return JSON.stringify(sanitized) + } catch { + return fallback + } +} + function isPlainObject(value: unknown): value is JsonObject { return Object.prototype.toString.call(value) === '[object Object]' }