fix(order-service): 修复包含 Emoji 和孤立代理字符的订单在 PostgreSQL jsonb 下报错导致死循环重试进单的问题
1. 在 utils/json.ts 新增 sanitizeWellFormedUnicode、deepSanitizeUnicode 和 safeJsonStringify,自动清理空字节 (\0) 并将畸变孤立 UTF-16 代理字符 (Lone Surrogates) 安全转换为规范字符,杜绝 PostgreSQL jsonb 解析错误; 2. 拼多多订单映射器与通用进单服务 (order-service) 全面接入 Unicode 深度清洗与安全序列化,确保买家昵称/留言中的各类特殊 Emoji 和生僻字符平滑落库; 3. 数据库仓储层 (order-repo / order-item-repo) 针对 raw_payload_json、spec_json、item_snapshot_json 等 jsonb 字段增加兜底安全序列化保护; 4. 补充 utils/json.test.ts 与 pdd-order-mapper.test.ts 单元测试,覆盖畸变字符与空字节场景。
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import {
|
||||
parseJsonObject,
|
||||
sanitizeWellFormedUnicode,
|
||||
deepSanitizeUnicode,
|
||||
safeJsonStringify,
|
||||
} from './json.js'
|
||||
|
||||
test('sanitizeWellFormedUnicode handles null bytes and lone surrogates', () => {
|
||||
// Null byte
|
||||
assert.equal(sanitizeWellFormedUnicode('hello\0world'), 'helloworld')
|
||||
|
||||
// Lone high surrogate \ud835
|
||||
const loneSurrogate = 'nickname\ud835test'
|
||||
const cleaned = sanitizeWellFormedUnicode(loneSurrogate)
|
||||
assert.equal(cleaned, 'nickname\uFFFDtest')
|
||||
|
||||
// Valid emoji surrogate pair \uD83D\uDE00 (😀) should remain intact
|
||||
const validEmoji = 'Hello 😀'
|
||||
assert.equal(sanitizeWellFormedUnicode(validEmoji), 'Hello 😀')
|
||||
})
|
||||
|
||||
test('deepSanitizeUnicode recursively cleans objects and arrays', () => {
|
||||
const input = {
|
||||
user: 'buyer\ud83c',
|
||||
nested: {
|
||||
comment: 'good\0deal',
|
||||
tags: ['cool\ud835', 'ok'],
|
||||
},
|
||||
}
|
||||
|
||||
const output = deepSanitizeUnicode(input)
|
||||
assert.deepEqual(output, {
|
||||
user: 'buyer\uFFFD',
|
||||
nested: {
|
||||
comment: 'gooddeal',
|
||||
tags: ['cool\uFFFD', 'ok'],
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test('safeJsonStringify produces valid JSON safe for postgres jsonb', () => {
|
||||
const input = {
|
||||
orderSn: '260919-123456',
|
||||
nickname: '\ud835\ud83c*buyer',
|
||||
nullByte: 'abc\0def',
|
||||
}
|
||||
|
||||
const json = safeJsonStringify(input)
|
||||
assert.ok(typeof json === 'string')
|
||||
const parsed = JSON.parse(json)
|
||||
assert.equal(parsed.nickname, '\uFFFD\uFFFD*buyer')
|
||||
assert.equal(parsed.nullByte, 'abcdef')
|
||||
})
|
||||
@@ -64,6 +64,66 @@ function shouldPreserveIntegerToken(rawDigits: unknown): boolean {
|
||||
return unsigned > '9007199254740991'
|
||||
}
|
||||
|
||||
export function sanitizeWellFormedUnicode(value: unknown): string {
|
||||
if (value === null || value === undefined) return ''
|
||||
const str = String(value)
|
||||
if (!str) return ''
|
||||
// 1. Remove null bytes (\u0000) which PostgreSQL text/jsonb columns strictly forbid
|
||||
const withoutNull = str.replace(/\0/g, '')
|
||||
// 2. Convert lone surrogates into replacement character U+FFFD
|
||||
const toWellFormedFn = (withoutNull as { toWellFormed?: () => string }).toWellFormed
|
||||
return typeof toWellFormedFn === 'function'
|
||||
? toWellFormedFn.call(withoutNull)
|
||||
: withoutNull.replace(
|
||||
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
|
||||
'\uFFFD',
|
||||
)
|
||||
}
|
||||
|
||||
export function deepSanitizeUnicode<T>(value: T): T {
|
||||
if (value === null || value === undefined) {
|
||||
return value
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
return sanitizeWellFormedUnicode(value) as unknown as T
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => deepSanitizeUnicode(item)) as unknown as T
|
||||
}
|
||||
if (isPlainObject(value)) {
|
||||
const res: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) {
|
||||
const cleanKey = sanitizeWellFormedUnicode(k)
|
||||
res[cleanKey] = deepSanitizeUnicode(v)
|
||||
}
|
||||
return res as unknown as T
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
export function safeJsonStringify(value: unknown, fallback = '{}'): string {
|
||||
if (value === undefined || value === null) {
|
||||
return fallback
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim()
|
||||
if (!trimmed) return fallback
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed)
|
||||
const sanitized = deepSanitizeUnicode(parsed)
|
||||
return JSON.stringify(sanitized)
|
||||
} catch {
|
||||
return JSON.stringify(sanitizeWellFormedUnicode(value))
|
||||
}
|
||||
}
|
||||
try {
|
||||
const sanitized = deepSanitizeUnicode(value)
|
||||
return JSON.stringify(sanitized)
|
||||
} catch {
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
function isPlainObject(value: unknown): value is JsonObject {
|
||||
return Object.prototype.toString.call(value) === '[object Object]'
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user