feat(pdd): 店铺连续同步失败 3 次后熔断，停止自动重试
背景：生产上「木塔网游专营店」Cookie 已过期，每约 24 秒重试一次、每轮刷一条 ERROR 日志，
持续数天。Cookie 失效不会自愈，继续重试只刷日志、白白占用同步周期。
行为：
- 失败 1~2 次：累计计数，继续重试；
- 第 3 次：熔断，打一条显式 ERROR（含"更新 Cookie 会自动恢复"提示）；
- 熔断后：自动同步直接跳过，不再刷错误日志；
- 恢复：更新 Cookie（后台编辑或浏览器插件）自动解除；手动同步成功一次亦可。
熔断状态持久化在店铺配置里，容器重启后依然生效。
改动：
- pdd-source-config-service：阈值常量 + consecutiveFailures/syncSuspendedAt/
syncSuspendedReason 持久化字段 + 纯函数 resolvePddStoreFailureState /
applyPddStoreBreakerState / isPddStoreSyncSuspended / shouldResetPddSyncBreaker。
- pdd-sync-service：syncAllPddStores 跳过熔断店铺并上报 suspendedCount；
syncStoreOrders 失败记录、成功清零（健康店铺不产生额外配置写）。
- pdd-order-sync-job：上报熔断数，且存在熔断店铺时强制留在 INFO 级
（否则"全部店铺已停同步"会因新单为 0 被降级成 debug 而不可见）。
- 前端加「已熔断 · 停止重试」标签与类型字段。
顺带修掉一个隐患：saveAdminPddStore 收的是 Partial 载荷，normalizePddStoreItem
会为缺失字段产出 undefined，导致一次「只改店铺名」的保存就把熔断洗掉。
由 applyPddStoreBreakerState 回填既有状态解决。
验证：9 项熔断单测（含局部保存不洗状态、Cookie 变更解除熔断）。
This commit is contained in:
yml2213
@@ -15,6 +15,13 @@ import { PurePddAntiContent } from './pdd-anti-content.js'
|
||||
|
||||
const PDD_SOURCE_FILE_PATH = path.join(PROJECT_ROOT, 'data', 'pdd-source.json')
|
||||
|
||||
/**
|
||||
* 连续同步失败达到该次数后熔断，停止对该店铺的自动重试。
|
||||
* 说明：Cookie 失效的店铺不会自愈，继续每轮重试只会持续刷错误日志并白白占用同步周期，
|
||||
* 因此连续失败到阈值后由人工介入（更新 Cookie 会自动恢复，见 savePddStore）。
|
||||
*/
|
||||
export const PDD_SYNC_MAX_CONSECUTIVE_FAILURES = 3
|
||||
|
||||
export interface PddStoreConfigItem {
|
||||
shopId: string
|
||||
shopName: string
|
||||
@@ -29,6 +36,12 @@ export interface PddStoreConfigItem {
|
||||
totalOrders?: number | undefined
|
||||
syncToken?: string | undefined
|
||||
lastExtensionSyncAt?: string | undefined
|
||||
/** 连续同步失败次数；成功一次即清零。 */
|
||||
consecutiveFailures?: number | undefined
|
||||
/** 熔断时间（ISO）。存在即表示已停止自动重试，直到 Cookie 更新或手动同步成功。 */
|
||||
syncSuspendedAt?: string | undefined
|
||||
/** 触发熔断时的最近一次失败原因。 */
|
||||
syncSuspendedReason?: string | undefined
|
||||
}
|
||||
|
||||
export interface PddSourcesConfig {
|
||||
@@ -101,15 +114,53 @@ export async function savePddStore(
|
||||
const config = getPddSourcesConfig()
|
||||
const normalizedItem = normalizePddStoreItem({ ...data, shopId: key })
|
||||
const existingIndex = config.stores.findIndex((s) => s.shopId === key)
|
||||
const existing = existingIndex >= 0 ? config.stores[existingIndex] : undefined
|
||||
const merged = applyPddStoreBreakerState(existing, normalizedItem, data)
|
||||
|
||||
if (existingIndex >= 0) {
|
||||
config.stores[existingIndex] = normalizedItem
|
||||
config.stores[existingIndex] = merged
|
||||
} else {
|
||||
config.stores.push(normalizedItem)
|
||||
config.stores.push(merged)
|
||||
}
|
||||
|
||||
await savePddSourcesConfig(config)
|
||||
return normalizedItem
|
||||
return merged
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存店铺时决定熔断字段的去留（纯函数，便于单测）：
|
||||
* 1. 局部更新（只改名字/开关等）必须沿用既有熔断状态——normalizePddStoreItem 会为
|
||||
* 缺失字段产出 undefined，若不回填，一次「只改店铺名」的保存就会把熔断洗掉，
|
||||
* 失效店铺重新开始每 20 秒刷错误日志；
|
||||
* 2. Cookie 发生变化视为人工介入，清空熔断并重新计数。
|
||||
*/
|
||||
export function applyPddStoreBreakerState(
|
||||
existing: PddStoreConfigItem | null | undefined,
|
||||
incoming: PddStoreConfigItem,
|
||||
payload: JsonObject = {},
|
||||
): PddStoreConfigItem {
|
||||
if (!existing) {
|
||||
return incoming
|
||||
}
|
||||
|
||||
const next: PddStoreConfigItem = { ...incoming }
|
||||
if (!hasOwn(payload, 'consecutiveFailures')) {
|
||||
next.consecutiveFailures = existing.consecutiveFailures
|
||||
}
|
||||
if (!hasOwn(payload, 'syncSuspendedAt')) {
|
||||
next.syncSuspendedAt = existing.syncSuspendedAt
|
||||
}
|
||||
if (!hasOwn(payload, 'syncSuspendedReason')) {
|
||||
next.syncSuspendedReason = existing.syncSuspendedReason
|
||||
}
|
||||
|
||||
if (shouldResetPddSyncBreaker(existing, next)) {
|
||||
next.consecutiveFailures = 0
|
||||
next.syncSuspendedAt = undefined
|
||||
next.syncSuspendedReason = undefined
|
||||
}
|
||||
|
||||
return next
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -148,6 +199,116 @@ export async function updatePddStoreCheckStatus(
|
||||
return store
|
||||
}
|
||||
|
||||
/** 店铺是否已因连续失败被熔断（暂停自动重试）。 */
|
||||
export function isPddStoreSyncSuspended(store: PddStoreConfigItem | null | undefined): boolean {
|
||||
return Boolean(store && store.syncSuspendedAt)
|
||||
}
|
||||
|
||||
/**
|
||||
* Cookie 是否发生了变化——变化即视为人工介入，需要解除熔断。
|
||||
* 只在原先确实有 Cookie 且内容不同时成立，避免首次录入时误判。
|
||||
*/
|
||||
export function shouldResetPddSyncBreaker(
|
||||
existing: Pick<PddStoreConfigItem, 'cookie'> | null | undefined,
|
||||
next: Pick<PddStoreConfigItem, 'cookie'>,
|
||||
): boolean {
|
||||
if (!existing) return false
|
||||
const before = String(existing.cookie || '').trim()
|
||||
const after = String(next.cookie || '').trim()
|
||||
return Boolean(before) && before !== after
|
||||
}
|
||||
|
||||
/**
|
||||
* 纯函数：由当前熔断状态推出「再失败一次」之后的状态。
|
||||
* 抽出来是为了让阈值行为可单测，不必依赖数据库。
|
||||
*/
|
||||
export function resolvePddStoreFailureState(
|
||||
current: Pick<
|
||||
PddStoreConfigItem,
|
||||
'consecutiveFailures' | 'syncSuspendedAt' | 'syncSuspendedReason'
|
||||
>,
|
||||
message: string,
|
||||
nowIso: string = new Date().toISOString(),
|
||||
): {
|
||||
consecutiveFailures: number
|
||||
syncSuspendedAt: string | undefined
|
||||
syncSuspendedReason: string | undefined
|
||||
suspended: boolean
|
||||
newlySuspended: boolean
|
||||
} {
|
||||
const failures = Math.max(0, Number(current.consecutiveFailures || 0)) + 1
|
||||
const reachedThreshold = failures >= PDD_SYNC_MAX_CONSECUTIVE_FAILURES
|
||||
const alreadySuspended = Boolean(current.syncSuspendedAt)
|
||||
const reason = String(message || '').trim() || '连续同步失败'
|
||||
|
||||
return {
|
||||
consecutiveFailures: failures,
|
||||
// 已经熔断过的店铺保留原始熔断时间，避免每次失败都刷新时间戳。
|
||||
syncSuspendedAt: alreadySuspended
|
||||
? String(current.syncSuspendedAt)
|
||||
: reachedThreshold
|
||||
? nowIso
|
||||
: undefined,
|
||||
syncSuspendedReason: alreadySuspended
|
||||
? current.syncSuspendedReason || reason
|
||||
: reachedThreshold
|
||||
? reason
|
||||
: undefined,
|
||||
suspended: alreadySuspended || reachedThreshold,
|
||||
newlySuspended: !alreadySuspended && reachedThreshold,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录一次同步失败，达到阈值即熔断该店铺的自动重试。
|
||||
* 熔断状态持久化在店铺配置里，容器重启后依然生效。
|
||||
*/
|
||||
export async function recordPddStoreSyncFailure(
|
||||
shopId: unknown,
|
||||
message: string,
|
||||
): Promise<{ failures: number; suspended: boolean; newlySuspended: boolean } | null> {
|
||||
const key = String(shopId || '').trim()
|
||||
if (!key) return null
|
||||
|
||||
const config = getPddSourcesConfig()
|
||||
const store = config.stores.find((s) => s.shopId === key)
|
||||
if (!store) return null
|
||||
|
||||
const next = resolvePddStoreFailureState(store, message)
|
||||
store.consecutiveFailures = next.consecutiveFailures
|
||||
store.syncSuspendedAt = next.syncSuspendedAt
|
||||
store.syncSuspendedReason = next.syncSuspendedReason
|
||||
await savePddSourcesConfig(config)
|
||||
|
||||
return {
|
||||
failures: next.consecutiveFailures,
|
||||
suspended: next.suspended,
|
||||
newlySuspended: next.newlySuspended,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步成功后清零连续失败计数并解除熔断。
|
||||
* 健康店铺（无计数、未熔断）直接返回，不产生多余的配置写。
|
||||
*/
|
||||
export async function clearPddStoreSyncFailures(shopId: unknown): Promise<boolean> {
|
||||
const key = String(shopId || '').trim()
|
||||
if (!key) return false
|
||||
|
||||
const config = getPddSourcesConfig()
|
||||
const store = config.stores.find((s) => s.shopId === key)
|
||||
if (!store) return false
|
||||
if (!store.consecutiveFailures && !store.syncSuspendedAt && !store.syncSuspendedReason) {
|
||||
return false
|
||||
}
|
||||
|
||||
store.consecutiveFailures = 0
|
||||
store.syncSuspendedAt = undefined
|
||||
store.syncSuspendedReason = undefined
|
||||
await savePddSourcesConfig(config)
|
||||
return true
|
||||
}
|
||||
|
||||
export async function deletePddStore(shopId: unknown): Promise<PddSourcesConfig> {
|
||||
const key = String(shopId || '').trim()
|
||||
if (!key) {
|
||||
@@ -261,6 +422,11 @@ export function normalizePddStoreItem(rawValue: unknown): PddStoreConfigItem {
|
||||
: undefined
|
||||
const syncToken = String(source.syncToken || '').trim() || undefined
|
||||
const lastExtensionSyncAt = String(source.lastExtensionSyncAt || '').trim() || undefined
|
||||
const consecutiveFailures = Number.isInteger(Number(source.consecutiveFailures))
|
||||
? Math.max(0, Number(source.consecutiveFailures))
|
||||
: undefined
|
||||
const syncSuspendedAt = String(source.syncSuspendedAt || '').trim() || undefined
|
||||
const syncSuspendedReason = String(source.syncSuspendedReason || '').trim() || undefined
|
||||
|
||||
return {
|
||||
shopId,
|
||||
@@ -276,6 +442,9 @@ export function normalizePddStoreItem(rawValue: unknown): PddStoreConfigItem {
|
||||
totalOrders,
|
||||
syncToken,
|
||||
lastExtensionSyncAt,
|
||||
consecutiveFailures,
|
||||
syncSuspendedAt,
|
||||
syncSuspendedReason,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -329,3 +498,8 @@ function createDefaultPddSourcesConfig(): PddSourcesConfig {
|
||||
function isPlainObject(value: unknown): value is JsonObject {
|
||||
return Object.prototype.toString.call(value) === '[object Object]'
|
||||
}
|
||||
|
||||
/** 判断调用方是否显式传了某个字段——用于区分「没传」和「显式清空」。 */
|
||||
function hasOwn(source: JsonObject, key: string): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(source, key)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
|
||||
import { APP_CONFIG_KEYS } from '../../../config/app-config-keys.js'
|
||||
import { setAppConfigCacheEntry } from '../../config/app-config-store.js'
|
||||
import { syncAllPddStores } from './pdd-sync-service.js'
|
||||
import {
|
||||
PDD_SYNC_MAX_CONSECUTIVE_FAILURES,
|
||||
applyPddStoreBreakerState,
|
||||
isPddStoreSyncSuspended,
|
||||
normalizePddStoreItem,
|
||||
resolvePddStoreFailureState,
|
||||
shouldResetPddSyncBreaker,
|
||||
type PddStoreConfigItem,
|
||||
} from './pdd-source-config-service.js'
|
||||
|
||||
test('连续失败未达阈值时只累计计数，不熔断', () => {
|
||||
let state = { consecutiveFailures: 0, syncSuspendedAt: undefined, syncSuspendedReason: undefined }
|
||||
|
||||
for (let i = 1; i < PDD_SYNC_MAX_CONSECUTIVE_FAILURES; i++) {
|
||||
const next = resolvePddStoreFailureState(state, `第 ${i} 次失败`)
|
||||
assert.equal(next.consecutiveFailures, i)
|
||||
assert.equal(next.suspended, false, `第 ${i} 次失败不应熔断`)
|
||||
assert.equal(next.newlySuspended, false)
|
||||
assert.equal(next.syncSuspendedAt, undefined)
|
||||
state = next
|
||||
}
|
||||
})
|
||||
|
||||
test('连续失败达到阈值即熔断，并记录原因', () => {
|
||||
const next = resolvePddStoreFailureState(
|
||||
{ consecutiveFailures: PDD_SYNC_MAX_CONSECUTIVE_FAILURES - 1 },
|
||||
'PDD API Error [43001]: 会话已过期',
|
||||
'2026-10-03T00:00:00.000Z',
|
||||
)
|
||||
|
||||
assert.equal(next.consecutiveFailures, PDD_SYNC_MAX_CONSECUTIVE_FAILURES)
|
||||
assert.equal(next.suspended, true)
|
||||
assert.equal(next.newlySuspended, true, '首次触发熔断要能让调用方打一次显眼日志')
|
||||
assert.equal(next.syncSuspendedAt, '2026-10-03T00:00:00.000Z')
|
||||
assert.match(String(next.syncSuspendedReason), /43001/)
|
||||
})
|
||||
|
||||
test('已熔断的店铺再次失败不会刷新熔断时间，也不再重复报新熔断', () => {
|
||||
const first = resolvePddStoreFailureState(
|
||||
{ consecutiveFailures: 2 },
|
||||
'会话已过期',
|
||||
'2026-10-03T00:00:00.000Z',
|
||||
)
|
||||
const second = resolvePddStoreFailureState(first, '会话已过期', '2026-10-03T09:30:00.000Z')
|
||||
|
||||
assert.equal(second.suspended, true)
|
||||
assert.equal(second.newlySuspended, false)
|
||||
assert.equal(second.syncSuspendedAt, first.syncSuspendedAt, '熔断时间必须保持首次触发那一刻')
|
||||
assert.equal(second.consecutiveFailures, 4)
|
||||
})
|
||||
|
||||
test('熔断原因缺失时回落到默认文案', () => {
|
||||
const next = resolvePddStoreFailureState({ consecutiveFailures: 2 }, ' ')
|
||||
assert.equal(next.syncSuspendedReason, '连续同步失败')
|
||||
})
|
||||
|
||||
test('isPddStoreSyncSuspended 只认熔断时间戳', () => {
|
||||
const base: PddStoreConfigItem = {
|
||||
shopId: 's1',
|
||||
shopName: '店铺1',
|
||||
cookie: 'a=1',
|
||||
userAgent: 'Mozilla/5.0',
|
||||
enabled: true,
|
||||
}
|
||||
|
||||
assert.equal(isPddStoreSyncSuspended(null), false)
|
||||
assert.equal(isPddStoreSyncSuspended(undefined), false)
|
||||
assert.equal(isPddStoreSyncSuspended({ ...base, consecutiveFailures: 5 }), false)
|
||||
assert.equal(
|
||||
isPddStoreSyncSuspended({ ...base, syncSuspendedAt: '2026-10-03T00:00:00.000Z' }),
|
||||
true,
|
||||
)
|
||||
})
|
||||
|
||||
test('更新 Cookie 视为人工介入，需要解除熔断', () => {
|
||||
assert.equal(shouldResetPddSyncBreaker({ cookie: 'old=1' }, { cookie: 'new=2' }), true)
|
||||
assert.equal(shouldResetPddSyncBreaker({ cookie: 'same=1' }, { cookie: 'same=1' }), false)
|
||||
// 首次录入（原先没有 Cookie）不算人工恢复
|
||||
assert.equal(shouldResetPddSyncBreaker({ cookie: '' }, { cookie: 'new=2' }), false)
|
||||
assert.equal(shouldResetPddSyncBreaker(null, { cookie: 'new=2' }), false)
|
||||
})
|
||||
|
||||
test('normalizePddStoreItem 保留熔断字段，避免保存配置时把状态洗掉', () => {
|
||||
const store = normalizePddStoreItem({
|
||||
shopId: 's1',
|
||||
shopName: '店铺1',
|
||||
cookie: 'a=1',
|
||||
consecutiveFailures: 3,
|
||||
syncSuspendedAt: '2026-10-03T00:00:00.000Z',
|
||||
syncSuspendedReason: '会话已过期',
|
||||
})
|
||||
|
||||
assert.equal(store.consecutiveFailures, 3)
|
||||
assert.equal(store.syncSuspendedAt, '2026-10-03T00:00:00.000Z')
|
||||
assert.equal(store.syncSuspendedReason, '会话已过期')
|
||||
|
||||
const negative = normalizePddStoreItem({ shopId: 's2', consecutiveFailures: -5 })
|
||||
assert.equal(negative.consecutiveFailures, 0, '负数计数要归零而不是原样带出')
|
||||
})
|
||||
|
||||
test('局部保存不得洗掉熔断状态，换 Cookie 才解除', () => {
|
||||
const existing: PddStoreConfigItem = {
|
||||
shopId: 's1',
|
||||
shopName: '旧名字',
|
||||
cookie: 'old=1',
|
||||
userAgent: 'Mozilla/5.0',
|
||||
enabled: true,
|
||||
consecutiveFailures: 3,
|
||||
syncSuspendedAt: '2026-10-03T00:00:00.000Z',
|
||||
syncSuspendedReason: '会话已过期',
|
||||
}
|
||||
|
||||
// 只改名字：normalize 会把熔断字段变成 undefined，必须由回填逻辑保住
|
||||
const renamed = applyPddStoreBreakerState(
|
||||
existing,
|
||||
normalizePddStoreItem({ ...existing, shopName: '新名字' }),
|
||||
{ shopName: '新名字' },
|
||||
)
|
||||
assert.equal(renamed.shopName, '新名字')
|
||||
assert.equal(renamed.consecutiveFailures, 3)
|
||||
assert.equal(renamed.syncSuspendedAt, '2026-10-03T00:00:00.000Z')
|
||||
assert.equal(renamed.syncSuspendedReason, '会话已过期')
|
||||
|
||||
// 换了 Cookie：视为人工恢复
|
||||
const recovered = applyPddStoreBreakerState(
|
||||
existing,
|
||||
normalizePddStoreItem({ ...existing, cookie: 'new=2' }),
|
||||
{ cookie: 'new=2' },
|
||||
)
|
||||
assert.equal(recovered.consecutiveFailures, 0)
|
||||
assert.equal(recovered.syncSuspendedAt, undefined)
|
||||
assert.equal(recovered.syncSuspendedReason, undefined)
|
||||
|
||||
// 新店铺直接透传
|
||||
const created = applyPddStoreBreakerState(
|
||||
null,
|
||||
normalizePddStoreItem({ shopId: 's2', cookie: 'a=1' }),
|
||||
{},
|
||||
)
|
||||
assert.equal(created.shopId, 's2')
|
||||
assert.equal(created.syncSuspendedAt, undefined)
|
||||
})
|
||||
|
||||
test('已熔断店铺不参与自动重试，且不计入失败', async () => {
|
||||
setAppConfigCacheEntry(APP_CONFIG_KEYS.pddSource, {
|
||||
enabled: true,
|
||||
stores: [
|
||||
{
|
||||
shopId: 'suspended_store',
|
||||
shopName: '熔断店铺',
|
||||
cookie: 'pdd_user_id=1;',
|
||||
enabled: true,
|
||||
consecutiveFailures: 3,
|
||||
syncSuspendedAt: '2026-10-03T00:00:00.000Z',
|
||||
syncSuspendedReason: 'PDD API Error [43001]: 会话已过期',
|
||||
},
|
||||
// 没有 Cookie 的店铺本来就不参与
|
||||
{ shopId: 'no_cookie_store', shopName: '无Cookie店铺', cookie: '', enabled: true },
|
||||
// 手工停用的店铺也不计入熔断数
|
||||
{
|
||||
shopId: 'disabled_store',
|
||||
shopName: '停用店铺',
|
||||
cookie: 'pdd_user_id=2;',
|
||||
enabled: false,
|
||||
syncSuspendedAt: '2026-10-03T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const result = await syncAllPddStores({})
|
||||
|
||||
assert.equal(result.enabled, true)
|
||||
assert.equal(result.suspendedCount, 1, '只有启用且带 Cookie 的熔断店铺才计入')
|
||||
assert.equal(result.syncedCount, 0, '熔断店铺不得进入同步循环')
|
||||
assert.equal(result.results.length, 0, '不应为熔断店铺产生任何同步结果')
|
||||
})
|
||||
@@ -14,7 +14,10 @@
|
||||
import { PddClient, type QueryOrdersOptions } from './pdd-client.js'
|
||||
import { mapPddOrderToSourceEvent, type PddRawOrderItem } from './pdd-order-mapper.js'
|
||||
import {
|
||||
clearPddStoreSyncFailures,
|
||||
getPddSourcesConfig,
|
||||
isPddStoreSyncSuspended,
|
||||
recordPddStoreSyncFailure,
|
||||
updatePddStoreCheckStatus,
|
||||
type PddStoreConfigItem,
|
||||
} from './pdd-source-config-service.js'
|
||||
@@ -60,6 +63,8 @@ export interface SyncAllStoresResult {
|
||||
enabled: boolean
|
||||
storeCount: number
|
||||
syncedCount: number
|
||||
/** 因连续失败被熔断、本轮跳过自动重试的店铺数（需人工更新 Cookie 恢复）。 */
|
||||
suspendedCount: number
|
||||
results: SyncStoreResult[]
|
||||
}
|
||||
|
||||
@@ -303,6 +308,9 @@ export async function syncStoreOrders(
|
||||
totalOrders: totalReported,
|
||||
}).catch(() => null)
|
||||
|
||||
// 成功一次即清零连续失败计数并解除熔断（健康店铺不会产生额外配置写）。
|
||||
await clearPddStoreSyncFailures(shopId).catch(() => false)
|
||||
|
||||
const storeDoneLog =
|
||||
totalNewRawOrders > 0 || processRes.failedCount > 0 || pagesFetched > 1 ? logInfo : logDebug
|
||||
storeDoneLog('[pdd-sync]', `店铺 ${shopName} 同步完成 (共拉取 ${pagesFetched} 页)`, {
|
||||
@@ -329,11 +337,35 @@ export async function syncStoreOrders(
|
||||
}
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err)
|
||||
logError('[pdd-sync]', `店铺 ${shopName} 拉取订单异常`, {
|
||||
shopId,
|
||||
error: errMsg,
|
||||
pagesFetched,
|
||||
})
|
||||
// 记录连续失败；达到阈值即熔断，此后该店铺不再参与自动重试。
|
||||
const breaker = await recordPddStoreSyncFailure(shopId, errMsg).catch(() => null)
|
||||
|
||||
if (breaker?.newlySuspended) {
|
||||
logError(
|
||||
'[pdd-sync]',
|
||||
`店铺 ${shopName} 连续 ${breaker.failures} 次同步失败，已停止自动重试`,
|
||||
{
|
||||
shopId,
|
||||
error: errMsg,
|
||||
consecutiveFailures: breaker.failures,
|
||||
hint: '更新 Cookie 会自动恢复；也可在后台手动触发一次同步',
|
||||
},
|
||||
)
|
||||
} else if (breaker?.suspended) {
|
||||
// 已熔断的店铺只在手动触发时才会走到这里，降级为 warn 避免刷 ERROR。
|
||||
logWarn('[pdd-sync]', `店铺 ${shopName} 同步失败（已熔断，自动重试保持暂停）`, {
|
||||
shopId,
|
||||
error: errMsg,
|
||||
consecutiveFailures: breaker.failures,
|
||||
})
|
||||
} else {
|
||||
logError('[pdd-sync]', `店铺 ${shopName} 拉取订单异常`, {
|
||||
shopId,
|
||||
error: errMsg,
|
||||
pagesFetched,
|
||||
consecutiveFailures: breaker?.failures,
|
||||
})
|
||||
}
|
||||
|
||||
if (isPddCookieExpiredError(errMsg)) {
|
||||
await updatePddStoreCheckStatus(shopId, {
|
||||
@@ -378,20 +410,37 @@ export async function syncAllPddStores(
|
||||
enabled: false,
|
||||
storeCount: config.stores.length,
|
||||
syncedCount: 0,
|
||||
suspendedCount: 0,
|
||||
results: [],
|
||||
}
|
||||
}
|
||||
|
||||
const activeStores = config.stores.filter((s) => s.enabled !== false && Boolean(s.cookie?.trim()))
|
||||
// 已熔断的店铺（连续失败达阈值）不再参与自动重试：Cookie 失效不会自愈，
|
||||
// 继续每轮重试只会持续刷 ERROR 日志并拖慢整轮同步。更新 Cookie 会自动恢复。
|
||||
const suspendedStores = activeStores.filter((store) => isPddStoreSyncSuspended(store))
|
||||
const runnableStores = activeStores.filter((store) => !isPddStoreSyncSuspended(store))
|
||||
const results: SyncStoreResult[] = []
|
||||
|
||||
for (let i = 0; i < activeStores.length; i++) {
|
||||
const store = activeStores[i]!
|
||||
if (suspendedStores.length > 0) {
|
||||
logWarn('[pdd-sync]', `跳过 ${suspendedStores.length} 个已熔断店铺的自动重试`, {
|
||||
stores: suspendedStores.map((store) => ({
|
||||
shopId: store.shopId,
|
||||
shopName: store.shopName,
|
||||
consecutiveFailures: store.consecutiveFailures,
|
||||
suspendedAt: store.syncSuspendedAt,
|
||||
reason: store.syncSuspendedReason,
|
||||
})),
|
||||
})
|
||||
}
|
||||
|
||||
for (let i = 0; i < runnableStores.length; i++) {
|
||||
const store = runnableStores[i]!
|
||||
const res = await syncStoreOrders(store, options)
|
||||
results.push(res)
|
||||
|
||||
// 店铺之间间隔 500ms，避免并发峰值
|
||||
if (i + 1 < activeStores.length) {
|
||||
if (i + 1 < runnableStores.length) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
}
|
||||
}
|
||||
@@ -399,7 +448,8 @@ export async function syncAllPddStores(
|
||||
return {
|
||||
enabled: true,
|
||||
storeCount: config.stores.length,
|
||||
syncedCount: activeStores.length,
|
||||
syncedCount: runnableStores.length,
|
||||
suspendedCount: suspendedStores.length,
|
||||
results,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ export async function runPddOrderSyncJob(job: JsonObject) {
|
||||
checkedCount: 0,
|
||||
okCount: 0,
|
||||
failedCount: 0,
|
||||
suspendedCount: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,10 +42,13 @@ export async function runPddOrderSyncJob(job: JsonObject) {
|
||||
const totalFetched = syncResult.results.reduce((acc, r) => acc + r.fetchedCount, 0)
|
||||
const totalNewOrders = syncResult.results.reduce((acc, r) => acc + r.newOrderCount, 0)
|
||||
|
||||
const jobLog = totalNewOrders > 0 || failedCount > 0 ? logInfo : logDebug
|
||||
// 有店铺被熔断时必须留在 info 级：否则"全部店铺已停止同步"会因新单为 0 被降级成 debug 而不可见。
|
||||
const jobLog =
|
||||
totalNewOrders > 0 || failedCount > 0 || syncResult.suspendedCount > 0 ? logInfo : logDebug
|
||||
jobLog('[pdd-order-sync-job]', '拼多多定时拉单执行完成', {
|
||||
storeCount: syncResult.storeCount,
|
||||
syncedCount: syncResult.syncedCount,
|
||||
suspendedCount: syncResult.suspendedCount,
|
||||
okCount,
|
||||
failedCount,
|
||||
totalFetched,
|
||||
@@ -55,11 +59,12 @@ export async function runPddOrderSyncJob(job: JsonObject) {
|
||||
return {
|
||||
ok: failedCount === 0,
|
||||
status: failedCount > 0 ? (okCount > 0 ? 'partial_error' : 'failed') : 'ok',
|
||||
message: `已同步 ${syncResult.syncedCount} 个店铺，拉取 ${totalFetched} 单，新单 ${totalNewOrders}，入库/更新 ${totalUpserted} 单${failedCount > 0 ? `，失败 ${failedCount} 个店铺` : ''}`,
|
||||
message: `已同步 ${syncResult.syncedCount} 个店铺，拉取 ${totalFetched} 单，新单 ${totalNewOrders}，入库/更新 ${totalUpserted} 单${failedCount > 0 ? `，失败 ${failedCount} 个店铺` : ''}${syncResult.suspendedCount > 0 ? `，${syncResult.suspendedCount} 个店铺已熔断待更新 Cookie` : ''}`,
|
||||
accountCount: syncResult.storeCount,
|
||||
checkedCount: syncResult.syncedCount,
|
||||
okCount,
|
||||
failedCount,
|
||||
suspendedCount: syncResult.suspendedCount,
|
||||
asset: totalNewOrders,
|
||||
totalUpserted,
|
||||
totalNewOrders,
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
SafetyCertificateOutlined,
|
||||
SaveOutlined,
|
||||
ShopOutlined,
|
||||
StopOutlined,
|
||||
ThunderboltOutlined,
|
||||
} from '@ant-design/icons'
|
||||
import {
|
||||
@@ -541,12 +542,25 @@ export function PddPlatformPanel() {
|
||||
const totalOrders = testRes ? testRes.totalOrders : record.totalOrders
|
||||
const timeDisplay = testRes?.testedAt || formatDisplayTime(record.lastCheckedAt)
|
||||
|
||||
// 连续失败已熔断：自动重试已经停了。必须显式提示，否则客服会以为只是
|
||||
// "暂时失效"，不知道新订单已经不再进来了。
|
||||
const suspendedTag = record.syncSuspendedAt ? (
|
||||
<Tooltip
|
||||
title={`连续失败 ${record.consecutiveFailures ?? 3} 次后已停止自动重试（${formatDisplayTime(record.syncSuspendedAt)}）。原因：${record.syncSuspendedReason || '未知'}。更新 Cookie 后会自动恢复。`}
|
||||
>
|
||||
<Tag icon={<StopOutlined />} color="volcano" style={{ cursor: 'pointer', margin: 0 }}>
|
||||
已熔断 · 停止重试
|
||||
</Tag>
|
||||
</Tooltip>
|
||||
) : null
|
||||
|
||||
if (isValid === true) {
|
||||
return (
|
||||
<Space direction="vertical" size={2}>
|
||||
<Tag icon={<CheckCircleOutlined />} color="success">
|
||||
有效 (在售: {totalOrders ?? '-'})
|
||||
</Tag>
|
||||
{suspendedTag}
|
||||
{timeDisplay && (
|
||||
<Text type="secondary" style={{ fontSize: '11px' }}>
|
||||
{timeDisplay} 校验
|
||||
@@ -569,6 +583,7 @@ export function PddPlatformPanel() {
|
||||
失效 (已告警)
|
||||
</Tag>
|
||||
</Tooltip>
|
||||
{suspendedTag}
|
||||
{timeDisplay && (
|
||||
<Text type="secondary" style={{ fontSize: '11px' }}>
|
||||
{timeDisplay} 校验
|
||||
@@ -583,7 +598,12 @@ export function PddPlatformPanel() {
|
||||
)
|
||||
}
|
||||
|
||||
return <Badge status="default" text={<Text type="secondary">未校验</Text>} />
|
||||
return (
|
||||
<Space direction="vertical" size={2}>
|
||||
<Badge status="default" text={<Text type="secondary">未校验</Text>} />
|
||||
{suspendedTag}
|
||||
</Space>
|
||||
)
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -17,6 +17,12 @@ export interface AdminPddStoreItem {
|
||||
totalOrders?: number
|
||||
syncToken?: string
|
||||
lastExtensionSyncAt?: string
|
||||
/** 连续同步失败次数；成功一次即清零。 */
|
||||
consecutiveFailures?: number
|
||||
/** 熔断时间（ISO）。存在即表示已停止自动重试，更新 Cookie 后自动恢复。 */
|
||||
syncSuspendedAt?: string
|
||||
/** 触发熔断时的最近一次失败原因。 */
|
||||
syncSuspendedReason?: string
|
||||
}
|
||||
|
||||
export interface AdminPddStoreSyncTokenResponse {
|
||||
@@ -57,5 +63,7 @@ export interface AdminPddSyncAllResponse {
|
||||
enabled: boolean
|
||||
storeCount: number
|
||||
syncedCount: number
|
||||
/** 因连续失败被熔断、本轮跳过自动重试的店铺数。 */
|
||||
suspendedCount?: number
|
||||
results: AdminPddSyncResult[]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user