Files
affiliate_dash/backend/internal/service/rock_redeem_confirm.go
T
zhaoxing c506c9e2ed fix: 上游限频全局冷却调整为 5 秒
- rockRateLimitCooldown 由 60 秒改为 5 秒,释放校验可快速重试
- 自动轮询仍保留各自的防抖与指数退避,避免持续触发 110001
- 同步更新注释与限频测试中的冷却设置
2026-11-15 14:23:16 +08:00

489 lines
16 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package service
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync/atomic"
"time"
"affiliate_dash/internal/model"
)
const (
rockRedeemPollInterval = 20 * time.Second
rockRedeemBatchSize = 20
rockRedeemCallGap = 2 * time.Second
rockRedeemTimeout = 15 * time.Minute
rockRedeemFirstRetry = 30 * time.Second
rockRedeemMaxRetry = 5 * time.Minute
)
var rockRedeemWorkerRunning atomic.Bool
// RockRedeemConfirmSummary 是一次 pending 确认批次的汇总。
type RockRedeemConfirmSummary struct {
Scanned int `json:"scanned"`
Confirmed int `json:"confirmed"`
Pending int `json:"pending"`
Failed int `json:"failed"`
TimedOut int `json:"timed_out"`
}
// SetRedeemConfirmHook 注册「上游确认发货后同步订单」的回调,由 main 装配 DeliveryService。
func (s *RockService) SetRedeemConfirmHook(hook func(ctx context.Context, order *model.FulfillmentOrder, token *model.RockToken) error) {
s.redeemConfirmHook = hook
}
// RunRedeemConfirmWorker 启动发货结果确认任务:每 20 秒扫描一次「发货确认中」标记。
//
// 频控保护:上游 110001 时全局冷却 5 秒,且每个 Token 使用指数退避,
// 避免把 0666.pro 的公开刷新接口打成持续频控。
func (s *RockService) RunRedeemConfirmWorker(ctx context.Context) {
lastCleanup := time.Now()
lastReconcile := time.Time{}
ticker := time.NewTicker(rockRedeemPollInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if rockUpstreamRateLimitActive() {
continue
}
if _, err := s.ProcessRedeemingTokens(ctx); err != nil && !errors.Is(err, errRockRedeemBusy) {
log.Printf("[Rock] 发货结果确认任务失败: %v", err)
}
if time.Since(lastReconcile) >= 2*time.Minute {
if err := s.ReconcileRockOrders(ctx); err != nil {
log.Printf("[Rock] 订单对账失败: %v", err)
}
lastReconcile = time.Now()
}
if time.Since(lastCleanup) >= 24*time.Hour {
s.CleanupRockDeliveryLogs(ctx)
lastCleanup = time.Now()
}
}
}
}
// ReconcileRockOrders 修复 Token 与订单状态之间的已知不一致:
// 1. Token 已 redeemed,但订单仍 paid/delivering/ship_failed,重新同步订单;
// 2. 订单已 delivered,Token 仍 bound 且已有 redeemed_at,修复 Token 终态;
//
// 历史失败订单和取消冲突不在这里回填;新异常由实际发货/取消流程直接记录。
func (s *RockService) ReconcileRockOrders(ctx context.Context) error {
if s.redeemConfirmHook == nil {
return nil
}
type row struct {
TokenID uint
}
var redeemRows []row
if err := s.db.WithContext(ctx).Raw(`
SELECT t.id AS token_id
FROM rock_tokens t
JOIN fulfillment_orders o ON o.order_no = t.assigned_order_no
WHERE t.deleted_at IS NULL
AND t.status = 'redeemed'
AND o.deleted_at IS NULL
AND o.order_status IN ('paid', 'delivering', 'ship_failed')
ORDER BY t.redeemed_at ASC, t.id ASC
LIMIT 50
`).Scan(&redeemRows).Error; err != nil {
return err
}
for _, item := range redeemRows {
if err := s.reconcileRedeemedOrder(ctx, item.TokenID); err != nil {
log.Printf("[Rock] 对账 redeemed Token %d 失败: %v", item.TokenID, err)
}
}
// 修复历史“订单已交付但 Token 仍 bound”的记录。
var boundRows []row
if err := s.db.WithContext(ctx).Raw(`
SELECT t.id AS token_id
FROM rock_tokens t
JOIN fulfillment_orders o ON o.order_no = t.assigned_order_no
WHERE t.deleted_at IS NULL
AND t.status = 'bound'
AND t.redeemed_at IS NOT NULL
AND o.deleted_at IS NULL
AND o.order_status = 'delivered'
ORDER BY t.id ASC
LIMIT 50
`).Scan(&boundRows).Error; err != nil {
return err
}
for _, item := range boundRows {
var token model.RockToken
if err := s.db.WithContext(ctx).First(&token, item.TokenID).Error; err != nil {
continue
}
oldStatus := token.Status
if err := s.db.WithContext(ctx).Model(&token).Updates(map[string]interface{}{
"status": model.RockStatusRedeemed,
"updated_at": time.Now(),
}).Error; err != nil {
continue
}
writeRockTokenStatusChange(s.db.WithContext(ctx), &token, oldStatus, model.RockStatusRedeemed,
rockTokenSourceRedeem, "订单对账修复:订单已交付", false, nil)
}
return nil
}
func (s *RockService) reconcileRedeemedOrder(ctx context.Context, tokenID uint) error {
var token model.RockToken
if err := s.db.WithContext(ctx).First(&token, tokenID).Error; err != nil {
return err
}
var order model.FulfillmentOrder
if err := s.db.WithContext(ctx).Where("order_no = ?", token.AssignedOrderNo).First(&order).Error; err != nil {
return err
}
if s.redeemConfirmHook == nil {
return nil
}
if err := s.redeemConfirmHook(ctx, &order, &token); err != nil {
_, _ = s.RecordRockIncident(ctx, &token, token.AssignedOrderNo,
model.RockIncidentCategoryOther,
"Token 已发货但订单同步失败: "+err.Error(), 0, "")
return err
}
return nil
}
var errRockRedeemBusy = errors.New("发货结果确认任务正在执行")
// ProcessRedeemingTokens 扫描并确认一批「发货确认中」的 Token。
//
// 判定口径是标记而非状态:status=bound + 已提交兑换 + 尚未拿到履约结果。
// 可被后台任务或测试直接调用。
func (s *RockService) ProcessRedeemingTokens(ctx context.Context) (*RockRedeemConfirmSummary, error) {
if !rockRedeemWorkerRunning.CompareAndSwap(false, true) {
return nil, errRockRedeemBusy
}
defer rockRedeemWorkerRunning.Store(false)
now := time.Now()
var tokens []model.RockToken
if err := s.db.WithContext(ctx).
Where("status = ?", model.RockStatusBound).
Where("redeem_requested_at IS NOT NULL").
Where("redeemed_at IS NULL").
Where("(redeem_next_retry_at IS NULL OR redeem_next_retry_at <= ?)", now).
Order("redeem_next_retry_at ASC").
Limit(rockRedeemBatchSize).
Find(&tokens).Error; err != nil {
return nil, err
}
summary := &RockRedeemConfirmSummary{Scanned: len(tokens)}
for i := range tokens {
if i > 0 {
if rockUpstreamRateLimitActive() {
break
}
select {
case <-ctx.Done():
return summary, ctx.Err()
case <-time.After(rockRedeemCallGap):
}
}
confirmed, timedOut, err := s.confirmRedeemingToken(ctx, &tokens[i])
if err != nil {
summary.Failed++
log.Printf("[Rock] 确认 Token %d 发货结果失败: %v", tokens[i].ID, err)
continue
}
if confirmed {
summary.Confirmed++
continue
}
summary.Pending++
if timedOut {
summary.TimedOut++
}
}
return summary, nil
}
// confirmRedeemingToken 查询一次上游结果并更新本地状态。
func (s *RockService) confirmRedeemingToken(ctx context.Context, token *model.RockToken) (confirmed bool, timedOut bool, err error) {
if token == nil {
return false, false, errors.New("Token 不存在")
}
cfg, err := s.GetConfig(ctx)
if err != nil {
return false, false, err
}
recorder := s.NewDeliveryRecorder(token.AssignedOrderNo, nil, "", "")
recorder.AttachToken(token)
ctx = withRockDeliveryRecorder(ctx, recorder)
ref, err := s.refreshRedeemStatus(ctx, token, cfg.BearerToken)
if err != nil {
if isRockRateLimitError(err) {
markRockUpstreamRateLimited()
}
if retryErr := s.bumpRedeemRetry(ctx, token, err.Error(), false); retryErr != nil {
return false, false, fmt.Errorf("刷新失败且记录重试状态失败: %w", retryErr)
}
if recorder != nil {
recorder.Internal(ctx, "redeem_confirm", "failed", err.Error(), nil)
}
return false, false, err
}
if ref == nil {
if retryErr := s.bumpRedeemRetry(ctx, token, "上游未返回刷新结果", false); retryErr != nil {
return false, false, retryErr
}
return false, false, errors.New("上游未返回刷新结果")
}
now := time.Now()
upstreamStatus := strings.ToLower(strings.TrimSpace(ref.Token.Status))
// 上游明确失效:终止处理并标记异常。
if upstreamStatus == model.RockStatusAbnormal {
if err := s.markRedeemingAbnormal(ctx, token, "上游状态异常(发货确认)", recorder); err != nil {
return false, false, err
}
return false, false, nil
}
deliveryEvidence := rockCurrentCycleDelivery(token, ref)
if deliveryEvidence.Current || deliveryEvidence.Ambiguous {
fulfillID := deliveryEvidence.FulfillID
if fulfillID == 0 {
fulfillID = extractRockFulfillID(ref, rockClaimBaseline(token))
}
redeemedAt := deliveryEvidence.At
if redeemedAt.IsZero() {
redeemedAt = now
}
fromStatus := token.Status
if err := s.db.WithContext(ctx).Unscoped().Model(token).Updates(map[string]interface{}{
"status": model.RockStatusRedeemed,
"upstream_fulfill_id": fulfillID,
"redeemed_at": redeemedAt,
"redeem_checked_at": now,
"redeem_next_retry_at": nil,
"redeem_last_message": "",
"redeem_last_error": "",
"redeem_timeout_warned_at": nil,
"updated_at": now,
}).Error; err != nil {
return false, false, fmt.Errorf("上游确认发货,但更新 Token 终态失败: %w", err)
}
message := "上游确认发货"
if deliveryEvidence.Ambiguous {
message = "上游确认发货,但兑换时间无法核对,已按已发货处理"
}
writeRockTokenStatusChange(s.db.WithContext(ctx), token, fromStatus, model.RockStatusRedeemed,
rockTokenSourceRedeem, message, false, nil)
token.Status = model.RockStatusRedeemed
token.UpstreamFulfillID = fulfillID
token.RedeemedAt = &redeemedAt
if recorder != nil {
recorder.Internal(ctx, "state_change", "success", message, map[string]interface{}{
"upstream_status": ref.Token.Status,
"upstream_redeemed_at": ref.Token.RedeemedAt,
"upstream_fulfill_id": fulfillID,
"ambiguous": deliveryEvidence.Ambiguous,
})
}
s.syncDeliveredOrder(ctx, token, recorder)
return true, false, nil
}
// 仍在官方处理中:保持 bound + 标记,指数退避。
waitingMessage := strings.TrimSpace(ref.Message)
if waitingMessage == "" {
waitingMessage = "官方处理中,请稍后刷新查看结果"
}
if err := s.db.WithContext(ctx).Unscoped().Model(token).Updates(map[string]interface{}{
"redeem_checked_at": now,
"redeem_last_message": waitingMessage,
"redeem_last_error": "",
"updated_at": now,
}).Error; err != nil {
return false, false, fmt.Errorf("更新发货确认状态失败: %w", err)
}
token.RedeemCheckedAt = &now
token.RedeemLastMessage = waitingMessage
attempts := token.RedeemAttempts
if err := s.bumpRedeemRetry(ctx, token, "", true); err != nil {
return false, false, err
}
if recorder != nil {
recorder.Internal(ctx, "redeem_confirm", "pending", waitingMessage, map[string]interface{}{
"upstream_status": ref.Token.Status,
"attempts": attempts,
})
}
// 超过等待上限仍未确认:保持处理中,仅告警一次(不做状态流转)。
if token.RedeemRequestedAt != nil && now.Sub(*token.RedeemRequestedAt) >= rockRedeemTimeout && token.RedeemTimeoutWarnedAt == nil {
warnAt := now
_ = s.db.WithContext(ctx).Unscoped().Model(token).Updates(map[string]interface{}{
"redeem_timeout_warned_at": warnAt,
"redeem_last_message": waitingMessage + "(已超过 15 分钟未确认,已告警)",
}).Error
if recorder != nil {
recorder.Internal(ctx, "redeem_timeout", "pending", "上游超过 15 分钟未确认发货", nil)
}
return false, true, nil
}
return false, false, nil
}
func (s *RockService) refreshRedeemStatus(ctx context.Context, token *model.RockToken, bearerToken string) (*RockRefreshResponse, error) {
var firstErr error
if strings.TrimSpace(token.RedeemCode) != "" {
ref, err := s.client.RefreshRedeemBinding(ctx, token.RedeemCode)
if err == nil {
return ref, nil
}
firstErr = err
// 频控时不要再打管理端接口,直接交给退避。
if isRockRateLimitError(err) {
return nil, err
}
}
if strings.TrimSpace(token.UpstreamTokenID) != "" {
ref, err := s.client.RefreshToken(ctx, bearerToken, token.UpstreamTokenID)
if err == nil {
return ref, nil
}
if firstErr == nil {
firstErr = err
}
return nil, err
}
if firstErr != nil {
return nil, firstErr
}
return nil, errors.New("缺少兑换码与上游 Token ID,无法确认发货结果")
}
func (s *RockService) bumpRedeemRetry(ctx context.Context, token *model.RockToken, lastError string, countAttempt bool) error {
if token == nil {
return errors.New("Token 不存在")
}
attempts := token.RedeemAttempts
if countAttempt {
attempts++
}
delay := rockRedeemFirstRetry
switch {
case attempts <= 1:
delay = rockRedeemFirstRetry
case attempts == 2:
delay = time.Minute
case attempts == 3:
delay = 2 * time.Minute
default:
delay = rockRedeemMaxRetry
}
if token.RedeemRequestedAt != nil && time.Since(*token.RedeemRequestedAt) >= rockRedeemTimeout {
delay = rockRedeemMaxRetry
}
next := time.Now().Add(delay)
updates := map[string]interface{}{
"redeem_attempts": attempts,
"redeem_next_retry_at": next,
"updated_at": time.Now(),
}
if strings.TrimSpace(lastError) != "" {
updates["redeem_last_error"] = lastError
}
if err := s.db.WithContext(ctx).Unscoped().Model(token).Updates(updates).Error; err != nil {
return err
}
token.RedeemAttempts = attempts
token.RedeemNextRetryAt = &next
return nil
}
func (s *RockService) markRedeemingAbnormal(ctx context.Context, token *model.RockToken, reason string, recorder *RockDeliveryRecorder) error {
fromStatus := token.Status
if err := s.db.WithContext(ctx).Unscoped().Model(token).Updates(map[string]interface{}{
"status": model.RockStatusAbnormal,
"redeem_last_error": reason,
"redeem_next_retry_at": nil,
"redeem_timeout_warned_at": nil,
"remark": appendRockRemark(token.Remark, reason),
"updated_at": time.Now(),
}).Error; err != nil {
return err
}
writeRockTokenStatusChange(s.db.WithContext(ctx), token, fromStatus, model.RockStatusAbnormal,
rockTokenSourceRedeem, reason, false, nil)
token.Status = model.RockStatusAbnormal
if recorder != nil {
recorder.Internal(ctx, "state_change", "failed", reason, map[string]interface{}{
"from_status": fromStatus,
"to_status": model.RockStatusAbnormal,
})
}
return nil
}
func (s *RockService) syncDeliveredOrder(ctx context.Context, token *model.RockToken, recorder *RockDeliveryRecorder) {
if s.redeemConfirmHook == nil || token == nil || strings.TrimSpace(token.AssignedOrderNo) == "" {
return
}
var order model.FulfillmentOrder
if err := s.db.WithContext(ctx).Where("order_no = ?", token.AssignedOrderNo).First(&order).Error; err != nil {
if recorder != nil {
recorder.Internal(ctx, "order_update", "failed", "读取订单失败: "+err.Error(), map[string]interface{}{
"order_no": token.AssignedOrderNo,
})
}
return
}
var lastErr error
for attempt := 0; attempt < 3; attempt++ {
if err := s.redeemConfirmHook(ctx, &order, token); err == nil {
if recorder != nil {
recorder.Internal(ctx, "order_update", "success", "订单已同步为已交付", map[string]interface{}{
"order_no": order.OrderNo,
})
}
return
} else {
lastErr = err
time.Sleep(time.Second)
}
}
if recorder != nil {
recorder.Internal(ctx, "order_update", "failed", fmt.Sprintf("同步订单失败: %v", lastErr), map[string]interface{}{
"order_no": order.OrderNo,
})
}
log.Printf("[Rock] Token %d 已确认发货,但订单 %s 同步失败: %v", token.ID, token.AssignedOrderNo, lastErr)
}
func appendRockRemark(existing, reason string) string {
existing = strings.TrimSpace(existing)
reason = strings.TrimSpace(reason)
if reason == "" {
return existing
}
if existing == "" {
return reason
}
if strings.Contains(existing, reason) {
return existing
}
return existing + "\n" + reason
}