- rockRateLimitCooldown 由 60 秒改为 5 秒,释放校验可快速重试 - 自动轮询仍保留各自的防抖与指数退避,避免持续触发 110001 - 同步更新注释与限频测试中的冷却设置
489 lines
16 KiB
Go
489 lines
16 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"strings"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"affiliate_dash/internal/model"
|
||
)
|
||
|
||
const (
|
||
rockRedeemPollInterval = 20 * time.Second
|
||
rockRedeemBatchSize = 20
|
||
rockRedeemCallGap = 2 * time.Second
|
||
rockRedeemTimeout = 15 * time.Minute
|
||
rockRedeemFirstRetry = 30 * time.Second
|
||
rockRedeemMaxRetry = 5 * time.Minute
|
||
)
|
||
|
||
var rockRedeemWorkerRunning atomic.Bool
|
||
|
||
// RockRedeemConfirmSummary 是一次 pending 确认批次的汇总。
|
||
type RockRedeemConfirmSummary struct {
|
||
Scanned int `json:"scanned"`
|
||
Confirmed int `json:"confirmed"`
|
||
Pending int `json:"pending"`
|
||
Failed int `json:"failed"`
|
||
TimedOut int `json:"timed_out"`
|
||
}
|
||
|
||
// SetRedeemConfirmHook 注册「上游确认发货后同步订单」的回调,由 main 装配 DeliveryService。
|
||
func (s *RockService) SetRedeemConfirmHook(hook func(ctx context.Context, order *model.FulfillmentOrder, token *model.RockToken) error) {
|
||
s.redeemConfirmHook = hook
|
||
}
|
||
|
||
// RunRedeemConfirmWorker 启动发货结果确认任务:每 20 秒扫描一次「发货确认中」标记。
|
||
//
|
||
// 频控保护:上游 110001 时全局冷却 5 秒,且每个 Token 使用指数退避,
|
||
// 避免把 0666.pro 的公开刷新接口打成持续频控。
|
||
func (s *RockService) RunRedeemConfirmWorker(ctx context.Context) {
|
||
lastCleanup := time.Now()
|
||
lastReconcile := time.Time{}
|
||
ticker := time.NewTicker(rockRedeemPollInterval)
|
||
defer ticker.Stop()
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
if rockUpstreamRateLimitActive() {
|
||
continue
|
||
}
|
||
if _, err := s.ProcessRedeemingTokens(ctx); err != nil && !errors.Is(err, errRockRedeemBusy) {
|
||
log.Printf("[Rock] 发货结果确认任务失败: %v", err)
|
||
}
|
||
if time.Since(lastReconcile) >= 2*time.Minute {
|
||
if err := s.ReconcileRockOrders(ctx); err != nil {
|
||
log.Printf("[Rock] 订单对账失败: %v", err)
|
||
}
|
||
lastReconcile = time.Now()
|
||
}
|
||
if time.Since(lastCleanup) >= 24*time.Hour {
|
||
s.CleanupRockDeliveryLogs(ctx)
|
||
lastCleanup = time.Now()
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ReconcileRockOrders 修复 Token 与订单状态之间的已知不一致:
|
||
// 1. Token 已 redeemed,但订单仍 paid/delivering/ship_failed,重新同步订单;
|
||
// 2. 订单已 delivered,Token 仍 bound 且已有 redeemed_at,修复 Token 终态;
|
||
//
|
||
// 历史失败订单和取消冲突不在这里回填;新异常由实际发货/取消流程直接记录。
|
||
func (s *RockService) ReconcileRockOrders(ctx context.Context) error {
|
||
if s.redeemConfirmHook == nil {
|
||
return nil
|
||
}
|
||
type row struct {
|
||
TokenID uint
|
||
}
|
||
var redeemRows []row
|
||
if err := s.db.WithContext(ctx).Raw(`
|
||
SELECT t.id AS token_id
|
||
FROM rock_tokens t
|
||
JOIN fulfillment_orders o ON o.order_no = t.assigned_order_no
|
||
WHERE t.deleted_at IS NULL
|
||
AND t.status = 'redeemed'
|
||
AND o.deleted_at IS NULL
|
||
AND o.order_status IN ('paid', 'delivering', 'ship_failed')
|
||
ORDER BY t.redeemed_at ASC, t.id ASC
|
||
LIMIT 50
|
||
`).Scan(&redeemRows).Error; err != nil {
|
||
return err
|
||
}
|
||
for _, item := range redeemRows {
|
||
if err := s.reconcileRedeemedOrder(ctx, item.TokenID); err != nil {
|
||
log.Printf("[Rock] 对账 redeemed Token %d 失败: %v", item.TokenID, err)
|
||
}
|
||
}
|
||
|
||
// 修复历史“订单已交付但 Token 仍 bound”的记录。
|
||
var boundRows []row
|
||
if err := s.db.WithContext(ctx).Raw(`
|
||
SELECT t.id AS token_id
|
||
FROM rock_tokens t
|
||
JOIN fulfillment_orders o ON o.order_no = t.assigned_order_no
|
||
WHERE t.deleted_at IS NULL
|
||
AND t.status = 'bound'
|
||
AND t.redeemed_at IS NOT NULL
|
||
AND o.deleted_at IS NULL
|
||
AND o.order_status = 'delivered'
|
||
ORDER BY t.id ASC
|
||
LIMIT 50
|
||
`).Scan(&boundRows).Error; err != nil {
|
||
return err
|
||
}
|
||
for _, item := range boundRows {
|
||
var token model.RockToken
|
||
if err := s.db.WithContext(ctx).First(&token, item.TokenID).Error; err != nil {
|
||
continue
|
||
}
|
||
oldStatus := token.Status
|
||
if err := s.db.WithContext(ctx).Model(&token).Updates(map[string]interface{}{
|
||
"status": model.RockStatusRedeemed,
|
||
"updated_at": time.Now(),
|
||
}).Error; err != nil {
|
||
continue
|
||
}
|
||
writeRockTokenStatusChange(s.db.WithContext(ctx), &token, oldStatus, model.RockStatusRedeemed,
|
||
rockTokenSourceRedeem, "订单对账修复:订单已交付", false, nil)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *RockService) reconcileRedeemedOrder(ctx context.Context, tokenID uint) error {
|
||
var token model.RockToken
|
||
if err := s.db.WithContext(ctx).First(&token, tokenID).Error; err != nil {
|
||
return err
|
||
}
|
||
var order model.FulfillmentOrder
|
||
if err := s.db.WithContext(ctx).Where("order_no = ?", token.AssignedOrderNo).First(&order).Error; err != nil {
|
||
return err
|
||
}
|
||
if s.redeemConfirmHook == nil {
|
||
return nil
|
||
}
|
||
if err := s.redeemConfirmHook(ctx, &order, &token); err != nil {
|
||
_, _ = s.RecordRockIncident(ctx, &token, token.AssignedOrderNo,
|
||
model.RockIncidentCategoryOther,
|
||
"Token 已发货但订单同步失败: "+err.Error(), 0, "")
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
var errRockRedeemBusy = errors.New("发货结果确认任务正在执行")
|
||
|
||
// ProcessRedeemingTokens 扫描并确认一批「发货确认中」的 Token。
|
||
//
|
||
// 判定口径是标记而非状态:status=bound + 已提交兑换 + 尚未拿到履约结果。
|
||
// 可被后台任务或测试直接调用。
|
||
func (s *RockService) ProcessRedeemingTokens(ctx context.Context) (*RockRedeemConfirmSummary, error) {
|
||
if !rockRedeemWorkerRunning.CompareAndSwap(false, true) {
|
||
return nil, errRockRedeemBusy
|
||
}
|
||
defer rockRedeemWorkerRunning.Store(false)
|
||
|
||
now := time.Now()
|
||
var tokens []model.RockToken
|
||
if err := s.db.WithContext(ctx).
|
||
Where("status = ?", model.RockStatusBound).
|
||
Where("redeem_requested_at IS NOT NULL").
|
||
Where("redeemed_at IS NULL").
|
||
Where("(redeem_next_retry_at IS NULL OR redeem_next_retry_at <= ?)", now).
|
||
Order("redeem_next_retry_at ASC").
|
||
Limit(rockRedeemBatchSize).
|
||
Find(&tokens).Error; err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
summary := &RockRedeemConfirmSummary{Scanned: len(tokens)}
|
||
for i := range tokens {
|
||
if i > 0 {
|
||
if rockUpstreamRateLimitActive() {
|
||
break
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return summary, ctx.Err()
|
||
case <-time.After(rockRedeemCallGap):
|
||
}
|
||
}
|
||
confirmed, timedOut, err := s.confirmRedeemingToken(ctx, &tokens[i])
|
||
if err != nil {
|
||
summary.Failed++
|
||
log.Printf("[Rock] 确认 Token %d 发货结果失败: %v", tokens[i].ID, err)
|
||
continue
|
||
}
|
||
if confirmed {
|
||
summary.Confirmed++
|
||
continue
|
||
}
|
||
summary.Pending++
|
||
if timedOut {
|
||
summary.TimedOut++
|
||
}
|
||
}
|
||
return summary, nil
|
||
}
|
||
|
||
// confirmRedeemingToken 查询一次上游结果并更新本地状态。
|
||
func (s *RockService) confirmRedeemingToken(ctx context.Context, token *model.RockToken) (confirmed bool, timedOut bool, err error) {
|
||
if token == nil {
|
||
return false, false, errors.New("Token 不存在")
|
||
}
|
||
cfg, err := s.GetConfig(ctx)
|
||
if err != nil {
|
||
return false, false, err
|
||
}
|
||
|
||
recorder := s.NewDeliveryRecorder(token.AssignedOrderNo, nil, "", "")
|
||
recorder.AttachToken(token)
|
||
ctx = withRockDeliveryRecorder(ctx, recorder)
|
||
|
||
ref, err := s.refreshRedeemStatus(ctx, token, cfg.BearerToken)
|
||
if err != nil {
|
||
if isRockRateLimitError(err) {
|
||
markRockUpstreamRateLimited()
|
||
}
|
||
if retryErr := s.bumpRedeemRetry(ctx, token, err.Error(), false); retryErr != nil {
|
||
return false, false, fmt.Errorf("刷新失败且记录重试状态失败: %w", retryErr)
|
||
}
|
||
if recorder != nil {
|
||
recorder.Internal(ctx, "redeem_confirm", "failed", err.Error(), nil)
|
||
}
|
||
return false, false, err
|
||
}
|
||
if ref == nil {
|
||
if retryErr := s.bumpRedeemRetry(ctx, token, "上游未返回刷新结果", false); retryErr != nil {
|
||
return false, false, retryErr
|
||
}
|
||
return false, false, errors.New("上游未返回刷新结果")
|
||
}
|
||
|
||
now := time.Now()
|
||
upstreamStatus := strings.ToLower(strings.TrimSpace(ref.Token.Status))
|
||
|
||
// 上游明确失效:终止处理并标记异常。
|
||
if upstreamStatus == model.RockStatusAbnormal {
|
||
if err := s.markRedeemingAbnormal(ctx, token, "上游状态异常(发货确认)", recorder); err != nil {
|
||
return false, false, err
|
||
}
|
||
return false, false, nil
|
||
}
|
||
|
||
deliveryEvidence := rockCurrentCycleDelivery(token, ref)
|
||
if deliveryEvidence.Current || deliveryEvidence.Ambiguous {
|
||
fulfillID := deliveryEvidence.FulfillID
|
||
if fulfillID == 0 {
|
||
fulfillID = extractRockFulfillID(ref, rockClaimBaseline(token))
|
||
}
|
||
redeemedAt := deliveryEvidence.At
|
||
if redeemedAt.IsZero() {
|
||
redeemedAt = now
|
||
}
|
||
fromStatus := token.Status
|
||
if err := s.db.WithContext(ctx).Unscoped().Model(token).Updates(map[string]interface{}{
|
||
"status": model.RockStatusRedeemed,
|
||
"upstream_fulfill_id": fulfillID,
|
||
"redeemed_at": redeemedAt,
|
||
"redeem_checked_at": now,
|
||
"redeem_next_retry_at": nil,
|
||
"redeem_last_message": "",
|
||
"redeem_last_error": "",
|
||
"redeem_timeout_warned_at": nil,
|
||
"updated_at": now,
|
||
}).Error; err != nil {
|
||
return false, false, fmt.Errorf("上游确认发货,但更新 Token 终态失败: %w", err)
|
||
}
|
||
message := "上游确认发货"
|
||
if deliveryEvidence.Ambiguous {
|
||
message = "上游确认发货,但兑换时间无法核对,已按已发货处理"
|
||
}
|
||
writeRockTokenStatusChange(s.db.WithContext(ctx), token, fromStatus, model.RockStatusRedeemed,
|
||
rockTokenSourceRedeem, message, false, nil)
|
||
token.Status = model.RockStatusRedeemed
|
||
token.UpstreamFulfillID = fulfillID
|
||
token.RedeemedAt = &redeemedAt
|
||
if recorder != nil {
|
||
recorder.Internal(ctx, "state_change", "success", message, map[string]interface{}{
|
||
"upstream_status": ref.Token.Status,
|
||
"upstream_redeemed_at": ref.Token.RedeemedAt,
|
||
"upstream_fulfill_id": fulfillID,
|
||
"ambiguous": deliveryEvidence.Ambiguous,
|
||
})
|
||
}
|
||
s.syncDeliveredOrder(ctx, token, recorder)
|
||
return true, false, nil
|
||
}
|
||
|
||
// 仍在官方处理中:保持 bound + 标记,指数退避。
|
||
waitingMessage := strings.TrimSpace(ref.Message)
|
||
if waitingMessage == "" {
|
||
waitingMessage = "官方处理中,请稍后刷新查看结果"
|
||
}
|
||
if err := s.db.WithContext(ctx).Unscoped().Model(token).Updates(map[string]interface{}{
|
||
"redeem_checked_at": now,
|
||
"redeem_last_message": waitingMessage,
|
||
"redeem_last_error": "",
|
||
"updated_at": now,
|
||
}).Error; err != nil {
|
||
return false, false, fmt.Errorf("更新发货确认状态失败: %w", err)
|
||
}
|
||
token.RedeemCheckedAt = &now
|
||
token.RedeemLastMessage = waitingMessage
|
||
|
||
attempts := token.RedeemAttempts
|
||
if err := s.bumpRedeemRetry(ctx, token, "", true); err != nil {
|
||
return false, false, err
|
||
}
|
||
if recorder != nil {
|
||
recorder.Internal(ctx, "redeem_confirm", "pending", waitingMessage, map[string]interface{}{
|
||
"upstream_status": ref.Token.Status,
|
||
"attempts": attempts,
|
||
})
|
||
}
|
||
|
||
// 超过等待上限仍未确认:保持处理中,仅告警一次(不做状态流转)。
|
||
if token.RedeemRequestedAt != nil && now.Sub(*token.RedeemRequestedAt) >= rockRedeemTimeout && token.RedeemTimeoutWarnedAt == nil {
|
||
warnAt := now
|
||
_ = s.db.WithContext(ctx).Unscoped().Model(token).Updates(map[string]interface{}{
|
||
"redeem_timeout_warned_at": warnAt,
|
||
"redeem_last_message": waitingMessage + "(已超过 15 分钟未确认,已告警)",
|
||
}).Error
|
||
if recorder != nil {
|
||
recorder.Internal(ctx, "redeem_timeout", "pending", "上游超过 15 分钟未确认发货", nil)
|
||
}
|
||
return false, true, nil
|
||
}
|
||
return false, false, nil
|
||
}
|
||
|
||
func (s *RockService) refreshRedeemStatus(ctx context.Context, token *model.RockToken, bearerToken string) (*RockRefreshResponse, error) {
|
||
var firstErr error
|
||
if strings.TrimSpace(token.RedeemCode) != "" {
|
||
ref, err := s.client.RefreshRedeemBinding(ctx, token.RedeemCode)
|
||
if err == nil {
|
||
return ref, nil
|
||
}
|
||
firstErr = err
|
||
// 频控时不要再打管理端接口,直接交给退避。
|
||
if isRockRateLimitError(err) {
|
||
return nil, err
|
||
}
|
||
}
|
||
if strings.TrimSpace(token.UpstreamTokenID) != "" {
|
||
ref, err := s.client.RefreshToken(ctx, bearerToken, token.UpstreamTokenID)
|
||
if err == nil {
|
||
return ref, nil
|
||
}
|
||
if firstErr == nil {
|
||
firstErr = err
|
||
}
|
||
return nil, err
|
||
}
|
||
if firstErr != nil {
|
||
return nil, firstErr
|
||
}
|
||
return nil, errors.New("缺少兑换码与上游 Token ID,无法确认发货结果")
|
||
}
|
||
|
||
func (s *RockService) bumpRedeemRetry(ctx context.Context, token *model.RockToken, lastError string, countAttempt bool) error {
|
||
if token == nil {
|
||
return errors.New("Token 不存在")
|
||
}
|
||
attempts := token.RedeemAttempts
|
||
if countAttempt {
|
||
attempts++
|
||
}
|
||
delay := rockRedeemFirstRetry
|
||
switch {
|
||
case attempts <= 1:
|
||
delay = rockRedeemFirstRetry
|
||
case attempts == 2:
|
||
delay = time.Minute
|
||
case attempts == 3:
|
||
delay = 2 * time.Minute
|
||
default:
|
||
delay = rockRedeemMaxRetry
|
||
}
|
||
if token.RedeemRequestedAt != nil && time.Since(*token.RedeemRequestedAt) >= rockRedeemTimeout {
|
||
delay = rockRedeemMaxRetry
|
||
}
|
||
next := time.Now().Add(delay)
|
||
updates := map[string]interface{}{
|
||
"redeem_attempts": attempts,
|
||
"redeem_next_retry_at": next,
|
||
"updated_at": time.Now(),
|
||
}
|
||
if strings.TrimSpace(lastError) != "" {
|
||
updates["redeem_last_error"] = lastError
|
||
}
|
||
if err := s.db.WithContext(ctx).Unscoped().Model(token).Updates(updates).Error; err != nil {
|
||
return err
|
||
}
|
||
token.RedeemAttempts = attempts
|
||
token.RedeemNextRetryAt = &next
|
||
return nil
|
||
}
|
||
|
||
func (s *RockService) markRedeemingAbnormal(ctx context.Context, token *model.RockToken, reason string, recorder *RockDeliveryRecorder) error {
|
||
fromStatus := token.Status
|
||
if err := s.db.WithContext(ctx).Unscoped().Model(token).Updates(map[string]interface{}{
|
||
"status": model.RockStatusAbnormal,
|
||
"redeem_last_error": reason,
|
||
"redeem_next_retry_at": nil,
|
||
"redeem_timeout_warned_at": nil,
|
||
"remark": appendRockRemark(token.Remark, reason),
|
||
"updated_at": time.Now(),
|
||
}).Error; err != nil {
|
||
return err
|
||
}
|
||
writeRockTokenStatusChange(s.db.WithContext(ctx), token, fromStatus, model.RockStatusAbnormal,
|
||
rockTokenSourceRedeem, reason, false, nil)
|
||
token.Status = model.RockStatusAbnormal
|
||
if recorder != nil {
|
||
recorder.Internal(ctx, "state_change", "failed", reason, map[string]interface{}{
|
||
"from_status": fromStatus,
|
||
"to_status": model.RockStatusAbnormal,
|
||
})
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *RockService) syncDeliveredOrder(ctx context.Context, token *model.RockToken, recorder *RockDeliveryRecorder) {
|
||
if s.redeemConfirmHook == nil || token == nil || strings.TrimSpace(token.AssignedOrderNo) == "" {
|
||
return
|
||
}
|
||
var order model.FulfillmentOrder
|
||
if err := s.db.WithContext(ctx).Where("order_no = ?", token.AssignedOrderNo).First(&order).Error; err != nil {
|
||
if recorder != nil {
|
||
recorder.Internal(ctx, "order_update", "failed", "读取订单失败: "+err.Error(), map[string]interface{}{
|
||
"order_no": token.AssignedOrderNo,
|
||
})
|
||
}
|
||
return
|
||
}
|
||
var lastErr error
|
||
for attempt := 0; attempt < 3; attempt++ {
|
||
if err := s.redeemConfirmHook(ctx, &order, token); err == nil {
|
||
if recorder != nil {
|
||
recorder.Internal(ctx, "order_update", "success", "订单已同步为已交付", map[string]interface{}{
|
||
"order_no": order.OrderNo,
|
||
})
|
||
}
|
||
return
|
||
} else {
|
||
lastErr = err
|
||
time.Sleep(time.Second)
|
||
}
|
||
}
|
||
if recorder != nil {
|
||
recorder.Internal(ctx, "order_update", "failed", fmt.Sprintf("同步订单失败: %v", lastErr), map[string]interface{}{
|
||
"order_no": order.OrderNo,
|
||
})
|
||
}
|
||
log.Printf("[Rock] Token %d 已确认发货,但订单 %s 同步失败: %v", token.ID, token.AssignedOrderNo, lastErr)
|
||
}
|
||
|
||
func appendRockRemark(existing, reason string) string {
|
||
existing = strings.TrimSpace(existing)
|
||
reason = strings.TrimSpace(reason)
|
||
if reason == "" {
|
||
return existing
|
||
}
|
||
if existing == "" {
|
||
return reason
|
||
}
|
||
if strings.Contains(existing, reason) {
|
||
return existing
|
||
}
|
||
return existing + "\n" + reason
|
||
}
|