Files
dygou_go/internal/seckill/throttle_backoff_test.go
yml2213 288109de2f 按 10-23 三机抢购日志复盘修正重试节奏与状态口径,发布0.3.8
10-23 三台机器 13 个实例抢同一次 18:00 的精英手册,把当天日志翻完发现
四个问题都指向"请求发得太猛、发得太早、结论分得太粗":

- 被限流的账号仍按 100ms 节奏猛打。单实例锁单 950~1650 次(约 20 QPS)时
  18~24 个账号被"您点击太快",而锁单仅 148 次的实例 30 个账号一次全中——
  可见"点击太快/活动火爆"是账号级封禁,继续打只会加深封禁。
- 定时器提前约 100ms 触发,想用这点提前量抵消 RTT,结果 148 次请求
  被明确拒绝:"不在时间范围内 不能兑换"。它们全部落在斗鱼估算时间
  -0.026s ~ +0.172s,也就是活动还没开闸就冲进去了。
- 积分基线写盘用 os.Rename 覆盖旧文件,Windows 上会报 Access is denied,
  三机日志里出现 281 次"积分基线写入失败",基线没落盘导致 pc-1 四个实例
  0 条核对记录,且基线为 0 时会误报"积分未见减少"。
- "已达本次活动兑换上限"被记成账号级终止失败。这其实是"该账号在本次活动
  里已经换到过了",10-23 日志里这些账号紧接着就报了"积分已扣减,兑换确认"。

- 限流改为单账号阶梯降频:新增 throttleBackoff,第 1 次等 1~2s、第 2 次
  2~3s、第 3 次 3~4s……上限 5s,区间内随机取值避免同步醒来。throttleStreak
  独立计数,非限流的失败会把它归零——只有"连续被限流"才越等越久。
  每个账号跑在自己的 goroutine 里,降频只压它自己的下一次尝试,
  不占用并发额度、不影响其它账号的节奏。
- 触发时刻改为精确命中:dynamicAdvance 返回 0,不再提前。另加兜底守卫——
  服务端回"不在时间范围内"时该次尝试不计失败、不消耗 attempts/failStreak、
  不触发降频,用 50~150ms 快速重试直到活动真正开始。结果集里的
  "失败|不在时间范围内…"同步改写成"等待活动开始",避免界面闪红。
- 积分基线改异步写盘:flushCh 队列 + 后台协程串行写,抢购收尾不再等写盘;
  新增 Flush 供需要确定落盘的场合。另按 build tag 分平台解决 Windows
  os.Rename 不能覆盖已有文件的问题(replace_file_unix/windows.go)。
- "已达兑换上限"单列:新增 outcomeLimitReached,结果集写"已抢到|"前缀,
  汇总行加"已达兑换上限=N",既不计入失败也不混进"账号终止"。
- 账号不做全局去重:同一账号可以买多个皮肤,不同实例也可以抢不同皮肤。
- 新增测试 throttle_backoff_test.go(12 个用例):降频阶梯递增、单账号级
  隔离、非限流失败重置阶梯、兑换上限不计失败且立即收手、精确命中不提前、
  活动未开始不触发降频且不记失败、Flush 同步落盘。
- 三个原有测试因降频语义变化需要调整时间尺度:新增 speedUpThrottleBackoff
  只压缩阶梯的时间尺度,不改变阶梯结构。
2026-10-24 12:27:21 +08:00

398 lines
14 KiB
Go

package seckill
import (
"net/http"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"douyu_go/internal/logger"
"douyu_go/internal/types"
)
// ========== 限流阶梯式降频 ==========
// TestThrottleBackoffEscalates 锁住"被限流后逐级拉长等待"这条设计:
// 第 1 次 1~2s、第 2 次 2~3s、第 3 次 3~4s……上限 5s。
//
// 依据 2026-10-23 三机日志:单实例锁单 950~1650 次时 18~24 个账号被限流,
// 而锁单仅 148 次的实例 30 个账号一次全中——猛打只会加深账号级封禁。
func TestThrottleBackoffEscalates(t *testing.T) {
cases := []struct {
streak int
min time.Duration
max time.Duration
}{
{1, 1 * time.Second, 2 * time.Second},
{2, 2 * time.Second, 3 * time.Second},
{3, 3 * time.Second, 4 * time.Second},
{4, 4 * time.Second, 5 * time.Second},
{5, 5 * time.Second, 5 * time.Second}, // 封顶
{99, 5 * time.Second, 5 * time.Second},
}
for _, c := range cases {
for i := 0; i < 50; i++ {
got := throttleBackoff(c.streak)
if got < c.min || got > c.max {
t.Fatalf("throttleBackoff(%d) = %v, want [%v,%v]", c.streak, got, c.min, c.max)
}
}
}
}
// TestThrottleBackoffStreakBelowOne 兜底:streak 传 0 或负数时也要落在第一个台阶。
func TestThrottleBackoffStreakBelowOne(t *testing.T) {
for _, streak := range []int{0, -1, -100} {
got := throttleBackoff(streak)
if got < throttleBackoffBase || got > throttleBackoffBase+throttleBackoffStep {
t.Fatalf("throttleBackoff(%d) = %v, want 第一个台阶", streak, got)
}
}
}
// TestThrottledAccountBacksOffWhileOthersKeepGoing 单账号降频不能拖累其它账号。
//
// 场景:账号 slow 被限流("活动火爆"),账号 fast 遇到的是售罄(不触发降频)。
// 期望:slow 的等待被阶梯拉长(窗口内只试得起一两次),
// fast 照常按配置节奏快速重试——降频必须是单账号级的。
func TestThrottledAccountBacksOffWhileOthersKeepGoing(t *testing.T) {
speedUpPeace(t)
var slowCreates, fastCreates int32
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path != "/create" {
return
}
name := requestAccount(t, r)
if name == "slow" {
atomic.AddInt32(&slowCreates, 1)
_, _ = w.Write([]byte(`{"error":1644400007,"msg":"活动火爆,请稍后再试"}`))
return
}
atomic.AddInt32(&fastCreates, 1)
// 售罄不触发限流降频,fast 账号按配置的 20ms 节奏持续重试。
_, _ = w.Write([]byte(`{"error":1644402023,"msg":"商品本次活动库存已耗完"}`))
})
start := time.Now()
ctl := newRunControl(scheduler, start, start.Add(700*time.Millisecond), 8, 2)
settings := runSettings{
concurrency: 8,
seckillMode: types.SeckillModeAggressive,
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
}
var wg sync.WaitGroup
for _, name := range []string{"slow", "fast"} {
wg.Add(1)
go func(name string) {
defer wg.Done()
scheduler.runAccountLoop(peaceAttempt(name), settings, ctl)
}(name)
}
wg.Wait()
slow := int(atomic.LoadInt32(&slowCreates))
fast := int(atomic.LoadInt32(&fastCreates))
if fast < 3 {
t.Fatalf("未限流账号只尝试了 %d 次;降频必须是单账号级的,不能影响其它账号", fast)
}
if slow >= fast {
t.Fatalf("被限流账号尝试 %d 次,未限流账号 %d 次;被限流账号应明显更慢", slow, fast)
}
}
// TestThrottleStreakResetsOnNonThrottleFailure 非限流的失败要把限流阶梯归零:
// 只有"连续被限流"才应该越等越久。
func TestThrottleStreakResetsOnNonThrottleFailure(t *testing.T) {
speedUpPeace(t)
speedUpThrottleBackoff(t)
var creates int32
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path != "/create" {
return
}
// 交替:限流、售罄、限流、售罄……
if atomic.AddInt32(&creates, 1)%2 == 1 {
_, _ = w.Write([]byte(`{"error":1644400007,"msg":"活动火爆,请稍后再试"}`))
return
}
_, _ = w.Write([]byte(`{"error":1644402023,"msg":"商品本次活动库存已耗完"}`))
})
start := time.Now()
ctl := newRunControl(scheduler, start, start.Add(1200*time.Millisecond), 4, 1)
settings := runSettings{
concurrency: 4,
seckillMode: types.SeckillModeAggressive,
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
}
scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
// 阶梯被重置过,等待不会一路涨到 5s 上限,
// 所以在 1.2 秒窗口内应当能尝试相当多次。
if got := int(atomic.LoadInt32(&creates)); got < 6 {
t.Fatalf("锁单请求 = %d;非限流失败应重置限流阶梯,等待不该一路拉长", got)
}
}
// ========== 「已达兑换上限」不算失败 ==========
// TestLimitReachedIsNotFailure "已达本次活动兑换上限"必须单独归类:
// 它代表账号此前已经兑换成功,既不是失败,也不该混进"账号终止"。
func TestLimitReachedIsNotFailure(t *testing.T) {
speedUpPeace(t)
var creates int32
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path == "/create" {
atomic.AddInt32(&creates, 1)
_, _ = w.Write([]byte(`{"error":1,"msg":"您已达本次活动兑换上限"}`))
}
})
start := time.Now()
ctl := newRunControl(scheduler, start, start.Add(2*time.Second), 4, 1)
settings := runSettings{
concurrency: 4,
seckillMode: types.SeckillModeAggressive,
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
}
out := scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
if out.kind != outcomeLimitReached {
t.Fatalf("kind = %q, want %q (reason=%s)", out.kind, outcomeLimitReached, out.reason)
}
if out.kind == outcomeTerminal {
t.Fatal("已达兑换上限不应归入账号终止")
}
// 命中后立刻收手,不再重试。
if got := int(atomic.LoadInt32(&creates)); got != 1 {
t.Fatalf("锁单请求 = %d, want 1(已达上限应立即收手)", got)
}
// 结果集写"已抢到|"前缀,前端据此显示"已抢到"而不是"失败"。
key := resultKey(types.ExchangeAttempt{Account: types.Account{Name: "a"}})
got := scheduler.Results()[key]
if !strings.HasPrefix(got, "已抢到|") {
t.Fatalf("结果 = %q, want 以 已抢到| 开头", got)
}
if strings.Contains(got, "失败|") {
t.Fatalf("结果不应以失败标记: %q", got)
}
}
// TestLimitReachedNotCountedAsFailureInSummary 汇总行里"已达兑换上限"要单列,
// 不计入失败,也不混进"账号终止"。
func TestLimitReachedNotCountedAsFailureInSummary(t *testing.T) {
speedUpPeace(t)
seckillMaxDuration = 1200 * time.Millisecond
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/create":
name := requestAccount(t, r)
if name == "limited" {
_, _ = w.Write([]byte(`{"error":1,"msg":"您已达本次活动兑换上限"}`))
return
}
_, _ = w.Write([]byte(`{"error":0,"data":{"orderId":"order-1","expireSeconds":300}}`))
case "/pay":
_, _ = w.Write([]byte(`{"error":0,"data":{"exchangeId":9,"exchangeNum":"1"}}`))
case "/bind":
_, _ = w.Write([]byte(`{"error":0,"data":{"gameRole":{"roleName":"测试角色"}}}`))
case "/score":
_, _ = w.Write([]byte(`{"error":0,"data":{"balance":900}}`))
}
})
scheduler.paths = writeAccountFile(t, "winner", "limited")
scheduler.accounts = NewAccountStore(scheduler.paths.AccountFile)
scheduler.LoadAccounts()
scheduler.UpdateProducts([]types.Product{{CommodityID: "goods-1", CommodityName: "测试商品"}})
store := logger.NewMemoryLogStore(4000)
cleanup := logger.InitLogging(store, t.TempDir(), "测试实例", "/tmp/douyu-go-test")
defer cleanup()
scheduler.RunNow(map[string]string{"winner": "goods-1", "limited": "goods-1"})
var summary string
for _, entry := range store.Since(0) {
if strings.Contains(entry.Message, "抢购任务结束:") {
summary = entry.Message
}
}
if summary == "" {
t.Fatalf("缺少任务汇总行;实际日志:\n%s", dumpMessages(store))
}
if !strings.Contains(summary, "已达兑换上限=1") {
t.Fatalf("汇总行应单列已达兑换上限: %q", summary)
}
if !strings.Contains(summary, "账号终止=0") {
t.Fatalf("已达兑换上限不应计入账号终止: %q", summary)
}
if !strings.Contains(summary, "成功=1/2") {
t.Fatalf("汇总行成功数 = %q, want 1/2", summary)
}
}
// ========== 精确命中,不提前 ==========
// TestDynamicAdvanceNeverTriggersEarly 触发时刻必须精确命中,不能提前。
//
// 2026-10-23 三机日志里 148 次"不在时间范围内 不能兑换",
// 全部落在斗鱼估算时间 -0.026s ~ +0.172s——提前量必须为 0。
func TestDynamicAdvanceNeverTriggersEarly(t *testing.T) {
cals := []types.TimeCalibration{
{Offset: 100 * time.Millisecond, ErrorBound: 45 * time.Millisecond, RTT: 100 * time.Millisecond},
{Offset: 300 * time.Millisecond, ErrorBound: 48 * time.Millisecond, RTT: 95 * time.Millisecond},
{Offset: 0, ErrorBound: 0, RTT: 0},
}
for _, cal := range cals {
if got := dynamicAdvance(cal); got != 0 {
t.Errorf("dynamicAdvance(%+v) = %v, want 0(必须精确命中,不能提前)", cal, got)
}
}
}
// TestNotStartedDoesNotCountAsFailure "不在时间范围内"= 活动还没开始,
// 不是失败:这类请求不该消耗重试次数、不该触发降频。
func TestNotStartedDoesNotCountAsFailure(t *testing.T) {
speedUpPeace(t)
speedUpThrottleBackoff(t)
var creates int32
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/create":
// 前 3 次说"活动还没开始",之后正常返回。
if atomic.AddInt32(&creates, 1) <= 3 {
_, _ = w.Write([]byte(`{"error":1,"msg":"不在时间范围内 不能兑换"}`))
return
}
_, _ = w.Write([]byte(`{"error":0,"data":{"orderId":"order-1","expireSeconds":300}}`))
case "/pay":
_, _ = w.Write([]byte(`{"error":0,"data":{"exchangeId":9,"exchangeNum":"1"}}`))
case "/bind":
_, _ = w.Write([]byte(`{"error":0,"data":{"gameRole":{"roleName":"测试角色"}}}`))
}
})
start := time.Now()
ctl := newRunControl(scheduler, start, start.Add(3*time.Second), 4, 1)
settings := runSettings{
concurrency: 4,
seckillMode: types.SeckillModeAggressive,
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
}
out := scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
if !strings.Contains(out.reason, "成功|") {
t.Fatalf("结果 = %q, want 成功", out.reason)
}
}
// TestNotStartedDoesNotTriggerThrottleBackoff "活动未开始"不得触发限流降频。
//
// 判据:只返回"不在时间范围内"时,账号的重试节奏必须保持很快(不降频)。
func TestNotStartedDoesNotTriggerThrottleBackoff(t *testing.T) {
speedUpPeace(t)
// 刻意不加速降频阶梯:如果"活动未开始"错误地走了降频,
// 1.2 秒窗口内只可能尝试 1~2 次。
var creates int32
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path == "/create" {
atomic.AddInt32(&creates, 1)
_, _ = w.Write([]byte(`{"error":1,"msg":"不在时间范围内 不能兑换"}`))
}
})
start := time.Now()
ctl := newRunControl(scheduler, start, start.Add(1200*time.Millisecond), 4, 1)
settings := runSettings{
concurrency: 4,
seckillMode: types.SeckillModeAggressive,
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
}
out := scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
// 窗口结束时既没成功也没拿到库存,结论应是"活动始终未开始"。
if out.kind != outcomeExpired {
t.Fatalf("kind = %q, want %q (reason=%s)", out.kind, outcomeExpired, out.reason)
}
// 没有降频:应当尝试了相当多次(50~150ms 节奏 → 1.2 秒内 8 次以上)。
if got := int(atomic.LoadInt32(&creates)); got < 8 {
t.Fatalf("锁单请求 = %d;「活动未开始」不应触发限流降频,应当保持快速重试", got)
}
}
// TestNotStartedResultNotMarkedAsFailure "活动未开始"期间结果集不能显示成失败:
// 前端会把带"失败|"前缀的结果渲染成红色失败。
func TestNotStartedResultNotMarkedAsFailure(t *testing.T) {
speedUpPeace(t)
speedUpThrottleBackoff(t)
var creates int32
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path == "/create" {
atomic.AddInt32(&creates, 1)
_, _ = w.Write([]byte(`{"error":1,"msg":"不在时间范围内 不能兑换"}`))
}
})
start := time.Now()
ctl := newRunControl(scheduler, start, start.Add(400*time.Millisecond), 4, 1)
settings := runSettings{
concurrency: 4,
seckillMode: types.SeckillModeAggressive,
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
}
scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
key := resultKey(types.ExchangeAttempt{Account: types.Account{Name: "a"}})
got := scheduler.Results()[key]
if strings.Contains(got, "失败|不在时间范围内") {
t.Fatalf("结果集不应把「活动未开始」记成失败: %q", got)
}
}
// ========== 积分基线异步写盘 ==========
// TestScoreBaselineFlushIsSynchronous Flush 必须同步落盘(抢购不等待,但需要确定落盘时用它)。
func TestScoreBaselineFlushIsSynchronous(t *testing.T) {
path := filepath.Join(t.TempDir(), scoreBaselineFileName)
store := NewScoreBaselineStore(path)
defer store.Close()
store.Set("a", 1000)
store.Flush()
reloaded := NewScoreBaselineStore(path)
defer reloaded.Close()
reloaded.Load()
if got, ok, _ := reloaded.Get("a"); !ok || got != 1000 {
t.Fatalf("Flush 后应能读到基线, got %d ok=%v", got, ok)
}
}
// TestScoreBaselineCloseIsSafeAfterSet 关闭后再回写不能 panic。
func TestScoreBaselineCloseIsSafeAfterSet(t *testing.T) {
path := filepath.Join(t.TempDir(), scoreBaselineFileName)
store := NewScoreBaselineStore(path)
store.Set("a", 1000)
store.Close()
// 关闭后再写:应当安全地被忽略,而不是 panic。
store.Set("b", 2000)
store.SetMany(map[string]int{"c": 3000})
}