10-23 三台机器 13 个实例抢同一次 18:00 的精英手册,把当天日志翻完发现 四个问题都指向"请求发得太猛、发得太早、结论分得太粗": - 被限流的账号仍按 100ms 节奏猛打。单实例锁单 950~1650 次(约 20 QPS)时 18~24 个账号被"您点击太快",而锁单仅 148 次的实例 30 个账号一次全中—— 可见"点击太快/活动火爆"是账号级封禁,继续打只会加深封禁。 - 定时器提前约 100ms 触发,想用这点提前量抵消 RTT,结果 148 次请求 被明确拒绝:"不在时间范围内 不能兑换"。它们全部落在斗鱼估算时间 -0.026s ~ +0.172s,也就是活动还没开闸就冲进去了。 - 积分基线写盘用 os.Rename 覆盖旧文件,Windows 上会报 Access is denied, 三机日志里出现 281 次"积分基线写入失败",基线没落盘导致 pc-1 四个实例 0 条核对记录,且基线为 0 时会误报"积分未见减少"。 - "已达本次活动兑换上限"被记成账号级终止失败。这其实是"该账号在本次活动 里已经换到过了",10-23 日志里这些账号紧接着就报了"积分已扣减,兑换确认"。 - 限流改为单账号阶梯降频:新增 throttleBackoff,第 1 次等 1~2s、第 2 次 2~3s、第 3 次 3~4s……上限 5s,区间内随机取值避免同步醒来。throttleStreak 独立计数,非限流的失败会把它归零——只有"连续被限流"才越等越久。 每个账号跑在自己的 goroutine 里,降频只压它自己的下一次尝试, 不占用并发额度、不影响其它账号的节奏。 - 触发时刻改为精确命中:dynamicAdvance 返回 0,不再提前。另加兜底守卫—— 服务端回"不在时间范围内"时该次尝试不计失败、不消耗 attempts/failStreak、 不触发降频,用 50~150ms 快速重试直到活动真正开始。结果集里的 "失败|不在时间范围内…"同步改写成"等待活动开始",避免界面闪红。 - 积分基线改异步写盘:flushCh 队列 + 后台协程串行写,抢购收尾不再等写盘; 新增 Flush 供需要确定落盘的场合。另按 build tag 分平台解决 Windows os.Rename 不能覆盖已有文件的问题(replace_file_unix/windows.go)。 - "已达兑换上限"单列:新增 outcomeLimitReached,结果集写"已抢到|"前缀, 汇总行加"已达兑换上限=N",既不计入失败也不混进"账号终止"。 - 账号不做全局去重:同一账号可以买多个皮肤,不同实例也可以抢不同皮肤。 - 新增测试 throttle_backoff_test.go(12 个用例):降频阶梯递增、单账号级 隔离、非限流失败重置阶梯、兑换上限不计失败且立即收手、精确命中不提前、 活动未开始不触发降频且不记失败、Flush 同步落盘。 - 三个原有测试因降频语义变化需要调整时间尺度:新增 speedUpThrottleBackoff 只压缩阶梯的时间尺度,不改变阶梯结构。
398 lines
14 KiB
Go
398 lines
14 KiB
Go
package seckill
|
|
|
|
import (
|
|
"net/http"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"douyu_go/internal/logger"
|
|
"douyu_go/internal/types"
|
|
)
|
|
|
|
// ========== 限流阶梯式降频 ==========
|
|
|
|
// TestThrottleBackoffEscalates 锁住"被限流后逐级拉长等待"这条设计:
|
|
// 第 1 次 1~2s、第 2 次 2~3s、第 3 次 3~4s……上限 5s。
|
|
//
|
|
// 依据 2026-10-23 三机日志:单实例锁单 950~1650 次时 18~24 个账号被限流,
|
|
// 而锁单仅 148 次的实例 30 个账号一次全中——猛打只会加深账号级封禁。
|
|
func TestThrottleBackoffEscalates(t *testing.T) {
|
|
cases := []struct {
|
|
streak int
|
|
min time.Duration
|
|
max time.Duration
|
|
}{
|
|
{1, 1 * time.Second, 2 * time.Second},
|
|
{2, 2 * time.Second, 3 * time.Second},
|
|
{3, 3 * time.Second, 4 * time.Second},
|
|
{4, 4 * time.Second, 5 * time.Second},
|
|
{5, 5 * time.Second, 5 * time.Second}, // 封顶
|
|
{99, 5 * time.Second, 5 * time.Second},
|
|
}
|
|
for _, c := range cases {
|
|
for i := 0; i < 50; i++ {
|
|
got := throttleBackoff(c.streak)
|
|
if got < c.min || got > c.max {
|
|
t.Fatalf("throttleBackoff(%d) = %v, want [%v,%v]", c.streak, got, c.min, c.max)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestThrottleBackoffStreakBelowOne 兜底:streak 传 0 或负数时也要落在第一个台阶。
|
|
func TestThrottleBackoffStreakBelowOne(t *testing.T) {
|
|
for _, streak := range []int{0, -1, -100} {
|
|
got := throttleBackoff(streak)
|
|
if got < throttleBackoffBase || got > throttleBackoffBase+throttleBackoffStep {
|
|
t.Fatalf("throttleBackoff(%d) = %v, want 第一个台阶", streak, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestThrottledAccountBacksOffWhileOthersKeepGoing 单账号降频不能拖累其它账号。
|
|
//
|
|
// 场景:账号 slow 被限流("活动火爆"),账号 fast 遇到的是售罄(不触发降频)。
|
|
// 期望:slow 的等待被阶梯拉长(窗口内只试得起一两次),
|
|
// fast 照常按配置节奏快速重试——降频必须是单账号级的。
|
|
func TestThrottledAccountBacksOffWhileOthersKeepGoing(t *testing.T) {
|
|
speedUpPeace(t)
|
|
|
|
var slowCreates, fastCreates int32
|
|
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if r.URL.Path != "/create" {
|
|
return
|
|
}
|
|
name := requestAccount(t, r)
|
|
if name == "slow" {
|
|
atomic.AddInt32(&slowCreates, 1)
|
|
_, _ = w.Write([]byte(`{"error":1644400007,"msg":"活动火爆,请稍后再试"}`))
|
|
return
|
|
}
|
|
atomic.AddInt32(&fastCreates, 1)
|
|
// 售罄不触发限流降频,fast 账号按配置的 20ms 节奏持续重试。
|
|
_, _ = w.Write([]byte(`{"error":1644402023,"msg":"商品本次活动库存已耗完"}`))
|
|
})
|
|
|
|
start := time.Now()
|
|
ctl := newRunControl(scheduler, start, start.Add(700*time.Millisecond), 8, 2)
|
|
settings := runSettings{
|
|
concurrency: 8,
|
|
seckillMode: types.SeckillModeAggressive,
|
|
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
for _, name := range []string{"slow", "fast"} {
|
|
wg.Add(1)
|
|
go func(name string) {
|
|
defer wg.Done()
|
|
scheduler.runAccountLoop(peaceAttempt(name), settings, ctl)
|
|
}(name)
|
|
}
|
|
wg.Wait()
|
|
|
|
slow := int(atomic.LoadInt32(&slowCreates))
|
|
fast := int(atomic.LoadInt32(&fastCreates))
|
|
if fast < 3 {
|
|
t.Fatalf("未限流账号只尝试了 %d 次;降频必须是单账号级的,不能影响其它账号", fast)
|
|
}
|
|
if slow >= fast {
|
|
t.Fatalf("被限流账号尝试 %d 次,未限流账号 %d 次;被限流账号应明显更慢", slow, fast)
|
|
}
|
|
}
|
|
|
|
// TestThrottleStreakResetsOnNonThrottleFailure 非限流的失败要把限流阶梯归零:
|
|
// 只有"连续被限流"才应该越等越久。
|
|
func TestThrottleStreakResetsOnNonThrottleFailure(t *testing.T) {
|
|
speedUpPeace(t)
|
|
speedUpThrottleBackoff(t)
|
|
|
|
var creates int32
|
|
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if r.URL.Path != "/create" {
|
|
return
|
|
}
|
|
// 交替:限流、售罄、限流、售罄……
|
|
if atomic.AddInt32(&creates, 1)%2 == 1 {
|
|
_, _ = w.Write([]byte(`{"error":1644400007,"msg":"活动火爆,请稍后再试"}`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"error":1644402023,"msg":"商品本次活动库存已耗完"}`))
|
|
})
|
|
|
|
start := time.Now()
|
|
ctl := newRunControl(scheduler, start, start.Add(1200*time.Millisecond), 4, 1)
|
|
settings := runSettings{
|
|
concurrency: 4,
|
|
seckillMode: types.SeckillModeAggressive,
|
|
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
|
|
}
|
|
scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
|
|
|
|
// 阶梯被重置过,等待不会一路涨到 5s 上限,
|
|
// 所以在 1.2 秒窗口内应当能尝试相当多次。
|
|
if got := int(atomic.LoadInt32(&creates)); got < 6 {
|
|
t.Fatalf("锁单请求 = %d;非限流失败应重置限流阶梯,等待不该一路拉长", got)
|
|
}
|
|
}
|
|
|
|
// ========== 「已达兑换上限」不算失败 ==========
|
|
|
|
// TestLimitReachedIsNotFailure "已达本次活动兑换上限"必须单独归类:
|
|
// 它代表账号此前已经兑换成功,既不是失败,也不该混进"账号终止"。
|
|
func TestLimitReachedIsNotFailure(t *testing.T) {
|
|
speedUpPeace(t)
|
|
|
|
var creates int32
|
|
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if r.URL.Path == "/create" {
|
|
atomic.AddInt32(&creates, 1)
|
|
_, _ = w.Write([]byte(`{"error":1,"msg":"您已达本次活动兑换上限"}`))
|
|
}
|
|
})
|
|
|
|
start := time.Now()
|
|
ctl := newRunControl(scheduler, start, start.Add(2*time.Second), 4, 1)
|
|
settings := runSettings{
|
|
concurrency: 4,
|
|
seckillMode: types.SeckillModeAggressive,
|
|
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
|
|
}
|
|
out := scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
|
|
|
|
if out.kind != outcomeLimitReached {
|
|
t.Fatalf("kind = %q, want %q (reason=%s)", out.kind, outcomeLimitReached, out.reason)
|
|
}
|
|
if out.kind == outcomeTerminal {
|
|
t.Fatal("已达兑换上限不应归入账号终止")
|
|
}
|
|
// 命中后立刻收手,不再重试。
|
|
if got := int(atomic.LoadInt32(&creates)); got != 1 {
|
|
t.Fatalf("锁单请求 = %d, want 1(已达上限应立即收手)", got)
|
|
}
|
|
// 结果集写"已抢到|"前缀,前端据此显示"已抢到"而不是"失败"。
|
|
key := resultKey(types.ExchangeAttempt{Account: types.Account{Name: "a"}})
|
|
got := scheduler.Results()[key]
|
|
if !strings.HasPrefix(got, "已抢到|") {
|
|
t.Fatalf("结果 = %q, want 以 已抢到| 开头", got)
|
|
}
|
|
if strings.Contains(got, "失败|") {
|
|
t.Fatalf("结果不应以失败标记: %q", got)
|
|
}
|
|
}
|
|
|
|
// TestLimitReachedNotCountedAsFailureInSummary 汇总行里"已达兑换上限"要单列,
|
|
// 不计入失败,也不混进"账号终止"。
|
|
func TestLimitReachedNotCountedAsFailureInSummary(t *testing.T) {
|
|
speedUpPeace(t)
|
|
seckillMaxDuration = 1200 * time.Millisecond
|
|
|
|
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.Path {
|
|
case "/create":
|
|
name := requestAccount(t, r)
|
|
if name == "limited" {
|
|
_, _ = w.Write([]byte(`{"error":1,"msg":"您已达本次活动兑换上限"}`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"error":0,"data":{"orderId":"order-1","expireSeconds":300}}`))
|
|
case "/pay":
|
|
_, _ = w.Write([]byte(`{"error":0,"data":{"exchangeId":9,"exchangeNum":"1"}}`))
|
|
case "/bind":
|
|
_, _ = w.Write([]byte(`{"error":0,"data":{"gameRole":{"roleName":"测试角色"}}}`))
|
|
case "/score":
|
|
_, _ = w.Write([]byte(`{"error":0,"data":{"balance":900}}`))
|
|
}
|
|
})
|
|
scheduler.paths = writeAccountFile(t, "winner", "limited")
|
|
scheduler.accounts = NewAccountStore(scheduler.paths.AccountFile)
|
|
scheduler.LoadAccounts()
|
|
scheduler.UpdateProducts([]types.Product{{CommodityID: "goods-1", CommodityName: "测试商品"}})
|
|
|
|
store := logger.NewMemoryLogStore(4000)
|
|
cleanup := logger.InitLogging(store, t.TempDir(), "测试实例", "/tmp/douyu-go-test")
|
|
defer cleanup()
|
|
scheduler.RunNow(map[string]string{"winner": "goods-1", "limited": "goods-1"})
|
|
|
|
var summary string
|
|
for _, entry := range store.Since(0) {
|
|
if strings.Contains(entry.Message, "抢购任务结束:") {
|
|
summary = entry.Message
|
|
}
|
|
}
|
|
if summary == "" {
|
|
t.Fatalf("缺少任务汇总行;实际日志:\n%s", dumpMessages(store))
|
|
}
|
|
if !strings.Contains(summary, "已达兑换上限=1") {
|
|
t.Fatalf("汇总行应单列已达兑换上限: %q", summary)
|
|
}
|
|
if !strings.Contains(summary, "账号终止=0") {
|
|
t.Fatalf("已达兑换上限不应计入账号终止: %q", summary)
|
|
}
|
|
if !strings.Contains(summary, "成功=1/2") {
|
|
t.Fatalf("汇总行成功数 = %q, want 1/2", summary)
|
|
}
|
|
}
|
|
|
|
// ========== 精确命中,不提前 ==========
|
|
|
|
// TestDynamicAdvanceNeverTriggersEarly 触发时刻必须精确命中,不能提前。
|
|
//
|
|
// 2026-10-23 三机日志里 148 次"不在时间范围内 不能兑换",
|
|
// 全部落在斗鱼估算时间 -0.026s ~ +0.172s——提前量必须为 0。
|
|
func TestDynamicAdvanceNeverTriggersEarly(t *testing.T) {
|
|
cals := []types.TimeCalibration{
|
|
{Offset: 100 * time.Millisecond, ErrorBound: 45 * time.Millisecond, RTT: 100 * time.Millisecond},
|
|
{Offset: 300 * time.Millisecond, ErrorBound: 48 * time.Millisecond, RTT: 95 * time.Millisecond},
|
|
{Offset: 0, ErrorBound: 0, RTT: 0},
|
|
}
|
|
for _, cal := range cals {
|
|
if got := dynamicAdvance(cal); got != 0 {
|
|
t.Errorf("dynamicAdvance(%+v) = %v, want 0(必须精确命中,不能提前)", cal, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestNotStartedDoesNotCountAsFailure "不在时间范围内"= 活动还没开始,
|
|
// 不是失败:这类请求不该消耗重试次数、不该触发降频。
|
|
func TestNotStartedDoesNotCountAsFailure(t *testing.T) {
|
|
speedUpPeace(t)
|
|
speedUpThrottleBackoff(t)
|
|
|
|
var creates int32
|
|
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch r.URL.Path {
|
|
case "/create":
|
|
// 前 3 次说"活动还没开始",之后正常返回。
|
|
if atomic.AddInt32(&creates, 1) <= 3 {
|
|
_, _ = w.Write([]byte(`{"error":1,"msg":"不在时间范围内 不能兑换"}`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"error":0,"data":{"orderId":"order-1","expireSeconds":300}}`))
|
|
case "/pay":
|
|
_, _ = w.Write([]byte(`{"error":0,"data":{"exchangeId":9,"exchangeNum":"1"}}`))
|
|
case "/bind":
|
|
_, _ = w.Write([]byte(`{"error":0,"data":{"gameRole":{"roleName":"测试角色"}}}`))
|
|
}
|
|
})
|
|
|
|
start := time.Now()
|
|
ctl := newRunControl(scheduler, start, start.Add(3*time.Second), 4, 1)
|
|
settings := runSettings{
|
|
concurrency: 4,
|
|
seckillMode: types.SeckillModeAggressive,
|
|
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
|
|
}
|
|
out := scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
|
|
|
|
if !strings.Contains(out.reason, "成功|") {
|
|
t.Fatalf("结果 = %q, want 成功", out.reason)
|
|
}
|
|
}
|
|
|
|
// TestNotStartedDoesNotTriggerThrottleBackoff "活动未开始"不得触发限流降频。
|
|
//
|
|
// 判据:只返回"不在时间范围内"时,账号的重试节奏必须保持很快(不降频)。
|
|
func TestNotStartedDoesNotTriggerThrottleBackoff(t *testing.T) {
|
|
speedUpPeace(t)
|
|
|
|
// 刻意不加速降频阶梯:如果"活动未开始"错误地走了降频,
|
|
// 1.2 秒窗口内只可能尝试 1~2 次。
|
|
var creates int32
|
|
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if r.URL.Path == "/create" {
|
|
atomic.AddInt32(&creates, 1)
|
|
_, _ = w.Write([]byte(`{"error":1,"msg":"不在时间范围内 不能兑换"}`))
|
|
}
|
|
})
|
|
|
|
start := time.Now()
|
|
ctl := newRunControl(scheduler, start, start.Add(1200*time.Millisecond), 4, 1)
|
|
settings := runSettings{
|
|
concurrency: 4,
|
|
seckillMode: types.SeckillModeAggressive,
|
|
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
|
|
}
|
|
out := scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
|
|
|
|
// 窗口结束时既没成功也没拿到库存,结论应是"活动始终未开始"。
|
|
if out.kind != outcomeExpired {
|
|
t.Fatalf("kind = %q, want %q (reason=%s)", out.kind, outcomeExpired, out.reason)
|
|
}
|
|
// 没有降频:应当尝试了相当多次(50~150ms 节奏 → 1.2 秒内 8 次以上)。
|
|
if got := int(atomic.LoadInt32(&creates)); got < 8 {
|
|
t.Fatalf("锁单请求 = %d;「活动未开始」不应触发限流降频,应当保持快速重试", got)
|
|
}
|
|
}
|
|
|
|
// TestNotStartedResultNotMarkedAsFailure "活动未开始"期间结果集不能显示成失败:
|
|
// 前端会把带"失败|"前缀的结果渲染成红色失败。
|
|
func TestNotStartedResultNotMarkedAsFailure(t *testing.T) {
|
|
speedUpPeace(t)
|
|
speedUpThrottleBackoff(t)
|
|
|
|
var creates int32
|
|
scheduler := newOrderTestScheduler(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if r.URL.Path == "/create" {
|
|
atomic.AddInt32(&creates, 1)
|
|
_, _ = w.Write([]byte(`{"error":1,"msg":"不在时间范围内 不能兑换"}`))
|
|
}
|
|
})
|
|
|
|
start := time.Now()
|
|
ctl := newRunControl(scheduler, start, start.Add(400*time.Millisecond), 4, 1)
|
|
settings := runSettings{
|
|
concurrency: 4,
|
|
seckillMode: types.SeckillModeAggressive,
|
|
retryStages: []types.RetryStage{{MaxElapsed: 75, Interval: 20}},
|
|
}
|
|
scheduler.runAccountLoop(peaceAttempt("a"), settings, ctl)
|
|
|
|
key := resultKey(types.ExchangeAttempt{Account: types.Account{Name: "a"}})
|
|
got := scheduler.Results()[key]
|
|
if strings.Contains(got, "失败|不在时间范围内") {
|
|
t.Fatalf("结果集不应把「活动未开始」记成失败: %q", got)
|
|
}
|
|
}
|
|
|
|
// ========== 积分基线异步写盘 ==========
|
|
|
|
// TestScoreBaselineFlushIsSynchronous Flush 必须同步落盘(抢购不等待,但需要确定落盘时用它)。
|
|
func TestScoreBaselineFlushIsSynchronous(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), scoreBaselineFileName)
|
|
store := NewScoreBaselineStore(path)
|
|
defer store.Close()
|
|
|
|
store.Set("a", 1000)
|
|
store.Flush()
|
|
|
|
reloaded := NewScoreBaselineStore(path)
|
|
defer reloaded.Close()
|
|
reloaded.Load()
|
|
if got, ok, _ := reloaded.Get("a"); !ok || got != 1000 {
|
|
t.Fatalf("Flush 后应能读到基线, got %d ok=%v", got, ok)
|
|
}
|
|
}
|
|
|
|
// TestScoreBaselineCloseIsSafeAfterSet 关闭后再回写不能 panic。
|
|
func TestScoreBaselineCloseIsSafeAfterSet(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), scoreBaselineFileName)
|
|
store := NewScoreBaselineStore(path)
|
|
store.Set("a", 1000)
|
|
store.Close()
|
|
// 关闭后再写:应当安全地被忽略,而不是 panic。
|
|
store.Set("b", 2000)
|
|
store.SetMany(map[string]int{"c": 3000})
|
|
}
|