feat(proxy): 增加代理池演练并修复账号密码提取,发布 0.5.1
This commit is contained in:
+3
-2
@@ -48,7 +48,8 @@ type App struct {
|
||||
productsAt time.Time
|
||||
|
||||
// accountsMu 串行化内部账号库修改,避免并发导入、编辑或删除互相覆盖。
|
||||
accountsMu sync.Mutex
|
||||
accountsMu sync.Mutex
|
||||
proxyDrillMu sync.Mutex
|
||||
|
||||
closeLog func()
|
||||
closeTrace func()
|
||||
@@ -256,7 +257,7 @@ func (a *App) Startup(ctx context.Context) {
|
||||
|
||||
proxyURL = a.effectiveProxyURL(a.getCfg())
|
||||
if proxyURL != "" {
|
||||
if _, err := a.getClient().ProbeServerTime(); err != nil {
|
||||
if _, err := a.getClient().ProbeProxyEndpoint(); err != nil {
|
||||
logger.LogEvent("代理", "warn", "启动代理探测失败: %v", err)
|
||||
currentCfg := a.getCfg()
|
||||
if currentCfg.ProxyEnabled &&
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"douyu_go/internal/activity"
|
||||
"douyu_go/internal/client"
|
||||
"douyu_go/internal/constants"
|
||||
"douyu_go/internal/logger"
|
||||
proxyprovider "douyu_go/internal/proxy"
|
||||
"douyu_go/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultProxyDrillAccounts = 30
|
||||
maxProxyDrillAccounts = 200
|
||||
proxyDrillExtractRounds = 3
|
||||
proxyDrillMaxRounds = 20
|
||||
)
|
||||
|
||||
type proxyDrillPlan struct {
|
||||
accounts int
|
||||
groupSize int
|
||||
active int
|
||||
standby int
|
||||
}
|
||||
|
||||
type proxyDrillTarget struct {
|
||||
role string
|
||||
group int
|
||||
requests int
|
||||
endpoint proxyprovider.Endpoint
|
||||
}
|
||||
|
||||
type proxyDrillProbe struct {
|
||||
success bool
|
||||
latency time.Duration
|
||||
bytes int64
|
||||
err error
|
||||
}
|
||||
|
||||
// RunProxyDrill 使用无需 Cookie 的商品接口验证代理池,不读取账号、不锁单、不支付。
|
||||
func (a *App) RunProxyDrill(req types.ProxyDrillRequest) types.ProxyDrillResult {
|
||||
a.proxyDrillMu.Lock()
|
||||
defer a.proxyDrillMu.Unlock()
|
||||
|
||||
startedAt := time.Now()
|
||||
cfg := a.getCfg()
|
||||
if strings.TrimSpace(cfg.ProxyAPI) == "" {
|
||||
return types.ProxyDrillResult{Error: "请先填写代理提取 API"}
|
||||
}
|
||||
if scheduler := a.getScheduler(); scheduler != nil && scheduler.IsRunning() {
|
||||
return types.ProxyDrillResult{Error: "抢购任务运行中,不能同时执行代理池演练"}
|
||||
}
|
||||
|
||||
act := activity.Get(cfg.Activity)
|
||||
if act == nil {
|
||||
act = activity.Default()
|
||||
}
|
||||
if strings.TrimSpace(act.StoreHomeURL) == "" {
|
||||
return types.ProxyDrillResult{Error: "当前活动没有可用于代理演练的只读接口"}
|
||||
}
|
||||
probeEndpoint := act.StoreHomeURL
|
||||
if act.IsXiaodian {
|
||||
probeEndpoint = cfg.TimeAPIURL
|
||||
}
|
||||
|
||||
accounts := req.SimulatedAccounts
|
||||
if accounts <= 0 {
|
||||
accounts = defaultProxyDrillAccounts
|
||||
}
|
||||
if accounts > maxProxyDrillAccounts {
|
||||
accounts = maxProxyDrillAccounts
|
||||
}
|
||||
groupSize := cfg.AccountsPerProxy
|
||||
if groupSize <= 0 {
|
||||
groupSize = constants.DefaultAccountsPerProxy
|
||||
}
|
||||
if groupSize > constants.MaxAccountsPerProxy {
|
||||
groupSize = constants.MaxAccountsPerProxy
|
||||
}
|
||||
plan := proxyDrillPlan{
|
||||
accounts: accounts,
|
||||
groupSize: groupSize,
|
||||
active: (accounts + groupSize - 1) / groupSize,
|
||||
}
|
||||
plan.standby = proxyDrillStandbyCount(plan.active, cfg.ProxySparePercent)
|
||||
desired := plan.active + plan.standby
|
||||
|
||||
result := types.ProxyDrillResult{
|
||||
Endpoint: probeEndpoint,
|
||||
SimulatedAccounts: plan.accounts,
|
||||
AccountsPerProxy: plan.groupSize,
|
||||
ActiveCount: plan.active,
|
||||
StandbyCount: plan.standby,
|
||||
Items: []types.ProxyDrillItem{},
|
||||
}
|
||||
logger.LogEvent("代理", "info", "开始代理池演练: 模拟账号=%d,活动代理=%d,备用代理=%d,接口=%s",
|
||||
plan.accounts, plan.active, plan.standby, probeEndpoint)
|
||||
|
||||
endpoints, err := a.extractProxyDrillPool(cfg, desired)
|
||||
if len(endpoints) < plan.active {
|
||||
result.Error = fmt.Sprintf("代理数量不足: 需要%d个活动代理,实际%d个", plan.active, len(endpoints))
|
||||
if err != nil {
|
||||
result.Error += ";" + err.Error()
|
||||
}
|
||||
result.DurationMs = time.Since(startedAt).Milliseconds()
|
||||
return result
|
||||
}
|
||||
|
||||
targets := buildProxyDrillTargets(plan, endpoints)
|
||||
result.Items = make([]types.ProxyDrillItem, len(targets))
|
||||
var wg sync.WaitGroup
|
||||
for i, target := range targets {
|
||||
wg.Add(1)
|
||||
go func(index int, target proxyDrillTarget) {
|
||||
defer wg.Done()
|
||||
result.Items[index] = a.runProxyDrillTarget(act, cfg.TimeAPIURL, target)
|
||||
}(i, target)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for _, item := range result.Items {
|
||||
if item.Success != item.Requests || item.Requests == 0 {
|
||||
continue
|
||||
}
|
||||
if item.Role == "活动代理" {
|
||||
result.HealthyActive++
|
||||
} else {
|
||||
result.HealthyStandby++
|
||||
}
|
||||
}
|
||||
result.Success = result.HealthyActive >= plan.active && result.HealthyStandby >= plan.standby
|
||||
if !result.Success {
|
||||
result.Error = "活动代理或备用代理未全部通过演练"
|
||||
}
|
||||
result.DurationMs = time.Since(startedAt).Milliseconds()
|
||||
logger.LogEvent("代理", "success", "代理池演练完成: 成功=%v,活动代理=%d/%d,备用代理=%d/%d,耗时=%s",
|
||||
result.Success, result.HealthyActive, plan.active, result.HealthyStandby, plan.standby,
|
||||
time.Duration(result.DurationMs)*time.Millisecond)
|
||||
return result
|
||||
}
|
||||
|
||||
func proxyDrillStandbyCount(activeCount, sparePercent int) int {
|
||||
if activeCount <= 0 {
|
||||
return 0
|
||||
}
|
||||
if sparePercent <= 0 {
|
||||
sparePercent = constants.DefaultProxySparePercent
|
||||
}
|
||||
if sparePercent > constants.MaxProxySparePercent {
|
||||
sparePercent = constants.MaxProxySparePercent
|
||||
}
|
||||
count := (activeCount*sparePercent + 99) / 100
|
||||
if count < 1 {
|
||||
count = 1
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (a *App) extractProxyDrillPool(cfg types.Config, desired int) ([]proxyprovider.Endpoint, error) {
|
||||
rounds := proxyDrillExtractRounds
|
||||
if desired > rounds {
|
||||
rounds = desired
|
||||
}
|
||||
if rounds > proxyDrillMaxRounds {
|
||||
rounds = proxyDrillMaxRounds
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, desired)
|
||||
endpoints := make([]proxyprovider.Endpoint, 0, desired)
|
||||
var lastErr error
|
||||
for round := 0; round < rounds && len(endpoints) < desired; round++ {
|
||||
batch, err := extractProxyEndpoints(cfg, cfg.ProxyAPI, desired-len(endpoints))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
if proxyprovider.IsFatal(err) {
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, endpoint := range batch {
|
||||
if !endpoint.ExpiresAt.IsZero() && !time.Now().Before(endpoint.ExpiresAt) {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[endpoint.URL]; exists {
|
||||
continue
|
||||
}
|
||||
seen[endpoint.URL] = struct{}{}
|
||||
endpoints = append(endpoints, endpoint)
|
||||
if len(endpoints) >= desired {
|
||||
break
|
||||
}
|
||||
}
|
||||
if round < rounds-1 {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if len(endpoints) < desired {
|
||||
if lastErr != nil {
|
||||
return endpoints, lastErr
|
||||
}
|
||||
return endpoints, fmt.Errorf("代理 API 仅返回 %d/%d 个有效代理", len(endpoints), desired)
|
||||
}
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
func buildProxyDrillTargets(plan proxyDrillPlan, endpoints []proxyprovider.Endpoint) []proxyDrillTarget {
|
||||
targets := make([]proxyDrillTarget, 0, len(endpoints))
|
||||
remaining := plan.accounts
|
||||
for i := 0; i < plan.active; i++ {
|
||||
requests := min(plan.groupSize, remaining)
|
||||
if requests < 1 {
|
||||
requests = 1
|
||||
}
|
||||
remaining -= requests
|
||||
targets = append(targets, proxyDrillTarget{
|
||||
role: "活动代理",
|
||||
group: i + 1,
|
||||
requests: requests,
|
||||
endpoint: endpoints[i],
|
||||
})
|
||||
}
|
||||
for i := plan.active; i < len(endpoints); i++ {
|
||||
requests := min(plan.groupSize, plan.accounts)
|
||||
if requests < 1 {
|
||||
requests = 1
|
||||
}
|
||||
targets = append(targets, proxyDrillTarget{
|
||||
role: "备用代理",
|
||||
group: i - plan.active + 1,
|
||||
requests: requests,
|
||||
endpoint: endpoints[i],
|
||||
})
|
||||
}
|
||||
return targets
|
||||
}
|
||||
|
||||
func (a *App) runProxyDrillTarget(act *activity.Activity, timeAPIURL string, target proxyDrillTarget) types.ProxyDrillItem {
|
||||
item := types.ProxyDrillItem{
|
||||
Role: target.role,
|
||||
Group: target.group,
|
||||
Proxy: client.MaskProxyURL(target.endpoint.URL),
|
||||
IP: target.endpoint.IP,
|
||||
Region: target.endpoint.Region,
|
||||
Provider: target.endpoint.Provider,
|
||||
Requests: target.requests,
|
||||
}
|
||||
if !target.endpoint.ExpiresAt.IsZero() {
|
||||
item.ExpiresAt = target.endpoint.ExpiresAt.UnixMilli()
|
||||
}
|
||||
proxyClient, err := client.NewWithProxy(act, timeAPIURL, a.getTrace(), target.endpoint.URL)
|
||||
if err != nil {
|
||||
item.Failed = target.requests
|
||||
item.Error = err.Error()
|
||||
return item
|
||||
}
|
||||
defer closeProxyDrillClient(proxyClient)
|
||||
|
||||
results := make(chan proxyDrillProbe, target.requests)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < target.requests; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
probe, err := proxyClient.ProbeProxyEndpoint()
|
||||
results <- proxyDrillProbe{
|
||||
success: err == nil,
|
||||
latency: probe.Latency,
|
||||
bytes: probe.Bytes,
|
||||
err: err,
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
var totalLatency time.Duration
|
||||
for probe := range results {
|
||||
if probe.success {
|
||||
item.Success++
|
||||
} else {
|
||||
item.Failed++
|
||||
if item.Error == "" && probe.err != nil {
|
||||
item.Error = probe.err.Error()
|
||||
}
|
||||
}
|
||||
totalLatency += probe.latency
|
||||
item.Bytes += probe.bytes
|
||||
if probe.latency > time.Duration(item.MaxLatencyMs)*time.Millisecond {
|
||||
item.MaxLatencyMs = probe.latency.Milliseconds()
|
||||
}
|
||||
}
|
||||
if target.requests > 0 {
|
||||
item.AvgLatencyMs = (totalLatency / time.Duration(target.requests)).Milliseconds()
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func closeProxyDrillClient(cli *client.DouyuClient) {
|
||||
if cli == nil {
|
||||
return
|
||||
}
|
||||
if cli.HTTPClient != nil {
|
||||
cli.HTTPClient.CloseIdleConnections()
|
||||
}
|
||||
if cli.TimeHTTPClient != nil {
|
||||
cli.TimeHTTPClient.CloseIdleConnections()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
proxyprovider "douyu_go/internal/proxy"
|
||||
"douyu_go/internal/types"
|
||||
)
|
||||
|
||||
func TestProxyDrillPlanAndTargets(t *testing.T) {
|
||||
if got := proxyDrillStandbyCount(6, 25); got != 2 {
|
||||
t.Fatalf("proxyDrillStandbyCount(6, 25)=%d, want 2", got)
|
||||
}
|
||||
plan := proxyDrillPlan{accounts: 30, groupSize: 5, active: 6, standby: 2}
|
||||
endpoints := make([]proxyprovider.Endpoint, 8)
|
||||
targets := buildProxyDrillTargets(plan, endpoints)
|
||||
if len(targets) != 8 {
|
||||
t.Fatalf("targets=%d, want 8", len(targets))
|
||||
}
|
||||
totalRequests := 0
|
||||
for _, target := range targets {
|
||||
totalRequests += target.requests
|
||||
}
|
||||
if totalRequests != 40 {
|
||||
t.Fatalf("totalRequests=%d, want 40 virtual requests", totalRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractProxyDrillPoolDeduplicates(t *testing.T) {
|
||||
apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for i := 0; i < 8; i++ {
|
||||
fmt.Fprintf(w, "http://127.0.0.1:%d\n", 20000+i)
|
||||
}
|
||||
}))
|
||||
defer apiServer.Close()
|
||||
|
||||
app := &App{}
|
||||
cfg := types.Config{ProxyAPI: apiServer.URL}
|
||||
endpoints, err := app.extractProxyDrillPool(cfg, 8)
|
||||
if err != nil {
|
||||
t.Fatalf("extractProxyDrillPool() error = %v", err)
|
||||
}
|
||||
if len(endpoints) != 8 {
|
||||
t.Fatalf("endpoints=%d, want 8", len(endpoints))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user