974 lines
27 KiB
Go
974 lines
27 KiB
Go
package seckill
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net/url"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"douyu_go/internal/client"
|
|
"douyu_go/internal/constants"
|
|
"douyu_go/internal/logger"
|
|
proxyprovider "douyu_go/internal/proxy"
|
|
"douyu_go/internal/types"
|
|
)
|
|
|
|
const (
|
|
proxyFetchRounds = 3
|
|
proxyFetchMaxRounds = 20
|
|
proxyFetchRetryWait = 300 * time.Millisecond
|
|
proxyRequestTimeout = 15 * time.Second
|
|
proxyValidationTimeout = 3 * time.Second
|
|
proxyValidationWorkers = 3
|
|
proxyValidateAttempts = 1
|
|
)
|
|
|
|
type proxySettingsSnapshot struct {
|
|
enabled bool
|
|
proxyURL string
|
|
proxyAPI string
|
|
autoFetchOnTimer bool
|
|
accountsPerProxy int
|
|
sparePercent int
|
|
autoWhitelist bool
|
|
whitelistReplace bool
|
|
whitelistPassword string
|
|
whitelistSignKey string
|
|
}
|
|
|
|
type validatedProxy struct {
|
|
url string
|
|
client *client.DouyuClient
|
|
rtt time.Duration
|
|
ip string
|
|
region string
|
|
source string
|
|
expiresAt time.Time
|
|
}
|
|
|
|
type proxyRuntimeGroup struct {
|
|
accountNames []string
|
|
activeURL string
|
|
consecutiveFailures int
|
|
failed bool
|
|
}
|
|
|
|
type preparedProxyPool struct {
|
|
accountClients map[string]*client.DouyuClient
|
|
activeClients []*client.DouyuClient
|
|
groups map[*client.DouyuClient]*proxyRuntimeGroup
|
|
standbys []validatedProxy
|
|
allClients []*client.DouyuClient
|
|
excludedURLs map[string]struct{}
|
|
standbyTarget int
|
|
expiresAt time.Time
|
|
}
|
|
|
|
type proxySwitchEvent struct {
|
|
accountNames []string
|
|
fromURL string
|
|
toURL string
|
|
toIP string
|
|
toRegion string
|
|
toRTT time.Duration
|
|
toSource string
|
|
}
|
|
|
|
// UpdateProxySettings 运行时更新代理配置。
|
|
func (s *Scheduler) UpdateProxySettings(cfg types.Config) {
|
|
enabled := cfg.ProxyEnabled
|
|
proxyURL := cfg.ProxyURL
|
|
proxyAPI := cfg.ProxyAPI
|
|
autoFetch := cfg.AutoFetchProxyOnSchedule
|
|
accountsPerProxy := cfg.AccountsPerProxy
|
|
sparePercent := cfg.ProxySparePercent
|
|
if accountsPerProxy <= 0 {
|
|
accountsPerProxy = constants.DefaultAccountsPerProxy
|
|
}
|
|
if accountsPerProxy > constants.MaxAccountsPerProxy {
|
|
accountsPerProxy = constants.MaxAccountsPerProxy
|
|
}
|
|
if sparePercent <= 0 {
|
|
sparePercent = constants.DefaultProxySparePercent
|
|
}
|
|
if sparePercent > constants.MaxProxySparePercent {
|
|
sparePercent = constants.MaxProxySparePercent
|
|
}
|
|
proxyURL = strings.TrimSpace(proxyURL)
|
|
proxyAPI = strings.TrimSpace(proxyAPI)
|
|
whitelistPassword := strings.TrimSpace(cfg.ProxyWhitelistPassword)
|
|
whitelistSignKey := strings.TrimSpace(cfg.ProxyWhitelistSignKey)
|
|
s.proxyMu.Lock()
|
|
changed := s.proxyEnabled != enabled ||
|
|
s.proxyURL != proxyURL ||
|
|
s.proxyAPI != proxyAPI ||
|
|
s.autoFetchProxyOnSchedule != autoFetch ||
|
|
s.accountsPerProxy != accountsPerProxy ||
|
|
s.proxySparePercent != sparePercent ||
|
|
s.autoWhitelist != cfg.ProxyAutoWhitelist ||
|
|
s.whitelistReplace != cfg.ProxyWhitelistReplace ||
|
|
s.whitelistPassword != whitelistPassword ||
|
|
s.whitelistSignKey != whitelistSignKey
|
|
s.proxyEnabled = enabled
|
|
s.proxyURL = proxyURL
|
|
s.proxyAPI = proxyAPI
|
|
s.autoFetchProxyOnSchedule = autoFetch
|
|
s.accountsPerProxy = accountsPerProxy
|
|
s.proxySparePercent = sparePercent
|
|
s.autoWhitelist = cfg.ProxyAutoWhitelist
|
|
s.whitelistReplace = cfg.ProxyWhitelistReplace
|
|
s.whitelistPassword = whitelistPassword
|
|
s.whitelistSignKey = whitelistSignKey
|
|
s.proxyMu.Unlock()
|
|
|
|
if !changed {
|
|
return
|
|
}
|
|
s.clientMu.Lock()
|
|
s.accountClientsDirty = true
|
|
s.clientMu.Unlock()
|
|
}
|
|
|
|
func (s *Scheduler) proxySettings() proxySettingsSnapshot {
|
|
s.proxyMu.RLock()
|
|
defer s.proxyMu.RUnlock()
|
|
size := s.accountsPerProxy
|
|
if size <= 0 {
|
|
size = constants.DefaultAccountsPerProxy
|
|
}
|
|
if size > constants.MaxAccountsPerProxy {
|
|
size = constants.MaxAccountsPerProxy
|
|
}
|
|
sparePercent := s.proxySparePercent
|
|
if sparePercent <= 0 {
|
|
sparePercent = constants.DefaultProxySparePercent
|
|
}
|
|
if sparePercent > constants.MaxProxySparePercent {
|
|
sparePercent = constants.MaxProxySparePercent
|
|
}
|
|
return proxySettingsSnapshot{
|
|
enabled: s.proxyEnabled,
|
|
proxyURL: s.proxyURL,
|
|
proxyAPI: s.proxyAPI,
|
|
autoFetchOnTimer: s.autoFetchProxyOnSchedule,
|
|
accountsPerProxy: size,
|
|
sparePercent: sparePercent,
|
|
autoWhitelist: s.autoWhitelist,
|
|
whitelistReplace: s.whitelistReplace,
|
|
whitelistPassword: s.whitelistPassword,
|
|
whitelistSignKey: s.whitelistSignKey,
|
|
}
|
|
}
|
|
|
|
func (s *Scheduler) closeAccountClients() {
|
|
s.clientMu.Lock()
|
|
oldClients := append([]*client.DouyuClient(nil), s.proxyClientPool...)
|
|
for _, cli := range s.accountClients {
|
|
oldClients = append(oldClients, cli)
|
|
}
|
|
s.accountClients = make(map[string]*client.DouyuClient)
|
|
s.accountClientOrder = nil
|
|
s.accountClientsDirty = true
|
|
s.accountClientsExpireAt = time.Time{}
|
|
s.proxyGroups = make(map[*client.DouyuClient]*proxyRuntimeGroup)
|
|
s.proxyStandbys = nil
|
|
s.proxyClientPool = nil
|
|
s.proxyExcludedURLs = make(map[string]struct{})
|
|
s.proxyStandbyTarget = 0
|
|
s.proxyPoolGeneration++
|
|
s.proxyRefillInFlight = false
|
|
s.clientMu.Unlock()
|
|
closeDouyuClientList(oldClients)
|
|
}
|
|
|
|
func (s *Scheduler) replaceAccountClients(pool preparedProxyPool) {
|
|
if pool.accountClients == nil {
|
|
pool.accountClients = make(map[string]*client.DouyuClient)
|
|
}
|
|
if pool.groups == nil {
|
|
pool.groups = make(map[*client.DouyuClient]*proxyRuntimeGroup)
|
|
}
|
|
if pool.excludedURLs == nil {
|
|
pool.excludedURLs = make(map[string]struct{})
|
|
}
|
|
s.clientMu.Lock()
|
|
oldClients := append([]*client.DouyuClient(nil), s.proxyClientPool...)
|
|
for _, cli := range s.accountClients {
|
|
oldClients = append(oldClients, cli)
|
|
}
|
|
s.accountClients = pool.accountClients
|
|
s.accountClientOrder = pool.activeClients
|
|
s.accountClientsDirty = false
|
|
s.accountClientsExpireAt = pool.expiresAt
|
|
s.proxyGroups = pool.groups
|
|
s.proxyStandbys = pool.standbys
|
|
s.proxyClientPool = pool.allClients
|
|
s.proxyExcludedURLs = pool.excludedURLs
|
|
s.proxyStandbyTarget = pool.standbyTarget
|
|
s.proxyPoolGeneration++
|
|
s.proxyRefillInFlight = false
|
|
s.clientMu.Unlock()
|
|
closeDouyuClientList(oldClients)
|
|
}
|
|
|
|
func closeDouyuClients(clients map[string]*client.DouyuClient) {
|
|
items := make([]*client.DouyuClient, 0, len(clients))
|
|
for _, cli := range clients {
|
|
items = append(items, cli)
|
|
}
|
|
closeDouyuClientList(items)
|
|
}
|
|
|
|
func closeDouyuClientList(clients []*client.DouyuClient) {
|
|
seen := make(map[*client.DouyuClient]struct{}, len(clients))
|
|
for _, cli := range clients {
|
|
if cli == nil {
|
|
continue
|
|
}
|
|
if _, ok := seen[cli]; ok {
|
|
continue
|
|
}
|
|
seen[cli] = struct{}{}
|
|
if cli.HTTPClient != nil {
|
|
cli.HTTPClient.CloseIdleConnections()
|
|
}
|
|
if cli.TimeHTTPClient != nil {
|
|
cli.TimeHTTPClient.CloseIdleConnections()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Scheduler) accountClientsCover(accountNames []string) bool {
|
|
if len(accountNames) == 0 {
|
|
return true
|
|
}
|
|
s.clientMu.RLock()
|
|
defer s.clientMu.RUnlock()
|
|
if len(s.accountClients) == 0 {
|
|
return false
|
|
}
|
|
if s.accountClientsDirty {
|
|
return false
|
|
}
|
|
if len(s.proxyStandbys) < s.proxyStandbyTarget {
|
|
return false
|
|
}
|
|
if !s.accountClientsExpireAt.IsZero() && !time.Now().Before(s.accountClientsExpireAt.Add(-30*time.Second)) {
|
|
return false
|
|
}
|
|
for _, name := range accountNames {
|
|
if s.accountClients[name] == nil {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (s *Scheduler) calibrationClient() *client.DouyuClient {
|
|
s.clientMu.RLock()
|
|
defer s.clientMu.RUnlock()
|
|
if len(s.accountClientOrder) > 0 && s.accountClientOrder[0] != nil {
|
|
return s.accountClientOrder[0]
|
|
}
|
|
return s.client
|
|
}
|
|
|
|
func (s *Scheduler) allClients() []*client.DouyuClient {
|
|
s.clientMu.RLock()
|
|
defer s.clientMu.RUnlock()
|
|
out := make([]*client.DouyuClient, 0, len(s.accountClientOrder)+1)
|
|
seen := make(map[*client.DouyuClient]struct{}, len(s.accountClientOrder)+1)
|
|
if s.client != nil {
|
|
seen[s.client] = struct{}{}
|
|
out = append(out, s.client)
|
|
}
|
|
for _, cli := range s.accountClientOrder {
|
|
if cli == nil {
|
|
continue
|
|
}
|
|
if _, ok := seen[cli]; ok {
|
|
continue
|
|
}
|
|
seen[cli] = struct{}{}
|
|
out = append(out, cli)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// prepareProxyGroups 将本次参与抢购的账号按固定顺序分组,并为每组准备独立客户端。
|
|
// forceRefresh=true 用于定时任务启动前主动刷新短效代理。
|
|
func (s *Scheduler) prepareProxyGroups(accountProducts map[string]string, forceRefresh bool) error {
|
|
s.proxyPrepareMu.Lock()
|
|
defer s.proxyPrepareMu.Unlock()
|
|
|
|
cfg := s.proxySettings()
|
|
names := selectedAccountNames(accountProducts)
|
|
if len(names) == 0 {
|
|
return nil
|
|
}
|
|
if !cfg.enabled {
|
|
s.closeAccountClients()
|
|
return nil
|
|
}
|
|
if !forceRefresh && s.accountClientsCover(names) {
|
|
return nil
|
|
}
|
|
|
|
groups := chunkAccountNames(names, cfg.accountsPerProxy)
|
|
activeCount := len(groups)
|
|
standbyTarget := proxyStandbyCount(activeCount, cfg.sparePercent)
|
|
base := s.clientSnapshot()
|
|
if base == nil {
|
|
return errors.New("基础客户端未初始化")
|
|
}
|
|
|
|
poolStartedAt := time.Now()
|
|
proxies, err := s.buildProxyPool(base, cfg, activeCount, activeCount+standbyTarget, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(proxies) < activeCount {
|
|
return fmt.Errorf("代理数量不足: 需要%d个活动代理,实际%d个", activeCount, len(proxies))
|
|
}
|
|
if len(proxies) < activeCount+standbyTarget {
|
|
logger.LogEvent("代理", "warn", "备用代理不足: 目标%d个,实际%d个;活动代理不受影响",
|
|
standbyTarget, len(proxies)-activeCount)
|
|
}
|
|
|
|
accountClients := make(map[string]*client.DouyuClient, len(names))
|
|
clientOrder := make([]*client.DouyuClient, 0, activeCount)
|
|
runtimeGroups := make(map[*client.DouyuClient]*proxyRuntimeGroup, activeCount)
|
|
for i, group := range groups {
|
|
proxy := proxies[i]
|
|
clientOrder = append(clientOrder, proxy.client)
|
|
runtimeGroups[proxy.client] = &proxyRuntimeGroup{
|
|
accountNames: append([]string(nil), group...),
|
|
activeURL: proxy.url,
|
|
}
|
|
for _, name := range group {
|
|
accountClients[name] = proxy.client
|
|
}
|
|
logger.LogEvent("代理", "success", "代理组%d: 账号=%d 出口=%s IP=%s 归属=%s RTT=%s 来源=%s",
|
|
i+1, len(group), client.MaskProxyURL(proxy.url), proxy.ip, proxy.region,
|
|
proxy.rtt.Round(time.Millisecond), proxy.source)
|
|
}
|
|
standbys := append([]validatedProxy(nil), proxies[activeCount:]...)
|
|
allClients := make([]*client.DouyuClient, 0, len(proxies))
|
|
excludedURLs := make(map[string]struct{}, len(proxies))
|
|
for _, proxy := range proxies {
|
|
allClients = append(allClients, proxy.client)
|
|
excludedURLs[proxy.url] = struct{}{}
|
|
proxyClient := proxy.client
|
|
client.MonitorProxyHealth(proxyClient, func(err error) {
|
|
s.recordProxyResult(proxyClient, err)
|
|
})
|
|
}
|
|
expiresAt := time.Time{}
|
|
if cfg.proxyAPI != "" {
|
|
expiresAt = proxyPoolExpiresAt(cfg.proxyAPI, poolStartedAt)
|
|
}
|
|
for _, proxy := range proxies {
|
|
if proxy.expiresAt.IsZero() {
|
|
continue
|
|
}
|
|
if expiresAt.IsZero() || proxy.expiresAt.Before(expiresAt) {
|
|
expiresAt = proxy.expiresAt
|
|
}
|
|
}
|
|
s.replaceAccountClients(preparedProxyPool{
|
|
accountClients: accountClients,
|
|
activeClients: clientOrder,
|
|
groups: runtimeGroups,
|
|
standbys: standbys,
|
|
allClients: allClients,
|
|
excludedURLs: excludedURLs,
|
|
standbyTarget: effectiveProxyStandbyTarget(cfg, standbyTarget, len(standbys)),
|
|
expiresAt: expiresAt,
|
|
})
|
|
logger.LogEvent("代理", "success", "代理池准备完成: 账号=%d,每组=%d,活动代理=%d,备用代理=%d/%d",
|
|
len(names), cfg.accountsPerProxy, activeCount, len(standbys), standbyTarget)
|
|
return nil
|
|
}
|
|
|
|
func effectiveProxyStandbyTarget(cfg proxySettingsSnapshot, target, actual int) int {
|
|
if cfg.proxyAPI == "" {
|
|
return actual
|
|
}
|
|
return target
|
|
}
|
|
|
|
// PrepareProxyGroups 供立即抢购入口在启动前同步准备代理,便于把错误直接返回给界面。
|
|
func (s *Scheduler) PrepareProxyGroups(accountProducts map[string]string) error {
|
|
return s.prepareProxyGroups(accountProducts, false)
|
|
}
|
|
|
|
func selectedAccountNames(accountProducts map[string]string) []string {
|
|
names := make([]string, 0, len(accountProducts))
|
|
for name, productID := range accountProducts {
|
|
name = strings.TrimSpace(name)
|
|
if name == "" || strings.TrimSpace(productID) == "" {
|
|
continue
|
|
}
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
return names
|
|
}
|
|
|
|
func chunkAccountNames(names []string, size int) [][]string {
|
|
if size <= 0 {
|
|
size = constants.DefaultAccountsPerProxy
|
|
}
|
|
if size > constants.MaxAccountsPerProxy {
|
|
size = constants.MaxAccountsPerProxy
|
|
}
|
|
groups := make([][]string, 0, (len(names)+size-1)/size)
|
|
for start := 0; start < len(names); start += size {
|
|
end := start + size
|
|
if end > len(names) {
|
|
end = len(names)
|
|
}
|
|
groups = append(groups, append([]string(nil), names[start:end]...))
|
|
}
|
|
return groups
|
|
}
|
|
|
|
func proxyStandbyCount(activeCount, sparePercent int) int {
|
|
if activeCount <= 0 {
|
|
return 0
|
|
}
|
|
if sparePercent <= 0 {
|
|
sparePercent = constants.DefaultProxySparePercent
|
|
}
|
|
if sparePercent > constants.MaxProxySparePercent {
|
|
sparePercent = constants.MaxProxySparePercent
|
|
}
|
|
count := (activeCount*sparePercent + 99) / 100
|
|
if count < 1 {
|
|
count = 1
|
|
}
|
|
return count
|
|
}
|
|
|
|
func (s *Scheduler) buildProxyPool(base *client.DouyuClient, cfg proxySettingsSnapshot, required, desired int, excludedURLs map[string]struct{}) ([]validatedProxy, error) {
|
|
if desired < required {
|
|
desired = required
|
|
}
|
|
if required <= 0 {
|
|
return nil, nil
|
|
}
|
|
if cfg.proxyAPI != "" {
|
|
proxies, err := s.fetchAndValidateProxyPool(base, cfg, desired, excludedURLs)
|
|
if len(proxies) >= required {
|
|
if len(proxies) < desired {
|
|
logger.LogEvent("代理", "warn", "动态代理备用数量不足: 目标%d个,实际%d个", desired, len(proxies))
|
|
}
|
|
return proxies, nil
|
|
}
|
|
closeValidatedProxies(proxies)
|
|
if err != nil {
|
|
logger.LogEvent("代理", "warn", "动态代理提取未完全成功: %v", err)
|
|
}
|
|
}
|
|
|
|
staticURLs := uniqueStrings(splitProxyURLs(cfg.proxyURL))
|
|
if len(staticURLs) == 0 {
|
|
return nil, errors.New("代理已启用,但没有可用的提取 API 或静态代理")
|
|
}
|
|
if len(staticURLs) == 1 {
|
|
logger.LogEvent("代理", "warn", "当前只有一个静态代理,%d 个账号组将共享该出口且没有可用备用代理", required)
|
|
}
|
|
|
|
validateCount := len(staticURLs)
|
|
if validateCount > desired {
|
|
validateCount = desired
|
|
}
|
|
templates := make([]validatedProxy, 0, validateCount)
|
|
for _, rawURL := range staticURLs[:validateCount] {
|
|
proxy, err := s.validateProxy(base, rawURL, "静态")
|
|
if err != nil {
|
|
closeValidatedProxies(templates)
|
|
return nil, fmt.Errorf("静态代理不可用: %w", err)
|
|
}
|
|
templates = append(templates, proxy)
|
|
}
|
|
|
|
total := required
|
|
if len(templates) > required {
|
|
total += len(templates) - required
|
|
if total > desired {
|
|
total = desired
|
|
}
|
|
}
|
|
proxies := make([]validatedProxy, 0, total)
|
|
var cloneErr error
|
|
for i := 0; i < total; i++ {
|
|
template := templates[i%len(templates)]
|
|
proxy := template
|
|
if i >= len(templates) {
|
|
proxy, cloneErr = cloneValidatedProxy(base, template)
|
|
if cloneErr != nil {
|
|
closeValidatedProxies(proxies)
|
|
closeValidatedProxies(templates)
|
|
return nil, fmt.Errorf("创建共享静态代理连接失败: %w", cloneErr)
|
|
}
|
|
}
|
|
proxies = append(proxies, proxy)
|
|
}
|
|
if total < len(templates) {
|
|
closeValidatedProxies(templates[total:])
|
|
}
|
|
return proxies, nil
|
|
}
|
|
|
|
func uniqueStrings(items []string) []string {
|
|
seen := make(map[string]struct{}, len(items))
|
|
out := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
if _, ok := seen[item]; ok {
|
|
continue
|
|
}
|
|
seen[item] = struct{}{}
|
|
out = append(out, item)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneValidatedProxy(base *client.DouyuClient, template validatedProxy) (validatedProxy, error) {
|
|
cli, err := client.NewWithProxy(base.Activity, base.TimeAPIURL, base.Trace, template.url)
|
|
if err != nil {
|
|
return validatedProxy{}, err
|
|
}
|
|
template.client = cli
|
|
return template, nil
|
|
}
|
|
|
|
func closeValidatedProxies(proxies []validatedProxy) {
|
|
clients := make([]*client.DouyuClient, 0, len(proxies))
|
|
for _, proxy := range proxies {
|
|
clients = append(clients, proxy.client)
|
|
}
|
|
closeDouyuClientList(clients)
|
|
}
|
|
|
|
func splitProxyURLs(raw string) []string {
|
|
parts := strings.FieldsFunc(raw, func(r rune) bool {
|
|
return r == '\n' || r == '\r' || r == ';' || r == ','
|
|
})
|
|
out := make([]string, 0, len(parts))
|
|
for _, part := range parts {
|
|
part = strings.TrimSpace(part)
|
|
if part != "" {
|
|
out = append(out, part)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Scheduler) fetchAndValidateProxyPool(base *client.DouyuClient, cfg proxySettingsSnapshot, count int, excludedURLs map[string]struct{}) ([]validatedProxy, error) {
|
|
service := s.proxyService
|
|
if service == nil {
|
|
service = proxyprovider.NewService()
|
|
s.proxyService = service
|
|
}
|
|
|
|
proxies := make([]validatedProxy, 0, count)
|
|
seen := make(map[string]struct{}, count+len(excludedURLs))
|
|
for rawURL := range excludedURLs {
|
|
seen[rawURL] = struct{}{}
|
|
}
|
|
var lastErr error
|
|
|
|
rounds := proxyFetchRounds
|
|
if count > rounds {
|
|
rounds = count
|
|
}
|
|
if rounds > proxyFetchMaxRounds {
|
|
rounds = proxyFetchMaxRounds
|
|
}
|
|
for round := 0; round < rounds; round++ {
|
|
need := count - len(proxies)
|
|
if need <= 0 {
|
|
break
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), proxyRequestTimeout)
|
|
endpoints, err := service.ExtractWithWhitelist(ctx, cfg.proxyAPI, need, proxyprovider.WhitelistConfig{
|
|
APIURL: cfg.proxyAPI,
|
|
LoginPassword: cfg.whitelistPassword,
|
|
SignKey: cfg.whitelistSignKey,
|
|
ReplaceOldest: cfg.whitelistReplace,
|
|
AutoWhitelist: cfg.autoWhitelist,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
lastErr = err
|
|
if proxyprovider.IsFatal(err) {
|
|
break
|
|
}
|
|
} else {
|
|
validated := s.validateProxyBatch(base, endpoints)
|
|
for _, item := range validated {
|
|
if item.err != nil {
|
|
lastErr = item.err
|
|
continue
|
|
}
|
|
if !item.endpoint.ExpiresAt.IsZero() && !time.Now().Before(item.endpoint.ExpiresAt) {
|
|
lastErr = fmt.Errorf("代理 %s 已过期", item.endpoint.IP)
|
|
closeDouyuClientList([]*client.DouyuClient{item.proxy.client})
|
|
continue
|
|
}
|
|
if _, exists := seen[item.endpoint.URL]; exists {
|
|
closeDouyuClientList([]*client.DouyuClient{item.proxy.client})
|
|
continue
|
|
}
|
|
seen[item.endpoint.URL] = struct{}{}
|
|
proxy := item.proxy
|
|
if proxy.ip == "" {
|
|
proxy.ip = item.endpoint.IP
|
|
}
|
|
if proxy.region == "" {
|
|
proxy.region = item.endpoint.Region
|
|
}
|
|
proxy.expiresAt = item.endpoint.ExpiresAt
|
|
proxies = append(proxies, proxy)
|
|
}
|
|
}
|
|
|
|
if round < rounds-1 {
|
|
time.Sleep(proxyFetchRetryWait)
|
|
}
|
|
}
|
|
|
|
if len(proxies) < count {
|
|
if lastErr != nil {
|
|
return proxies, lastErr
|
|
}
|
|
return proxies, fmt.Errorf("代理 API 仅返回 %d/%d 个有效代理", len(proxies), count)
|
|
}
|
|
return proxies, nil
|
|
}
|
|
|
|
func (s *Scheduler) recordProxyResult(proxyClient *client.DouyuClient, err error) {
|
|
if proxyClient == nil {
|
|
return
|
|
}
|
|
|
|
s.clientMu.Lock()
|
|
group := s.proxyGroups[proxyClient]
|
|
if group == nil {
|
|
s.clientMu.Unlock()
|
|
return
|
|
}
|
|
if err == nil {
|
|
group.consecutiveFailures = 0
|
|
s.clientMu.Unlock()
|
|
return
|
|
}
|
|
if !client.IsProxyTransportFailure(err) || group.failed {
|
|
s.clientMu.Unlock()
|
|
return
|
|
}
|
|
|
|
group.consecutiveFailures++
|
|
if group.consecutiveFailures < 2 {
|
|
failures := group.consecutiveFailures
|
|
activeURL := group.activeURL
|
|
s.clientMu.Unlock()
|
|
logger.LogEvent("代理", "debug", "代理网络异常计数: %s 连续失败=%d", client.MaskProxyURL(activeURL), failures)
|
|
return
|
|
}
|
|
|
|
event, expiredClients, switched := s.switchGroupToStandbyLocked(proxyClient, group)
|
|
needRefill := false
|
|
failedAccounts := 0
|
|
failedURL := ""
|
|
if switched {
|
|
needRefill = len(s.proxyStandbys) < s.proxyStandbyTarget && !s.proxyRefillInFlight
|
|
} else {
|
|
group.failed = true
|
|
failedAccounts = len(group.accountNames)
|
|
failedURL = group.activeURL
|
|
needRefill = !s.proxyRefillInFlight
|
|
}
|
|
s.clientMu.Unlock()
|
|
|
|
closeDouyuClientList(expiredClients)
|
|
if switched {
|
|
logProxySwitch("代理故障切换", event)
|
|
} else {
|
|
logger.LogEvent("代理", "warn", "代理组连续失败且暂无备用: 账号=%d 出口=%s,正在尝试补提取",
|
|
failedAccounts, client.MaskProxyURL(failedURL))
|
|
}
|
|
if needRefill {
|
|
s.startProxyRefill()
|
|
}
|
|
}
|
|
|
|
func (s *Scheduler) switchGroupToStandbyLocked(oldClient *client.DouyuClient, group *proxyRuntimeGroup) (proxySwitchEvent, []*client.DouyuClient, bool) {
|
|
expiredClients := make([]*client.DouyuClient, 0, 1)
|
|
for len(s.proxyStandbys) > 0 {
|
|
standby := s.proxyStandbys[0]
|
|
s.proxyStandbys = s.proxyStandbys[1:]
|
|
if !standby.expiresAt.IsZero() && !time.Now().Before(standby.expiresAt) {
|
|
expiredClients = append(expiredClients, standby.client)
|
|
continue
|
|
}
|
|
|
|
event := proxySwitchEvent{
|
|
accountNames: append([]string(nil), group.accountNames...),
|
|
fromURL: group.activeURL,
|
|
toURL: standby.url,
|
|
toIP: standby.ip,
|
|
toRegion: standby.region,
|
|
toRTT: standby.rtt,
|
|
toSource: standby.source,
|
|
}
|
|
group.activeURL = standby.url
|
|
group.consecutiveFailures = 0
|
|
group.failed = false
|
|
for _, accountName := range group.accountNames {
|
|
s.accountClients[accountName] = standby.client
|
|
}
|
|
delete(s.proxyGroups, oldClient)
|
|
s.proxyGroups[standby.client] = group
|
|
replaceClientInOrder(s.accountClientOrder, oldClient, standby.client)
|
|
return event, expiredClients, true
|
|
}
|
|
return proxySwitchEvent{}, expiredClients, false
|
|
}
|
|
|
|
func replaceClientInOrder(order []*client.DouyuClient, oldClient, newClient *client.DouyuClient) {
|
|
for i, cli := range order {
|
|
if cli == oldClient {
|
|
order[i] = newClient
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Scheduler) startProxyRefill() {
|
|
cfg := s.proxySettings()
|
|
if !cfg.enabled || cfg.proxyAPI == "" {
|
|
return
|
|
}
|
|
base := s.clientSnapshot()
|
|
if base == nil {
|
|
return
|
|
}
|
|
|
|
s.clientMu.Lock()
|
|
need := s.proxyStandbyTarget - len(s.proxyStandbys)
|
|
if need <= 0 || s.proxyRefillInFlight {
|
|
s.clientMu.Unlock()
|
|
return
|
|
}
|
|
s.proxyRefillInFlight = true
|
|
generation := s.proxyPoolGeneration
|
|
excludedURLs := copyStringSet(s.proxyExcludedURLs)
|
|
s.clientMu.Unlock()
|
|
|
|
go s.refillProxyStandbys(base, cfg, generation, need, excludedURLs)
|
|
}
|
|
|
|
func (s *Scheduler) refillProxyStandbys(base *client.DouyuClient, cfg proxySettingsSnapshot, generation uint64, need int, excludedURLs map[string]struct{}) {
|
|
proxies, err := s.fetchAndValidateProxyPool(base, cfg, need, excludedURLs)
|
|
if err != nil {
|
|
logger.LogEvent("代理", "warn", "备用代理补提取未完全成功: %v", err)
|
|
}
|
|
|
|
var extras []*client.DouyuClient
|
|
var switches []proxySwitchEvent
|
|
var expiredClients []*client.DouyuClient
|
|
|
|
s.clientMu.Lock()
|
|
if generation != s.proxyPoolGeneration {
|
|
s.clientMu.Unlock()
|
|
closeValidatedProxies(proxies)
|
|
return
|
|
}
|
|
|
|
currentNeed := s.proxyStandbyTarget - len(s.proxyStandbys)
|
|
if currentNeed < 0 {
|
|
currentNeed = 0
|
|
}
|
|
if len(proxies) > currentNeed {
|
|
for _, proxy := range proxies[currentNeed:] {
|
|
extras = append(extras, proxy.client)
|
|
}
|
|
proxies = proxies[:currentNeed]
|
|
}
|
|
for _, proxy := range proxies {
|
|
proxyClient := proxy.client
|
|
client.MonitorProxyHealth(proxyClient, func(err error) {
|
|
s.recordProxyResult(proxyClient, err)
|
|
})
|
|
s.proxyStandbys = append(s.proxyStandbys, proxy)
|
|
s.proxyClientPool = append(s.proxyClientPool, proxy.client)
|
|
s.proxyExcludedURLs[proxy.url] = struct{}{}
|
|
}
|
|
|
|
for {
|
|
pendingClient, pendingGroup := findFailedProxyGroup(s.proxyGroups)
|
|
if pendingClient == nil {
|
|
break
|
|
}
|
|
event, expired, switched := s.switchGroupToStandbyLocked(pendingClient, pendingGroup)
|
|
expiredClients = append(expiredClients, expired...)
|
|
if !switched {
|
|
break
|
|
}
|
|
switches = append(switches, event)
|
|
}
|
|
s.proxyRefillInFlight = false
|
|
remainingFailed := countFailedProxyGroups(s.proxyGroups)
|
|
standbyCount := len(s.proxyStandbys)
|
|
standbyTarget := s.proxyStandbyTarget
|
|
s.clientMu.Unlock()
|
|
|
|
closeDouyuClientList(extras)
|
|
closeDouyuClientList(expiredClients)
|
|
for _, event := range switches {
|
|
logProxySwitch("备用代理补提取成功", event)
|
|
}
|
|
if len(proxies) > 0 {
|
|
logger.LogEvent("代理", "success", "备用代理已补充: 新增=%d,剩余=%d,目标=%d",
|
|
len(proxies), standbyCount, standbyTarget)
|
|
}
|
|
if remainingFailed > 0 {
|
|
logger.LogEvent("代理", "warn", "仍有%d个代理组等待备用代理,请检查提取 API 配额与白名单", remainingFailed)
|
|
}
|
|
if standbyCount < standbyTarget && len(proxies) > 0 {
|
|
s.startProxyRefill()
|
|
}
|
|
}
|
|
|
|
func findFailedProxyGroup(groups map[*client.DouyuClient]*proxyRuntimeGroup) (*client.DouyuClient, *proxyRuntimeGroup) {
|
|
for cli, group := range groups {
|
|
if group != nil && group.failed {
|
|
return cli, group
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func countFailedProxyGroups(groups map[*client.DouyuClient]*proxyRuntimeGroup) int {
|
|
count := 0
|
|
for _, group := range groups {
|
|
if group != nil && group.failed {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func copyStringSet(source map[string]struct{}) map[string]struct{} {
|
|
out := make(map[string]struct{}, len(source))
|
|
for value := range source {
|
|
out[value] = struct{}{}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func logProxySwitch(label string, event proxySwitchEvent) {
|
|
logger.LogEvent("代理", "warn", "%s: 账号=%d %s -> %s IP=%s 归属=%s RTT=%s 来源=%s",
|
|
label, len(event.accountNames), client.MaskProxyURL(event.fromURL), client.MaskProxyURL(event.toURL),
|
|
event.toIP, event.toRegion, event.toRTT.Round(time.Millisecond), event.toSource)
|
|
}
|
|
|
|
func proxyPoolExpiresAt(apiURL string, now time.Time) time.Time {
|
|
minutes := 10
|
|
if u, err := url.Parse(strings.TrimSpace(apiURL)); err == nil {
|
|
query := u.Query()
|
|
raw := query.Get("minute")
|
|
if raw == "" {
|
|
raw = query.Get("min")
|
|
}
|
|
if value, err := strconv.Atoi(raw); err == nil && value > 0 && value <= 60 {
|
|
minutes = value
|
|
}
|
|
}
|
|
return now.Add(time.Duration(minutes) * time.Minute)
|
|
}
|
|
|
|
func (s *Scheduler) validateProxy(base *client.DouyuClient, proxyURL, source string) (validatedProxy, error) {
|
|
cli, err := client.NewWithProxy(base.Activity, base.TimeAPIURL, base.Trace, proxyURL)
|
|
if err != nil {
|
|
return validatedProxy{}, err
|
|
}
|
|
if cli.HTTPClient != nil && cli.HTTPClient.Timeout > proxyValidationTimeout {
|
|
cli.HTTPClient.Timeout = proxyValidationTimeout
|
|
}
|
|
var lastErr error
|
|
for attempt := 0; attempt < proxyValidateAttempts; attempt++ {
|
|
probe, probeErr := cli.ProbeProxyEndpoint()
|
|
if probeErr == nil {
|
|
return validatedProxy{
|
|
url: proxyURL,
|
|
client: cli,
|
|
rtt: probe.Latency,
|
|
source: source,
|
|
}, nil
|
|
}
|
|
lastErr = probeErr
|
|
}
|
|
if cli.HTTPClient != nil {
|
|
cli.HTTPClient.CloseIdleConnections()
|
|
}
|
|
return validatedProxy{}, lastErr
|
|
}
|
|
|
|
type proxyValidationResult struct {
|
|
endpoint proxyprovider.Endpoint
|
|
proxy validatedProxy
|
|
err error
|
|
}
|
|
|
|
func (s *Scheduler) validateProxyBatch(base *client.DouyuClient, endpoints []proxyprovider.Endpoint) []proxyValidationResult {
|
|
if len(endpoints) == 0 {
|
|
return nil
|
|
}
|
|
workers := proxyValidationWorkers
|
|
if len(endpoints) < workers {
|
|
workers = len(endpoints)
|
|
}
|
|
jobs := make(chan proxyprovider.Endpoint)
|
|
results := make(chan proxyValidationResult, len(endpoints))
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < workers; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for endpoint := range jobs {
|
|
proxy, err := s.validateProxy(base, endpoint.URL, "API/"+endpoint.Provider)
|
|
results <- proxyValidationResult{endpoint: endpoint, proxy: proxy, err: err}
|
|
}
|
|
}()
|
|
}
|
|
for _, endpoint := range endpoints {
|
|
jobs <- endpoint
|
|
}
|
|
close(jobs)
|
|
wg.Wait()
|
|
close(results)
|
|
|
|
out := make([]proxyValidationResult, 0, len(endpoints))
|
|
for result := range results {
|
|
out = append(out, result)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func accountProductsForTasks(tasks []types.ExchangeAttempt) map[string]string {
|
|
out := make(map[string]string, len(tasks))
|
|
for _, task := range tasks {
|
|
out[task.Account.Name] = task.Product.CommodityID
|
|
}
|
|
return out
|
|
}
|