feat: 优化代理准备并完善远端部署
- 增加严格代理、两分钟预取、白名单去重与分组状态展示\n- 增加远端注册令牌和安全地址校验\n- 增加 Docker Compose、Caddy 示例与控制中心部署文档
This commit is contained in:
@@ -98,7 +98,11 @@ func (p *GenericProvider) Parse(apiURL string, body []byte) ([]Endpoint, error)
|
||||
}
|
||||
|
||||
func isGenericFatalMessage(message string) bool {
|
||||
if strings.Contains(message, "提取频率过快") || strings.Contains(message, "请求过于频繁") {
|
||||
if strings.Contains(message, "提取频率过快") ||
|
||||
strings.Contains(message, "请求过于频繁") ||
|
||||
strings.Contains(message, "访问频率过快") ||
|
||||
strings.Contains(message, "白名单仅支持1秒") ||
|
||||
strings.Contains(message, "白名单仅支持 1 秒") {
|
||||
return false
|
||||
}
|
||||
for _, hint := range []string{
|
||||
|
||||
+55
-9
@@ -14,16 +14,20 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const whitelistRequestGap = 1200 * time.Millisecond
|
||||
|
||||
// IPZanProvider 接入品赞 https://service.ipzan.com/core-extract。
|
||||
type IPZanProvider struct {
|
||||
baseURL string
|
||||
now func() time.Time
|
||||
baseURL string
|
||||
now func() time.Time
|
||||
whitelistGap time.Duration
|
||||
}
|
||||
|
||||
func NewIPZanProvider() *IPZanProvider {
|
||||
return &IPZanProvider{
|
||||
baseURL: "https://service.ipzan.com",
|
||||
now: time.Now,
|
||||
baseURL: "https://service.ipzan.com",
|
||||
now: time.Now,
|
||||
whitelistGap: whitelistRequestGap,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,10 +48,8 @@ func (p *IPZanProvider) RequestURL(apiURL string, count int) (string, error) {
|
||||
}
|
||||
query := u.Query()
|
||||
query.Set("num", strconv.Itoa(count))
|
||||
if strings.EqualFold(strings.TrimSpace(query.Get("mode")), "auth") {
|
||||
// txt 只返回 IP:端口,会丢失每个代理各自的账号密码,随后必然触发 407。
|
||||
query.Set("format", "json")
|
||||
}
|
||||
// JSON 才能保留每个代理独立的账号密码;txt 只返回 IP:端口,鉴权模式会触发 407。
|
||||
query.Set("format", "json")
|
||||
u.RawQuery = query.Encode()
|
||||
return u.String(), nil
|
||||
}
|
||||
@@ -160,6 +162,25 @@ func (p *IPZanProvider) AddWhitelist(ctx context.Context, httpClient *http.Clien
|
||||
if strings.TrimSpace(cfg.LoginPassword) == "" || strings.TrimSpace(cfg.SignKey) == "" {
|
||||
return &ProviderError{Provider: "品赞", Message: "自动白名单需要品赞登录密码和签名密钥", Fatal: true}
|
||||
}
|
||||
guard, err := acquireWhitelistLock(ctx, p.baseURL, no, cfg.IP)
|
||||
if err != nil {
|
||||
return fmt.Errorf("等待本机白名单协调锁失败: %w", err)
|
||||
}
|
||||
defer guard.release()
|
||||
if guard.recentlyAdded() {
|
||||
return nil
|
||||
}
|
||||
exists, err := p.verifyWhitelist(ctx, httpClient, no, cfg.IP)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
guard.markAdded()
|
||||
return nil
|
||||
}
|
||||
if err := p.waitWhitelistSlot(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
sign, err := p.signWhitelist(cfg.LoginPassword, secret, cfg.SignKey)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -186,9 +207,24 @@ func (p *IPZanProvider) AddWhitelist(ctx context.Context, httpClient *http.Clien
|
||||
if err := p.doWhitelistRequest(req, httpClient, "添加白名单"); err != nil {
|
||||
return err
|
||||
}
|
||||
guard.markAdded()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) waitWhitelistSlot(ctx context.Context) error {
|
||||
if p.whitelistGap <= 0 {
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(p.whitelistGap)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) verifyWhitelist(ctx context.Context, httpClient *http.Client, no, ip string) (bool, error) {
|
||||
endpoint, err := url.Parse(strings.TrimRight(p.baseURL, "/") + "/whiteList-verify")
|
||||
if err != nil {
|
||||
@@ -297,7 +333,17 @@ func (p *IPZanProvider) doWhitelistRequestData(req *http.Request, httpClient *ht
|
||||
}
|
||||
|
||||
func isIPZanFatalMessage(message string) bool {
|
||||
if strings.Contains(message, "提取频率过快") {
|
||||
for _, retryable := range []string{
|
||||
"提取频率过快",
|
||||
"访问频率过快",
|
||||
"白名单仅支持1秒",
|
||||
"白名单仅支持 1 秒",
|
||||
} {
|
||||
if strings.Contains(message, retryable) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if strings.Contains(message, "白名单") && strings.Contains(message, "频繁") {
|
||||
return false
|
||||
}
|
||||
for _, hint := range []string{
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIPZanRequestURLAlwaysUsesJSON(t *testing.T) {
|
||||
provider := NewIPZanProvider()
|
||||
raw := "https://service.ipzan.com/core-extract?mode=auth&format=txt&num=1"
|
||||
got, err := provider.RequestURL(raw, 5)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u, err := url.Parse(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if format := u.Query().Get("format"); format != "json" {
|
||||
t.Fatalf("format=%q, want json", format)
|
||||
}
|
||||
if num := u.Query().Get("num"); num != "5" {
|
||||
t.Fatalf("num=%q, want 5", num)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPZanWhitelistRateLimitIsRetryable(t *testing.T) {
|
||||
for _, message := range []string{
|
||||
"访问频率过快,休息一下 白名单仅支持1秒访一次",
|
||||
"添加白名单失败: 访问频率过快",
|
||||
"提取频率过快",
|
||||
} {
|
||||
if isIPZanFatalMessage(message) {
|
||||
t.Fatalf("%q should be retryable", message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhitelistGuardSerializesAndRemembersSuccessfulAdd(t *testing.T) {
|
||||
const (
|
||||
baseURL = "https://example.test"
|
||||
no = "guard-test"
|
||||
ip = "203.0.113.99"
|
||||
)
|
||||
first, err := acquireWhitelistLock(context.Background(), baseURL, no, ip)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.markAdded()
|
||||
t.Cleanup(func() {
|
||||
_ = os.Remove(first.path)
|
||||
_ = os.Remove(first.markerPath)
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := acquireWhitelistLock(ctx, baseURL, no, ip); err == nil {
|
||||
t.Fatal("second process should wait while the first guard is held")
|
||||
}
|
||||
first.release()
|
||||
|
||||
second, err := acquireWhitelistLock(context.Background(), baseURL, no, ip)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer second.release()
|
||||
if !second.recentlyAdded() {
|
||||
t.Fatal("successful whitelist add should be reused by sibling instances")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
const whitelistLockStaleAfter = 2 * time.Minute
|
||||
const whitelistMarkerTTL = 10 * time.Minute
|
||||
|
||||
type whitelistGuard struct {
|
||||
path string
|
||||
markerPath string
|
||||
}
|
||||
|
||||
func acquireWhitelistLock(ctx context.Context, baseURL, no, ip string) (*whitelistGuard, error) {
|
||||
sum := sha256.Sum256([]byte(baseURL + "\x00" + no + "\x00" + ip))
|
||||
dir := filepath.Join(os.TempDir(), "douyu-go-whitelist")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := hex.EncodeToString(sum[:16])
|
||||
path := filepath.Join(dir, key+".lock")
|
||||
guard := &whitelistGuard{
|
||||
path: path,
|
||||
markerPath: filepath.Join(dir, key+".ok"),
|
||||
}
|
||||
for {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err == nil {
|
||||
_, _ = fmt.Fprintf(file, "%d\n%s\n", os.Getpid(), time.Now().Format(time.RFC3339Nano))
|
||||
_ = file.Close()
|
||||
return guard, nil
|
||||
}
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
return nil, err
|
||||
}
|
||||
if info, statErr := os.Stat(path); statErr == nil && time.Since(info.ModTime()) > whitelistLockStaleAfter {
|
||||
_ = os.Remove(path)
|
||||
continue
|
||||
}
|
||||
timer := time.NewTimer(200 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (g *whitelistGuard) recentlyAdded() bool {
|
||||
if g == nil {
|
||||
return false
|
||||
}
|
||||
info, err := os.Stat(g.markerPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
age := time.Since(info.ModTime())
|
||||
return age >= 0 && age < whitelistMarkerTTL
|
||||
}
|
||||
|
||||
func (g *whitelistGuard) markAdded() {
|
||||
if g == nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(g.markerPath, []byte(time.Now().Format(time.RFC3339Nano)), 0o600)
|
||||
}
|
||||
|
||||
func (g *whitelistGuard) release() {
|
||||
if g != nil {
|
||||
_ = os.Remove(g.path)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user