feat(proxy): 支持品赞分组代理与自动白名单
This commit is contained in:
@@ -0,0 +1,250 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
proxyTextRegex = regexp.MustCompile(`(?i)(?:(https?|socks5)://)?(?:[^@\s:/]+:[^@\s/]+@)?((?:\d{1,3}\.){3}\d{1,3}):(\d{2,5})`)
|
||||
)
|
||||
|
||||
// GenericProvider 兼容常见 JSON 与纯文本 IP:端口 返回。
|
||||
type GenericProvider struct{}
|
||||
|
||||
func NewGenericProvider() *GenericProvider {
|
||||
return &GenericProvider{}
|
||||
}
|
||||
|
||||
func (p *GenericProvider) ID() string {
|
||||
return "generic"
|
||||
}
|
||||
|
||||
func (p *GenericProvider) Match(u *url.URL) bool {
|
||||
return u != nil
|
||||
}
|
||||
|
||||
func (p *GenericProvider) RequestURL(apiURL string, count int) (string, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(apiURL))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("代理 API 地址无效: %w", err)
|
||||
}
|
||||
query := u.Query()
|
||||
changed := false
|
||||
for _, key := range []string{"num", "count", "quantity"} {
|
||||
if query.Has(key) {
|
||||
query.Set(key, strconv.Itoa(count))
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
u.RawQuery = query.Encode()
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (p *GenericProvider) Parse(apiURL string, body []byte) ([]Endpoint, error) {
|
||||
text := strings.TrimSpace(string(body))
|
||||
if text == "" {
|
||||
return nil, errors.New("代理 API 返回内容为空")
|
||||
}
|
||||
protocol := protocolFromAPI(apiURL)
|
||||
|
||||
var payload struct {
|
||||
Code *int `json:"code"`
|
||||
Status int `json:"status"`
|
||||
Info string `json:"info"`
|
||||
Msg string `json:"msg"`
|
||||
Message string `json:"message"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(body, &payload) == nil {
|
||||
message := firstNonEmpty(payload.Info, payload.Msg, payload.Message)
|
||||
hasError := (payload.Code != nil && *payload.Code != 0) ||
|
||||
(payload.Code == nil && payload.Status != 0 && payload.Status != 100 && payload.Status != 200)
|
||||
if hasError {
|
||||
if message == "" {
|
||||
message = "代理平台返回错误"
|
||||
}
|
||||
return nil, &ProviderError{
|
||||
Provider: "通用代理",
|
||||
Message: message,
|
||||
Fatal: isGenericFatalMessage(message),
|
||||
}
|
||||
}
|
||||
if len(payload.Data) > 0 {
|
||||
endpoints := parseEndpointJSON(payload.Data, protocol)
|
||||
if len(endpoints) > 0 {
|
||||
return endpoints, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
endpoints := parseEndpointJSON(body, protocol)
|
||||
if len(endpoints) > 0 {
|
||||
return endpoints, nil
|
||||
}
|
||||
endpoints = parseEndpointText(text, protocol)
|
||||
if len(endpoints) > 0 {
|
||||
return endpoints, nil
|
||||
}
|
||||
return nil, fmt.Errorf("未能从代理 API 返回中识别出有效 IP:端口: %s", truncateText(text, 120))
|
||||
}
|
||||
|
||||
func isGenericFatalMessage(message string) bool {
|
||||
if strings.Contains(message, "提取频率过快") || strings.Contains(message, "请求过于频繁") {
|
||||
return false
|
||||
}
|
||||
for _, hint := range []string{
|
||||
"白名单", "认证", "鉴权", "密钥", "密匙", "套餐", "过期", "禁用", "余额", "未授权",
|
||||
} {
|
||||
if strings.Contains(strings.ToLower(message), strings.ToLower(hint)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseEndpointJSON(raw []byte, protocol string) []Endpoint {
|
||||
var value any
|
||||
if json.Unmarshal(raw, &value) != nil {
|
||||
return nil
|
||||
}
|
||||
return collectJSONEndpoints(value, protocol)
|
||||
}
|
||||
|
||||
func collectJSONEndpoints(value any, protocol string) []Endpoint {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return parseEndpointText(typed, protocol)
|
||||
case []any:
|
||||
var out []Endpoint
|
||||
for _, item := range typed {
|
||||
out = append(out, collectJSONEndpoints(item, protocol)...)
|
||||
}
|
||||
return out
|
||||
case map[string]any:
|
||||
if endpoints := endpointsFromMap(typed, protocol); len(endpoints) > 0 {
|
||||
return endpoints
|
||||
}
|
||||
var out []Endpoint
|
||||
for _, key := range []string{"data", "list", "items", "result", "results", "proxies"} {
|
||||
if child, ok := typed[key]; ok {
|
||||
out = append(out, collectJSONEndpoints(child, protocol)...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func endpointsFromMap(item map[string]any, protocol string) []Endpoint {
|
||||
ip := stringValue(item["ip"], item["host"], item["server"])
|
||||
port := stringValue(item["port"], item["proxy_port"])
|
||||
if ip == "" || port == "" {
|
||||
return nil
|
||||
}
|
||||
itemProtocol := stringValue(item["protocol"], item["scheme"])
|
||||
if itemProtocol == "" {
|
||||
itemProtocol = protocol
|
||||
}
|
||||
username := stringValue(item["username"], item["user"], item["account"])
|
||||
password := stringValue(item["password"], item["pass"])
|
||||
expiresAt := timeValue(item["expired"], item["expire"], item["expiresAt"], item["expireAt"])
|
||||
region := strings.TrimSpace(strings.Join([]string{
|
||||
stringValue(item["prov"], item["province"]),
|
||||
stringValue(item["city"]),
|
||||
stringValue(item["region"]),
|
||||
}, " "))
|
||||
return []Endpoint{{
|
||||
URL: buildProxyURL(itemProtocol, ip, port, username, password),
|
||||
IP: ip,
|
||||
Port: port,
|
||||
Region: region,
|
||||
Protocol: itemProtocol,
|
||||
Username: username,
|
||||
Password: password,
|
||||
ExpiresAt: expiresAt,
|
||||
}}
|
||||
}
|
||||
|
||||
func parseEndpointText(text, protocol string) []Endpoint {
|
||||
matches := proxyTextRegex.FindAllStringSubmatch(text, -1)
|
||||
out := make([]Endpoint, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
if len(match) != 4 {
|
||||
continue
|
||||
}
|
||||
matchProtocol := match[1]
|
||||
if matchProtocol == "" {
|
||||
matchProtocol = protocol
|
||||
}
|
||||
ip, port := match[2], match[3]
|
||||
out = append(out, Endpoint{
|
||||
URL: buildProxyURL(matchProtocol, ip, port, "", ""),
|
||||
IP: ip,
|
||||
Port: port,
|
||||
Protocol: matchProtocol,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stringValue(values ...any) string {
|
||||
for _, value := range values {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
if strings.TrimSpace(typed) != "" {
|
||||
return strings.TrimSpace(typed)
|
||||
}
|
||||
case float64:
|
||||
return fmt.Sprintf("%.0f", typed)
|
||||
case json.Number:
|
||||
return typed.String()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func timeValue(values ...any) time.Time {
|
||||
for _, value := range values {
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return unixTimestamp(typed)
|
||||
case json.Number:
|
||||
number, err := typed.Float64()
|
||||
if err == nil {
|
||||
return unixTimestamp(number)
|
||||
}
|
||||
case string:
|
||||
typed = strings.TrimSpace(typed)
|
||||
if typed == "" {
|
||||
continue
|
||||
}
|
||||
if number, err := strconv.ParseFloat(typed, 64); err == nil {
|
||||
return unixTimestamp(number)
|
||||
}
|
||||
if parsed, err := time.Parse(time.RFC3339, typed); err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func unixTimestamp(value float64) time.Time {
|
||||
if value <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
if value >= 1e12 {
|
||||
return time.UnixMilli(int64(value))
|
||||
}
|
||||
return time.Unix(int64(value), 0)
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// IPZanProvider 接入品赞 https://service.ipzan.com/core-extract。
|
||||
type IPZanProvider struct {
|
||||
baseURL string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewIPZanProvider() *IPZanProvider {
|
||||
return &IPZanProvider{
|
||||
baseURL: "https://service.ipzan.com",
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) ID() string {
|
||||
return "ipzan"
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) Match(u *url.URL) bool {
|
||||
return u != nil &&
|
||||
strings.EqualFold(u.Hostname(), "service.ipzan.com") &&
|
||||
strings.TrimRight(u.Path, "/") == "/core-extract"
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) RequestURL(apiURL string, count int) (string, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(apiURL))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("品赞 API 地址无效: %w", err)
|
||||
}
|
||||
query := u.Query()
|
||||
query.Set("num", strconv.Itoa(count))
|
||||
u.RawQuery = query.Encode()
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) Parse(apiURL string, body []byte) ([]Endpoint, error) {
|
||||
text := strings.TrimSpace(string(body))
|
||||
if text == "" {
|
||||
return nil, &ProviderError{Provider: "品赞", Message: "API 返回内容为空", Fatal: false}
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Code *int `json:"code"`
|
||||
Status int `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Info string `json:"info"`
|
||||
Msg string `json:"msg"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
if json.Unmarshal(body, &payload) == nil {
|
||||
if payload.Code != nil && *payload.Code != 0 {
|
||||
message := firstNonEmpty(payload.Message, payload.Info, payload.Msg, "品赞返回未知错误")
|
||||
return nil, &ProviderError{
|
||||
Provider: "品赞",
|
||||
Message: fmt.Sprintf("%s (code=%d)", message, *payload.Code),
|
||||
Fatal: isIPZanFatalMessage(message),
|
||||
}
|
||||
}
|
||||
if len(payload.Data) > 0 {
|
||||
endpoints := parseEndpointJSON(payload.Data, protocolFromAPI(apiURL))
|
||||
if len(endpoints) > 0 {
|
||||
return endpoints, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
endpoints := parseEndpointText(text, protocolFromAPI(apiURL))
|
||||
if len(endpoints) == 0 {
|
||||
return nil, &ProviderError{
|
||||
Provider: "品赞",
|
||||
Message: "未能识别有效 IP:端口,原始返回: " + truncateText(text, 120),
|
||||
Fatal: false,
|
||||
}
|
||||
}
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) EnsureWhitelist(ctx context.Context, httpClient *http.Client, cfg WhitelistConfig) error {
|
||||
if strings.TrimSpace(cfg.IP) == "" {
|
||||
return &ProviderError{Provider: "品赞", Message: "自动白名单缺少公网 IP", Fatal: true}
|
||||
}
|
||||
apiURL, err := url.Parse(strings.TrimSpace(cfg.APIURL))
|
||||
if err != nil {
|
||||
return fmt.Errorf("品赞 API 地址无效: %w", err)
|
||||
}
|
||||
no := strings.TrimSpace(apiURL.Query().Get("no"))
|
||||
secret := strings.TrimSpace(apiURL.Query().Get("secret"))
|
||||
if no == "" || secret == "" {
|
||||
return &ProviderError{Provider: "品赞", Message: "提取 API 缺少 no 或 secret,无法自动加白名单", Fatal: true}
|
||||
}
|
||||
if strings.TrimSpace(cfg.LoginPassword) == "" || strings.TrimSpace(cfg.SignKey) == "" {
|
||||
return &ProviderError{Provider: "品赞", Message: "自动白名单需要品赞登录密码和签名密钥", Fatal: true}
|
||||
}
|
||||
|
||||
exists, err := p.verifyWhitelist(ctx, httpClient, no, cfg.IP)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
return p.AddWhitelist(ctx, httpClient, cfg)
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) AddWhitelist(ctx context.Context, httpClient *http.Client, cfg WhitelistConfig) error {
|
||||
if strings.TrimSpace(cfg.IP) == "" {
|
||||
return &ProviderError{Provider: "品赞", Message: "自动白名单缺少公网 IP", Fatal: true}
|
||||
}
|
||||
apiURL, err := url.Parse(strings.TrimSpace(cfg.APIURL))
|
||||
if err != nil {
|
||||
return fmt.Errorf("品赞 API 地址无效: %w", err)
|
||||
}
|
||||
no := strings.TrimSpace(apiURL.Query().Get("no"))
|
||||
secret := strings.TrimSpace(apiURL.Query().Get("secret"))
|
||||
if no == "" || secret == "" {
|
||||
return &ProviderError{Provider: "品赞", Message: "提取 API 缺少 no 或 secret,无法自动加白名单", Fatal: true}
|
||||
}
|
||||
if strings.TrimSpace(cfg.LoginPassword) == "" || strings.TrimSpace(cfg.SignKey) == "" {
|
||||
return &ProviderError{Provider: "品赞", Message: "自动白名单需要品赞登录密码和签名密钥", Fatal: true}
|
||||
}
|
||||
sign, err := p.signWhitelist(cfg.LoginPassword, secret, cfg.SignKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
form := url.Values{}
|
||||
form.Set("no", no)
|
||||
form.Set("sign", sign)
|
||||
form.Set("ip", cfg.IP)
|
||||
if cfg.ReplaceOldest {
|
||||
form.Set("replace", "1")
|
||||
} else {
|
||||
form.Set("replace", "0")
|
||||
}
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
strings.TrimRight(p.baseURL, "/")+"/whiteList-add",
|
||||
strings.NewReader(form.Encode()),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("创建品赞白名单请求失败: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
if err := p.doWhitelistRequest(req, httpClient, "添加白名单"); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) verifyWhitelist(ctx context.Context, httpClient *http.Client, no, ip string) (bool, error) {
|
||||
endpoint, err := url.Parse(strings.TrimRight(p.baseURL, "/") + "/whiteList-verify")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
query := endpoint.Query()
|
||||
query.Set("no", no)
|
||||
query.Set("ip", ip)
|
||||
endpoint.RawQuery = query.Encode()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("创建品赞白名单校验请求失败: %w", err)
|
||||
}
|
||||
body, err := p.doWhitelistRequestData(req, httpClient, "校验白名单")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var payload struct {
|
||||
Code *int `json:"code"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &payload); err != nil {
|
||||
return false, fmt.Errorf("解析品赞白名单校验响应失败: %w", err)
|
||||
}
|
||||
if payload.Code == nil || *payload.Code != 0 {
|
||||
return false, &ProviderError{Provider: "品赞", Message: "白名单校验失败", Fatal: true}
|
||||
}
|
||||
var exists bool
|
||||
if json.Unmarshal(payload.Data, &exists) == nil {
|
||||
return exists, nil
|
||||
}
|
||||
var existsText string
|
||||
if json.Unmarshal(payload.Data, &existsText) == nil {
|
||||
return strings.EqualFold(strings.TrimSpace(existsText), "true"), nil
|
||||
}
|
||||
return false, fmt.Errorf("品赞白名单校验响应格式无法识别: %s", truncateText(string(body), 120))
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) signWhitelist(loginPassword, secret, signKey string) (string, error) {
|
||||
key := []byte(signKey)
|
||||
switch len(key) {
|
||||
case 16, 24, 32:
|
||||
default:
|
||||
return "", &ProviderError{
|
||||
Provider: "品赞",
|
||||
Message: fmt.Sprintf("签名密钥长度无效: %d,应为 16/24/32 字节", len(key)),
|
||||
Fatal: true,
|
||||
}
|
||||
}
|
||||
plain := fmt.Sprintf("%s:%s:%d", loginPassword, secret, p.now().Unix())
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("初始化 AES 签名失败: %w", err)
|
||||
}
|
||||
padding := block.BlockSize() - len(plain)%block.BlockSize()
|
||||
data := make([]byte, 0, len(plain)+padding)
|
||||
data = append(data, []byte(plain)...)
|
||||
for i := 0; i < padding; i++ {
|
||||
data = append(data, byte(padding))
|
||||
}
|
||||
encrypted := make([]byte, len(data))
|
||||
for offset := 0; offset < len(data); offset += block.BlockSize() {
|
||||
block.Encrypt(encrypted[offset:offset+block.BlockSize()], data[offset:offset+block.BlockSize()])
|
||||
}
|
||||
return hex.EncodeToString(encrypted), nil
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) doWhitelistRequest(req *http.Request, httpClient *http.Client, action string) error {
|
||||
body, err := p.doWhitelistRequestData(req, httpClient, action)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var payload struct {
|
||||
Code *int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &payload); err != nil {
|
||||
return fmt.Errorf("解析品赞%s响应失败: %w", action, err)
|
||||
}
|
||||
if payload.Code == nil || *payload.Code != 0 {
|
||||
message := firstNonEmpty(payload.Message, "品赞返回未知错误")
|
||||
return &ProviderError{
|
||||
Provider: "品赞",
|
||||
Message: action + "失败: " + message,
|
||||
Fatal: isIPZanFatalMessage(message),
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *IPZanProvider) doWhitelistRequestData(req *http.Request, httpClient *http.Client, action string) ([]byte, error) {
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求品赞%s接口失败: %w", action, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取品赞%s响应失败: %w", action, err)
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
return nil, fmt.Errorf("品赞%s接口 HTTP %d: %s", action, resp.StatusCode, truncateText(string(body), 120))
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func isIPZanFatalMessage(message string) bool {
|
||||
if strings.Contains(message, "提取频率过快") {
|
||||
return false
|
||||
}
|
||||
for _, hint := range []string{
|
||||
"白名单", "secret", "密匙", "密钥", "套餐", "过期", "禁用", "余额", "身份未认证", "用户被禁用",
|
||||
} {
|
||||
if strings.Contains(strings.ToLower(message), strings.ToLower(hint)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var whitelistIPRegex = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
|
||||
|
||||
// Endpoint 是代理提供商返回的一条可连接代理。
|
||||
type Endpoint struct {
|
||||
URL string
|
||||
IP string
|
||||
Port string
|
||||
Region string
|
||||
Protocol string
|
||||
Username string
|
||||
Password string
|
||||
Provider string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// Provider 把不同代理平台的参数规则与响应格式隔离在各自实现中。
|
||||
type Provider interface {
|
||||
ID() string
|
||||
Match(*url.URL) bool
|
||||
RequestURL(apiURL string, count int) (string, error)
|
||||
Parse(apiURL string, body []byte) ([]Endpoint, error)
|
||||
}
|
||||
|
||||
// WhitelistConfig 是代理平台自动白名单所需参数。
|
||||
type WhitelistConfig struct {
|
||||
APIURL string
|
||||
IP string
|
||||
LoginPassword string
|
||||
SignKey string
|
||||
ReplaceOldest bool
|
||||
AutoWhitelist bool
|
||||
}
|
||||
|
||||
// WhitelistProvider 由支持自动白名单的代理平台实现。
|
||||
type WhitelistProvider interface {
|
||||
Provider
|
||||
EnsureWhitelist(context.Context, *http.Client, WhitelistConfig) error
|
||||
AddWhitelist(context.Context, *http.Client, WhitelistConfig) error
|
||||
}
|
||||
|
||||
// ProviderError 是代理平台返回的业务错误。
|
||||
type ProviderError struct {
|
||||
Provider string
|
||||
Message string
|
||||
Fatal bool
|
||||
}
|
||||
|
||||
func (e *ProviderError) Error() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
if e.Provider == "" {
|
||||
return e.Message
|
||||
}
|
||||
return e.Provider + ": " + e.Message
|
||||
}
|
||||
|
||||
// IsFatal 判断错误是否重试也不会恢复,例如密钥错误、白名单缺失或套餐过期。
|
||||
func IsFatal(err error) bool {
|
||||
var providerErr *ProviderError
|
||||
return errors.As(err, &providerErr) && providerErr.Fatal
|
||||
}
|
||||
|
||||
// Service 负责通用 HTTP 请求、provider 选择和结果去重。
|
||||
type Service struct {
|
||||
client *http.Client
|
||||
providers []Provider
|
||||
}
|
||||
|
||||
func NewService() *Service {
|
||||
return NewServiceWithProviders(
|
||||
&http.Client{Timeout: 10 * time.Second},
|
||||
NewIPZanProvider(),
|
||||
NewGenericProvider(),
|
||||
)
|
||||
}
|
||||
|
||||
func NewServiceWithProviders(client *http.Client, providers ...Provider) *Service {
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 10 * time.Second}
|
||||
}
|
||||
return &Service{client: client, providers: providers}
|
||||
}
|
||||
|
||||
// Extract 自动识别代理平台并请求所需数量;调用方可用剩余数量再次提取补足。
|
||||
func (s *Service) Extract(ctx context.Context, apiURL string, count int) ([]Endpoint, error) {
|
||||
if count <= 0 {
|
||||
count = 1
|
||||
}
|
||||
u, err := url.Parse(strings.TrimSpace(apiURL))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("代理 API 地址无效: %w", err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, errors.New("代理 API 地址必须以 http:// 或 https:// 开头")
|
||||
}
|
||||
provider := s.providerFor(u)
|
||||
if provider == nil {
|
||||
return nil, errors.New("没有匹配的代理提供商")
|
||||
}
|
||||
|
||||
requestURL, err := provider.RequestURL(apiURL, count)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := s.fetch(ctx, requestURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endpoints, err := provider.Parse(apiURL, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range endpoints {
|
||||
if endpoints[i].Provider == "" {
|
||||
endpoints[i].Provider = provider.ID()
|
||||
}
|
||||
}
|
||||
endpoints = dedupeEndpoints(endpoints, count)
|
||||
if len(endpoints) == 0 {
|
||||
return nil, errors.New("代理 API 返回的代理均已过期或无效")
|
||||
}
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
func (s *Service) ExtractOne(ctx context.Context, apiURL string) (Endpoint, error) {
|
||||
endpoints, err := s.Extract(ctx, apiURL, 1)
|
||||
if err != nil {
|
||||
return Endpoint{}, err
|
||||
}
|
||||
if len(endpoints) == 0 {
|
||||
return Endpoint{}, errors.New("代理 API 未返回有效地址")
|
||||
}
|
||||
return endpoints[0], nil
|
||||
}
|
||||
|
||||
// EnsureWhitelist 自动把指定公网 IP 加入当前 API 套餐白名单;已存在时直接跳过。
|
||||
func (s *Service) EnsureWhitelist(ctx context.Context, cfg WhitelistConfig) error {
|
||||
u, err := url.Parse(strings.TrimSpace(cfg.APIURL))
|
||||
if err != nil {
|
||||
return fmt.Errorf("代理 API 地址无效: %w", err)
|
||||
}
|
||||
provider := s.providerFor(u)
|
||||
manager, ok := provider.(WhitelistProvider)
|
||||
if !ok {
|
||||
return errors.New("当前代理提供商不支持自动白名单")
|
||||
}
|
||||
return manager.EnsureWhitelist(ctx, s.client, cfg)
|
||||
}
|
||||
|
||||
// AddWhitelist 直接添加白名单,不再调用独立的 verify 接口。
|
||||
func (s *Service) AddWhitelist(ctx context.Context, cfg WhitelistConfig) error {
|
||||
u, err := url.Parse(strings.TrimSpace(cfg.APIURL))
|
||||
if err != nil {
|
||||
return fmt.Errorf("代理 API 地址无效: %w", err)
|
||||
}
|
||||
provider := s.providerFor(u)
|
||||
manager, ok := provider.(WhitelistProvider)
|
||||
if !ok {
|
||||
return errors.New("当前代理提供商不支持自动白名单")
|
||||
}
|
||||
return manager.AddWhitelist(ctx, s.client, cfg)
|
||||
}
|
||||
|
||||
// ExtractWithWhitelist 先探测提取;若平台提示当前 IP 不在白名单,则直接使用错误中
|
||||
// 平台实际看到的来源 IP,添加后重试,避免第三方公网 IP 探测与实际出口不一致。
|
||||
func (s *Service) ExtractWithWhitelist(ctx context.Context, apiURL string, count int, cfg WhitelistConfig) ([]Endpoint, error) {
|
||||
endpoints, err := s.Extract(ctx, apiURL, count)
|
||||
if err == nil || !cfg.AutoWhitelist {
|
||||
return endpoints, err
|
||||
}
|
||||
ip, ok := whitelistIPFromError(err)
|
||||
if !ok {
|
||||
return nil, err
|
||||
}
|
||||
cfg.APIURL = apiURL
|
||||
cfg.IP = ip
|
||||
if addErr := s.AddWhitelist(ctx, cfg); addErr != nil {
|
||||
return nil, fmt.Errorf("品赞要求添加白名单 IP %s,自动添加失败: %w", ip, addErr)
|
||||
}
|
||||
return s.Extract(ctx, apiURL, count)
|
||||
}
|
||||
|
||||
func whitelistIPFromError(err error) (string, bool) {
|
||||
if err == nil || !strings.Contains(err.Error(), "白名单") {
|
||||
return "", false
|
||||
}
|
||||
ip := whitelistIPRegex.FindString(err.Error())
|
||||
return ip, ip != ""
|
||||
}
|
||||
|
||||
func (s *Service) providerFor(u *url.URL) Provider {
|
||||
for _, provider := range s.providers {
|
||||
if provider != nil && provider.Match(u) {
|
||||
return provider
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) fetch(ctx context.Context, rawURL string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建代理 API 请求失败: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("请求代理 API 失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 128*1024))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取代理 API 响应失败: %w", err)
|
||||
}
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
return nil, fmt.Errorf("代理 API HTTP %d: %s", resp.StatusCode, truncateText(string(body), 160))
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func dedupeEndpoints(endpoints []Endpoint, limit int) []Endpoint {
|
||||
out := make([]Endpoint, 0, min(len(endpoints), limit))
|
||||
seen := make(map[string]struct{}, len(endpoints))
|
||||
for _, endpoint := range endpoints {
|
||||
endpoint.URL = strings.TrimSpace(endpoint.URL)
|
||||
if endpoint.URL == "" {
|
||||
continue
|
||||
}
|
||||
if !endpoint.ExpiresAt.IsZero() && !time.Now().Before(endpoint.ExpiresAt) {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[endpoint.URL]; ok {
|
||||
continue
|
||||
}
|
||||
seen[endpoint.URL] = struct{}{}
|
||||
out = append(out, endpoint)
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func buildProxyURL(protocol, ip, port, username, password string) string {
|
||||
scheme := "http"
|
||||
if protocol == "3" || strings.EqualFold(protocol, "socks5") || strings.EqualFold(protocol, "socket5") {
|
||||
scheme = "socks5"
|
||||
}
|
||||
auth := ""
|
||||
if username != "" || password != "" {
|
||||
auth = url.UserPassword(username, password).String() + "@"
|
||||
}
|
||||
return fmt.Sprintf("%s://%s%s:%s", scheme, auth, strings.TrimSpace(ip), strings.TrimSpace(port))
|
||||
}
|
||||
|
||||
func protocolFromAPI(apiURL string) string {
|
||||
u, err := url.Parse(apiURL)
|
||||
if err != nil {
|
||||
return "1"
|
||||
}
|
||||
if value := strings.TrimSpace(u.Query().Get("protocol")); value != "" {
|
||||
return value
|
||||
}
|
||||
return "1"
|
||||
}
|
||||
|
||||
func truncateText(value string, maxLen int) string {
|
||||
runes := []rune(strings.TrimSpace(value))
|
||||
if len(runes) <= maxLen {
|
||||
return string(runes)
|
||||
}
|
||||
return string(runes[:maxLen]) + "..."
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIPZanRequestURLSetsRequestedCount(t *testing.T) {
|
||||
raw := "https://service.ipzan.com/core-extract?num=1&minute=10&protocol=1&secret=x"
|
||||
got, err := NewIPZanProvider().RequestURL(raw, 6)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(got, "num=6") || strings.Contains(got, "num=1") {
|
||||
t.Fatalf("RequestURL() = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPZanParseTextMultiple(t *testing.T) {
|
||||
body := []byte("1.2.3.4:8001\r\n5.6.7.8:8002\r\n")
|
||||
got, err := NewIPZanProvider().Parse(
|
||||
"https://service.ipzan.com/core-extract?protocol=3",
|
||||
body,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("len=%d, want 2", len(got))
|
||||
}
|
||||
if got[0].URL != "socks5://1.2.3.4:8001" {
|
||||
t.Fatalf("URL=%q", got[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPZanParseJSONError(t *testing.T) {
|
||||
body := []byte(`{"code":-1,"data":null,"status":200,"message":"请先将1.2.3.4加入到白名单再进行提取"}`)
|
||||
_, err := NewIPZanProvider().Parse("https://service.ipzan.com/core-extract", body)
|
||||
if err == nil || !IsFatal(err) {
|
||||
t.Fatalf("expected fatal provider error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPZanParseJSONList(t *testing.T) {
|
||||
body := []byte(`{"code":0,"status":200,"data":{"list":[{"ip":"1.2.3.4","port":8001,"account":"user-a","password":"pass-a","expired":1893456000000},{"ip":"5.6.7.8","port":"8002"}]}}`)
|
||||
got, err := NewIPZanProvider().Parse(
|
||||
"https://service.ipzan.com/core-extract?protocol=1",
|
||||
body,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 || got[0].URL != "http://user-a:pass-a@1.2.3.4:8001" || got[1].URL != "http://5.6.7.8:8002" {
|
||||
t.Fatalf("unexpected endpoints: %#v", got)
|
||||
}
|
||||
if got[0].ExpiresAt != time.UnixMilli(1893456000000) {
|
||||
t.Fatalf("ExpiresAt=%v", got[0].ExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenericParseJSONListWithAuth(t *testing.T) {
|
||||
body := []byte(`{"code":0,"data":{"list":[{"ip":"1.2.3.4","port":9000,"username":"u","password":"p"}]}}`)
|
||||
got, err := NewGenericProvider().Parse("https://example.test/api", body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].URL != "http://u:p@1.2.3.4:9000" {
|
||||
t.Fatalf("unexpected endpoints: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenericParseIPZanAccountPasswordSample(t *testing.T) {
|
||||
body := []byte(`{"data":{"list":[{"ip":"203.0.113.10","port":"15471","expired":1893456000000,"net":"电信","account":"test-account","password":"test-password"}]},"code":0,"message":"","status":200}`)
|
||||
got, err := NewGenericProvider().Parse("https://example.test/api?protocol=1", body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("len=%d, want 1", len(got))
|
||||
}
|
||||
if got[0].URL != "http://test-account:test-password@203.0.113.10:15471" {
|
||||
t.Fatalf("URL=%q", got[0].URL)
|
||||
}
|
||||
if got[0].Username != "test-account" || got[0].Password != "test-password" {
|
||||
t.Fatalf("credentials not parsed: %#v", got[0])
|
||||
}
|
||||
if got[0].ExpiresAt != time.UnixMilli(1893456000000) {
|
||||
t.Fatalf("ExpiresAt=%v", got[0].ExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenericProviderDoesNotTreatErrorIPAsProxy(t *testing.T) {
|
||||
body := []byte(`{"code":-1,"status":200,"message":"请先将1.2.3.4加入白名单"}`)
|
||||
_, err := NewGenericProvider().Parse("https://example.test/api", body)
|
||||
if err == nil || !IsFatal(err) {
|
||||
t.Fatalf("expected fatal provider error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenericProviderParsesLegacyJSONData(t *testing.T) {
|
||||
body := []byte(`{"status":100,"info":"成功","data":[{"ip":"1.2.3.4","port":"8001","prov":"浙江","city":"嘉兴"}]}`)
|
||||
got, err := NewGenericProvider().Parse("https://example.test/api", body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].URL != "http://1.2.3.4:8001" || got[0].Region != "浙江 嘉兴" {
|
||||
t.Fatalf("unexpected endpoints: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return fn(req)
|
||||
}
|
||||
|
||||
type autoWhitelistTestProvider struct {
|
||||
parseCalls int
|
||||
addedIP string
|
||||
}
|
||||
|
||||
func (p *autoWhitelistTestProvider) ID() string { return "test" }
|
||||
|
||||
func (p *autoWhitelistTestProvider) Match(*url.URL) bool { return true }
|
||||
|
||||
func (p *autoWhitelistTestProvider) RequestURL(apiURL string, count int) (string, error) {
|
||||
return apiURL, nil
|
||||
}
|
||||
|
||||
func (p *autoWhitelistTestProvider) Parse(string, []byte) ([]Endpoint, error) {
|
||||
p.parseCalls++
|
||||
if p.parseCalls == 1 {
|
||||
return nil, &ProviderError{
|
||||
Provider: "test",
|
||||
Message: "请先将1.2.3.4加入白名单再提取",
|
||||
Fatal: true,
|
||||
}
|
||||
}
|
||||
return []Endpoint{{URL: "http://1.2.3.4:9000", IP: "1.2.3.4"}}, nil
|
||||
}
|
||||
|
||||
func (p *autoWhitelistTestProvider) EnsureWhitelist(context.Context, *http.Client, WhitelistConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *autoWhitelistTestProvider) AddWhitelist(_ context.Context, _ *http.Client, cfg WhitelistConfig) error {
|
||||
p.addedIP = cfg.IP
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestExtractWithWhitelistUsesProviderReportedIP(t *testing.T) {
|
||||
provider := &autoWhitelistTestProvider{}
|
||||
client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: io.NopCloser(strings.NewReader("probe")),
|
||||
Header: make(http.Header),
|
||||
}, nil
|
||||
})}
|
||||
service := NewServiceWithProviders(client, provider)
|
||||
got, err := service.ExtractWithWhitelist(
|
||||
context.Background(),
|
||||
"https://example.test/extract",
|
||||
1,
|
||||
WhitelistConfig{AutoWhitelist: true},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if provider.addedIP != "1.2.3.4" {
|
||||
t.Fatalf("addedIP=%q", provider.addedIP)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("len=%d, want 1", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceSelectsIPZanProvider(t *testing.T) {
|
||||
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if got := req.URL.Query().Get("num"); got != "3" {
|
||||
t.Fatalf("num=%q, want 3", got)
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: io.NopCloser(strings.NewReader("1.2.3.4:8001\r\n5.6.7.8:8002\r\n9.9.9.9:8003")),
|
||||
Header: make(http.Header),
|
||||
}, nil
|
||||
})}
|
||||
service := NewServiceWithProviders(client, NewIPZanProvider(), NewGenericProvider())
|
||||
got, err := service.Extract(
|
||||
context.Background(),
|
||||
"https://service.ipzan.com/core-extract?num=1&protocol=1&secret=x",
|
||||
3,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("len=%d, want 3", len(got))
|
||||
}
|
||||
if got[0].Provider != "ipzan" {
|
||||
t.Fatalf("provider=%q, want ipzan", got[0].Provider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPZanEnsureWhitelistSignsAndAdds(t *testing.T) {
|
||||
const (
|
||||
loginPassword = "login-pass"
|
||||
secret = "extract-secret"
|
||||
signKey = "0123456789abcdef"
|
||||
publicIP = "1.2.3.4"
|
||||
)
|
||||
fixedNow := time.Unix(1700000000, 0)
|
||||
var addCalled bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/whiteList-verify":
|
||||
if r.URL.Query().Get("no") != "plan-no" || r.URL.Query().Get("ip") != publicIP {
|
||||
t.Fatalf("unexpected verify query: %s", r.URL.RawQuery)
|
||||
}
|
||||
io.WriteString(w, `{"code":0,"data":false,"status":200,"message":""}`)
|
||||
case "/whiteList-add":
|
||||
addCalled = true
|
||||
if err := r.ParseForm(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Form.Get("replace") != "1" {
|
||||
t.Fatalf("replace=%q", r.Form.Get("replace"))
|
||||
}
|
||||
encrypted, err := hex.DecodeString(r.Form.Get("sign"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
block, err := aes.NewCipher([]byte(signKey))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decrypted := make([]byte, len(encrypted))
|
||||
for offset := 0; offset < len(encrypted); offset += block.BlockSize() {
|
||||
block.Decrypt(decrypted[offset:offset+block.BlockSize()], encrypted[offset:offset+block.BlockSize()])
|
||||
}
|
||||
padding := int(decrypted[len(decrypted)-1])
|
||||
plain := string(decrypted[:len(decrypted)-padding])
|
||||
if want := loginPassword + ":" + secret + ":1700000000"; plain != want {
|
||||
t.Fatalf("sign plain=%q, want %q", plain, want)
|
||||
}
|
||||
io.WriteString(w, `{"code":0,"data":true,"status":200,"message":""}`)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
provider := &IPZanProvider{
|
||||
baseURL: server.URL,
|
||||
now: func() time.Time { return fixedNow },
|
||||
}
|
||||
err := provider.EnsureWhitelist(
|
||||
context.Background(),
|
||||
server.Client(),
|
||||
WhitelistConfig{
|
||||
APIURL: "https://service.ipzan.com/core-extract?no=plan-no&secret=" + secret,
|
||||
IP: publicIP,
|
||||
LoginPassword: loginPassword,
|
||||
SignKey: signKey,
|
||||
ReplaceOldest: true,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !addCalled {
|
||||
t.Fatal("whiteList-add was not called")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user