yml2213
36d55ed685
- 原来只验证douyu.com，能过但极验不通的代理会入池后立即失败 - 现在两关验证：先过douyu.com，再过api.geetest.com - 日志中09:04:03验证10/10全通过但极验连续失败的问题不会再出现 - 极验超时/不可达的代理在入池前就被过滤掉
153 lines
5.5 KiB
Python
153 lines
5.5 KiB
Python
"""代理可用性验证。"""
|
|
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from typing import Optional
|
|
import time as _time
|
|
|
|
import requests
|
|
from loguru import logger
|
|
|
|
|
|
def verify_proxy_url(proxy_url: str, timeout: tuple = (5, 8)) -> tuple[bool, str]:
|
|
"""
|
|
验证代理是否可用，依次验证：
|
|
1. 斗鱼主站 www.douyu.com（登录目标）
|
|
2. 极验接口 api.geetest.com（极验验证瓶颈）
|
|
|
|
两关都过才算可用。
|
|
|
|
Returns:
|
|
(是否可用, 消息)
|
|
"""
|
|
from utils.http_logger import log_http
|
|
|
|
proxies = {'http': proxy_url, 'https': proxy_url}
|
|
|
|
# ── 第1关：斗鱼主站 ──
|
|
started = _time.monotonic()
|
|
try:
|
|
response = requests.get(
|
|
'https://www.douyu.com',
|
|
proxies=proxies,
|
|
timeout=timeout,
|
|
headers={'User-Agent': 'Mozilla/5.0'},
|
|
)
|
|
elapsed = _time.monotonic() - started
|
|
response.raise_for_status()
|
|
log_http(
|
|
category="proxy_verify", method="GET", url="https://www.douyu.com",
|
|
status_code=response.status_code, response_body=f"斗鱼主站可达: {proxy_url}",
|
|
duration=elapsed, proxy=proxy_url, tag="proxy_verify",
|
|
)
|
|
except Exception as exc:
|
|
elapsed = _time.monotonic() - started
|
|
err_msg = str(exc)
|
|
if 'Tunnel connection failed' in err_msg or '503' in err_msg:
|
|
detail = '代理拒绝连接（白名单可能未生效）'
|
|
elif 'timed out' in err_msg.lower():
|
|
detail = '连接超时'
|
|
else:
|
|
detail = type(exc).__name__
|
|
logger.debug(f"代理验证失败 [{proxy_url}]: 斗鱼主站不可达: {detail}")
|
|
log_http(
|
|
category="proxy_verify", method="GET", url="https://www.douyu.com",
|
|
duration=elapsed, error=f"[{proxy_url}] {detail}: {err_msg[:200]}",
|
|
proxy=proxy_url, tag="proxy_verify",
|
|
)
|
|
return False, detail
|
|
|
|
# ── 第2关：极验接口 ──
|
|
started = _time.monotonic()
|
|
try:
|
|
response = requests.get(
|
|
'https://api.geetest.com',
|
|
proxies=proxies,
|
|
timeout=timeout,
|
|
headers={'User-Agent': 'Mozilla/5.0'},
|
|
# geetest 首页可能返回 4xx，只要能连上就算通
|
|
allow_redirects=True,
|
|
)
|
|
elapsed = _time.monotonic() - started
|
|
log_http(
|
|
category="proxy_verify", method="GET", url="https://api.geetest.com",
|
|
status_code=response.status_code, response_body=f"代理可用: {proxy_url}",
|
|
duration=elapsed, proxy=proxy_url, tag="proxy_verify",
|
|
)
|
|
return True, '代理可用 → 斗鱼+极验'
|
|
except Exception as exc:
|
|
elapsed = _time.monotonic() - started
|
|
err_msg = str(exc)
|
|
if 'timed out' in err_msg.lower():
|
|
detail = '极验接口超时'
|
|
else:
|
|
detail = f'极验不可达: {type(exc).__name__}'
|
|
logger.debug(f"代理验证失败 [{proxy_url}]: 斗鱼可达但{detail}")
|
|
log_http(
|
|
category="proxy_verify", method="GET", url="https://api.geetest.com",
|
|
duration=elapsed, error=f"[{proxy_url}] {detail}: {err_msg[:200]}",
|
|
proxy=proxy_url, tag="proxy_verify",
|
|
)
|
|
return False, detail
|
|
|
|
|
|
def verify_proxies_concurrent(
|
|
proxy_urls: list[str],
|
|
timeout: tuple = (5, 8),
|
|
max_workers: int = 5,
|
|
return_all: bool = False,
|
|
) -> tuple[Optional[str | list[str]], str]:
|
|
"""
|
|
并发验证多个代理 URL。
|
|
|
|
Returns:
|
|
return_all=False: (可用代理 URL 或 None, 消息)
|
|
return_all=True: (可用代理 URL 列表或 None, 消息)
|
|
"""
|
|
if not proxy_urls:
|
|
return None, '无代理可验证'
|
|
|
|
if len(proxy_urls) == 1:
|
|
ok, msg = verify_proxy_url(proxy_urls[0], timeout)
|
|
if ok:
|
|
return (proxy_urls if return_all else proxy_urls[0]), msg
|
|
return None, msg
|
|
|
|
if return_all:
|
|
available: list[str] = []
|
|
with ThreadPoolExecutor(max_workers=min(max_workers, len(proxy_urls))) as executor:
|
|
future_map = {
|
|
executor.submit(verify_proxy_url, proxy, timeout): proxy
|
|
for proxy in proxy_urls
|
|
}
|
|
for future in as_completed(future_map):
|
|
try:
|
|
ok, _ = future.result()
|
|
if ok:
|
|
available.append(future_map[future])
|
|
except Exception:
|
|
continue
|
|
if available:
|
|
logger.success(f"并发验证找到 {len(available)}/{len(proxy_urls)} 个可用代理")
|
|
return available, f'找到 {len(available)} 个可用代理'
|
|
return None, f'共 {len(proxy_urls)} 个代理均不可用'
|
|
|
|
with ThreadPoolExecutor(max_workers=min(max_workers, len(proxy_urls))) as executor:
|
|
future_map = {
|
|
executor.submit(verify_proxy_url, proxy, timeout): proxy
|
|
for proxy in proxy_urls
|
|
}
|
|
for future in as_completed(future_map):
|
|
proxy_url = future_map[future]
|
|
try:
|
|
ok, msg = future.result()
|
|
if ok:
|
|
logger.success(f"并发验证找到可用代理: {proxy_url}")
|
|
for item in future_map:
|
|
if item != future:
|
|
item.cancel()
|
|
return proxy_url, msg
|
|
except Exception:
|
|
continue
|
|
|
|
return None, f'共 {len(proxy_urls)} 个代理均不可用'
|