125 lines
4.3 KiB
Python
125 lines
4.3 KiB
Python
"""简化版代理获取器。
|
||
|
||
专为短效代理设计：无池、无冷却、无复用。
|
||
每次调用 fetch_new_proxy() 从 API 取 1 个新代理，用完即弃。
|
||
|
||
白名单策略：批次启动前调用 warmup_whitelist() 主动同步一次本地出口 IP；
|
||
之后 fetch_new_proxy 不再主动检测/同步，遇到代理 API 返回"白名单错误"
|
||
时由 ProxyResolver 内部被动同步兜底。
|
||
"""
|
||
|
||
import threading
|
||
import time
|
||
|
||
from loguru import logger
|
||
|
||
from .proxy_resolver import ProxyResolver
|
||
from .proxy_whitelist import DouyuWhitelistSyncer
|
||
|
||
|
||
class ProxyFetcher:
|
||
"""每次从代理 API 取 1 个新代理，无缓存无复用。"""
|
||
|
||
# 代理 API 最小调用间隔（秒）——防止高并发 + 多轮重试把代理 API 打爆
|
||
MIN_FETCH_INTERVAL = 1.5
|
||
|
||
def __init__(
|
||
self,
|
||
api_url: str,
|
||
whitelist_platform: str = "xiequ",
|
||
whitelist_credentials: dict | None = None,
|
||
stop_event: threading.Event | None = None,
|
||
):
|
||
self.api_url = api_url
|
||
self.stop_event = stop_event
|
||
|
||
self._whitelist_syncer = (
|
||
DouyuWhitelistSyncer(
|
||
platform=whitelist_platform or "xiequ",
|
||
credentials=whitelist_credentials,
|
||
)
|
||
if whitelist_credentials
|
||
else None
|
||
)
|
||
|
||
# 节流：跨线程共享的"上次取代理时刻"
|
||
self._last_fetch_at = 0.0
|
||
self._fetch_lock = threading.Lock()
|
||
|
||
def warmup_whitelist(self) -> tuple[bool, str]:
|
||
"""批次启动前调用一次：把当前本地出口 IP 同步到白名单。
|
||
|
||
Returns:
|
||
(是否需要关注的结果, 提示消息)
|
||
- 无白名单凭据时返回 (True, "无白名单凭据，跳过")
|
||
- 出口 IP 获取失败 / 同步失败时返回 (False, 原因)
|
||
- 同步成功时返回 (True, 同步消息)
|
||
"""
|
||
if not self._whitelist_syncer:
|
||
return True, "无白名单凭据，跳过"
|
||
local_ip = self._whitelist_syncer.get_local_exit_ip()
|
||
if not local_ip:
|
||
return False, "获取本地出口 IP 失败"
|
||
ok, msg = self._whitelist_syncer.sync_ip(local_ip)
|
||
if ok:
|
||
return True, f"出口 IP {local_ip} 已同步: {msg}"
|
||
return False, f"出口 IP {local_ip} 同步失败: {msg}"
|
||
|
||
def fetch_new_proxy(self, max_attempts: int = 3) -> str | None:
|
||
"""从代理 API 获取 1 个可用代理，失败返回 None。
|
||
|
||
不主动检测/同步本地出口 IP；如果代理 API 返回"白名单错误"，
|
||
由 ProxyResolver 内部被动同步兜底。
|
||
|
||
进程内节流：高并发场景下任意两次调用至少间隔 MIN_FETCH_INTERVAL 秒，
|
||
避免代理 API 被限频。
|
||
"""
|
||
with self._fetch_lock:
|
||
elapsed = time.monotonic() - self._last_fetch_at
|
||
if elapsed < self.MIN_FETCH_INTERVAL:
|
||
wait = self.MIN_FETCH_INTERVAL - elapsed
|
||
logger.debug(f"代理 API 节流等待 {wait:.2f}s")
|
||
if self.stop_event:
|
||
if self.stop_event.wait(wait):
|
||
return None
|
||
else:
|
||
time.sleep(wait)
|
||
self._last_fetch_at = time.monotonic()
|
||
|
||
if self.stop_event and self.stop_event.is_set():
|
||
return None
|
||
|
||
resolver = ProxyResolver(
|
||
api_url=self.api_url,
|
||
whitelist_syncer=self._whitelist_syncer,
|
||
sync_local_exit_ip=False,
|
||
sync_whitelist_once=False,
|
||
stop_event=self.stop_event,
|
||
)
|
||
result, msg = resolver.fetch_verified(
|
||
max_attempts=max_attempts,
|
||
return_all=False,
|
||
)
|
||
|
||
if isinstance(result, str):
|
||
logger.info(f"获取新代理: {result}")
|
||
return result
|
||
if isinstance(result, list) and result:
|
||
logger.info(f"获取新代理: {result[0]}")
|
||
return result[0]
|
||
logger.warning(f"获取代理失败: {msg}")
|
||
return None
|
||
|
||
|
||
def get_proxy_fetcher(
|
||
api_url: str,
|
||
whitelist_platform: str = "xiequ",
|
||
whitelist_credentials: dict | None = None,
|
||
) -> ProxyFetcher:
|
||
"""获取代理获取器实例。"""
|
||
return ProxyFetcher(
|
||
api_url,
|
||
whitelist_platform=whitelist_platform,
|
||
whitelist_credentials=whitelist_credentials,
|
||
)
|