增加了代理平台和日志
This commit is contained in:
+77
-30
@@ -19,10 +19,19 @@ class ProxyManager:
|
||||
_COOLDOWN_MAX = 120 # 最大冷却秒数
|
||||
_MAX_FAIL_COUNT = 3 # 连续失败此次数后永久移出池
|
||||
|
||||
def __init__(self, api_url: str = "", whitelist_uid: str = "", whitelist_ukey: str = ""):
|
||||
def __init__(
|
||||
self,
|
||||
api_url: str = "",
|
||||
whitelist_uid: str = "",
|
||||
whitelist_ukey: str = "",
|
||||
whitelist_platform: str = "xiequ",
|
||||
whitelist_credentials: dict = None,
|
||||
):
|
||||
self.api_url = api_url
|
||||
self.whitelist_uid = whitelist_uid
|
||||
self.whitelist_ukey = whitelist_ukey
|
||||
self.whitelist_platform = whitelist_platform
|
||||
self.whitelist_credentials = whitelist_credentials
|
||||
self.current_proxy: Optional[str] = None
|
||||
self._cond = threading.Condition()
|
||||
# 已验证可用的代理池: {proxy_url: validated_timestamp}
|
||||
@@ -31,11 +40,19 @@ class ProxyManager:
|
||||
self._in_use: set[str] = set()
|
||||
# 代理失败记录: {proxy_url: {"count": 失败次数, "cooldown_until": 冷却到期时间戳}}
|
||||
self._fail_records: dict[str, dict] = {}
|
||||
self._pool_ttl = 90
|
||||
self._pool_ttl = 180
|
||||
self._fetching = False
|
||||
|
||||
# 构建白名单同步器(优先使用新参数,向后兼容旧参数)
|
||||
_wl_platform = whitelist_platform or "xiequ"
|
||||
_wl_credentials = whitelist_credentials
|
||||
if not _wl_credentials and whitelist_uid and whitelist_ukey:
|
||||
_wl_platform = "xiequ"
|
||||
_wl_credentials = {"uid": whitelist_uid, "ukey": whitelist_ukey}
|
||||
|
||||
self._whitelist_syncer = (
|
||||
DouyuWhitelistSyncer(whitelist_uid, whitelist_ukey)
|
||||
if whitelist_uid and whitelist_ukey
|
||||
DouyuWhitelistSyncer(platform=_wl_platform, credentials=_wl_credentials)
|
||||
if _wl_credentials
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -70,49 +87,71 @@ class ProxyManager:
|
||||
return proxy
|
||||
return None
|
||||
|
||||
def get_proxy(self, max_attempts: int = 5) -> Optional[str]:
|
||||
def get_proxy(self, max_attempts: int = 5, max_retries: int = 3) -> Optional[str]:
|
||||
"""
|
||||
获取可用代理 IP,优先从已验证代理池复用。
|
||||
|
||||
线程安全:池空时只有一个线程调 API 获取并验证所有代理入池,
|
||||
其他线程等待后复用。
|
||||
其他线程等待后复用。等待线程被唤醒后会循环尝试取代理,
|
||||
避免多个线程竞争拿到同一个代理。
|
||||
|
||||
Args:
|
||||
max_attempts: 代理 API 获取尝试次数
|
||||
max_retries: 等待线程获取代理的重试次数(防止唤醒后池仍为空直接放弃)
|
||||
"""
|
||||
# ── 快速路径:池中有可用代理 ──
|
||||
with self._cond:
|
||||
proxy = self._pick_from_pool_locked()
|
||||
if proxy:
|
||||
logger.debug(f"从代理池复用: {proxy} (池剩余 {len(self._verified_pool)})")
|
||||
return proxy
|
||||
|
||||
if self._fetching:
|
||||
logger.debug("代理池空,等待其他线程获取...")
|
||||
self._cond.wait(timeout=60)
|
||||
# ── 池空:需要获取新代理 ──
|
||||
for retry in range(max_retries):
|
||||
with self._cond:
|
||||
# 再次检查(可能其他线程刚充盈了池)
|
||||
proxy = self._pick_from_pool_locked()
|
||||
if proxy:
|
||||
logger.debug(f"等待后从代理池复用: {proxy} (池剩余 {len(self._verified_pool)})")
|
||||
logger.debug(f"从代理池复用: {proxy} (池剩余 {len(self._verified_pool)})")
|
||||
return proxy
|
||||
return None
|
||||
|
||||
self._fetching = True
|
||||
if self._fetching:
|
||||
logger.debug("代理池空,等待其他线程获取...")
|
||||
self._cond.wait(timeout=60)
|
||||
# 等待后循环重试取代理(不再只尝试一次)
|
||||
proxy = self._pick_from_pool_locked()
|
||||
if proxy:
|
||||
logger.debug(f"等待后从代理池复用: {proxy} (池剩余 {len(self._verified_pool)})")
|
||||
return proxy
|
||||
# 池仍为空,但可能有其他等待线程会去获取,继续重试
|
||||
continue
|
||||
|
||||
try:
|
||||
proxies_all, msg = self._fetch_and_verify_all(max_attempts)
|
||||
finally:
|
||||
with self._cond:
|
||||
self._fetching = False
|
||||
self._cond.notify_all()
|
||||
# 当前线程负责获取
|
||||
self._fetching = True
|
||||
|
||||
if proxies_all:
|
||||
with self._cond:
|
||||
now = time.time()
|
||||
for proxy in proxies_all:
|
||||
self._verified_pool[proxy] = now
|
||||
first = proxies_all[0]
|
||||
self._in_use.add(first)
|
||||
self.current_proxy = first
|
||||
logger.info(f"代理池充盈 {len(proxies_all)} 个可用代理: {first} ...")
|
||||
return first
|
||||
try:
|
||||
proxies_all, msg = self._fetch_and_verify_all(max_attempts)
|
||||
finally:
|
||||
with self._cond:
|
||||
self._fetching = False
|
||||
self._cond.notify_all()
|
||||
|
||||
logger.warning(f"获取代理失败: {msg}")
|
||||
if proxies_all:
|
||||
with self._cond:
|
||||
now = time.time()
|
||||
for proxy in proxies_all:
|
||||
self._verified_pool[proxy] = now
|
||||
first = proxies_all[0]
|
||||
self._in_use.add(first)
|
||||
self.current_proxy = first
|
||||
logger.info(f"代理池充盈 {len(proxies_all)} 个可用代理: {first} ...")
|
||||
return first
|
||||
|
||||
logger.warning(f"获取代理失败: {msg}")
|
||||
# 获取失败,等一小段时间后重试
|
||||
time.sleep(2)
|
||||
|
||||
logger.warning(f"获取代理失败(已重试 {max_retries} 次)")
|
||||
return None
|
||||
|
||||
def _fetch_and_verify_all(self, max_attempts: int) -> tuple[Optional[list[str]], str]:
|
||||
@@ -206,6 +245,14 @@ def get_proxy_manager(
|
||||
api_url: str = "",
|
||||
whitelist_uid: str = "",
|
||||
whitelist_ukey: str = "",
|
||||
whitelist_platform: str = "xiequ",
|
||||
whitelist_credentials: dict = None,
|
||||
) -> ProxyManager:
|
||||
"""获取代理管理器实例。"""
|
||||
return ProxyManager(api_url, whitelist_uid=whitelist_uid, whitelist_ukey=whitelist_ukey)
|
||||
return ProxyManager(
|
||||
api_url,
|
||||
whitelist_uid=whitelist_uid,
|
||||
whitelist_ukey=whitelist_ukey,
|
||||
whitelist_platform=whitelist_platform,
|
||||
whitelist_credentials=whitelist_credentials,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user