优化代理管理与增加请求日志系统
代理优化(P0+P1): - 修复极验失败后代理刷新空操作bug(共享ProxyManager+白名单参数) - 极验请求超时从(3.05,12)调大到(10,30) - 白名单sync_ip加全局锁防并发限流,保留多个出口IP应对漂移 - 代理池缓存共享:Condition防并发获取+mark_bad移除坏代理 - 适配代理API的JSON响应格式(code/data/白名单错误) - 简化代理验证只验斗鱼主站,减少日志噪音 - 获取代理前主动同步白名单(解决ow=1模式不报白名单错误的问题) - 每次重试重新检测出口IP并同步白名单 请求日志系统: - 新增HttpLogger记录请求/响应详情到JSONL文件 - login.py的_request和proxy.py的verify_proxy_url接入日志 - 新增/api/logs路由查看和清空HTTP详情日志 - 前端新增请求日志页面(筛选/搜索/分页/自动刷新/详情查看) 其他: - 添加pysocks依赖支持SOCKS5代理 - gitignore添加*.log
This commit is contained in:
+69
-8
@@ -22,6 +22,7 @@ from core.geetest.common.network import (
|
||||
get_c_s,
|
||||
req_fullpage_validate,
|
||||
)
|
||||
from utils.http_logger import log_http
|
||||
|
||||
|
||||
class LoginResult:
|
||||
@@ -62,6 +63,9 @@ class DouyuLogin:
|
||||
timeout: tuple[float, float] = REQUEST_TIMEOUT,
|
||||
max_geetest_retries: int = 5,
|
||||
max_proxy_retries: int = 10,
|
||||
whitelist_uid: str = "",
|
||||
whitelist_ukey: str = "",
|
||||
proxy_manager: Optional[ProxyManager] = None,
|
||||
):
|
||||
self.account = account
|
||||
self.proxy = proxy
|
||||
@@ -70,9 +74,19 @@ class DouyuLogin:
|
||||
self.max_proxy_retries = max_proxy_retries # 0=无限切换直到成功
|
||||
self.session = requests.Session()
|
||||
|
||||
# 初始化代理管理器
|
||||
self.proxy_manager = get_proxy_manager(proxy_api_url) if proxy_api_url else None
|
||||
# 初始化代理管理器(优先使用外部传入的共享实例,避免并发刷新冲突)
|
||||
if proxy_manager:
|
||||
self.proxy_manager = proxy_manager
|
||||
elif proxy_api_url:
|
||||
self.proxy_manager = get_proxy_manager(
|
||||
proxy_api_url,
|
||||
whitelist_uid=whitelist_uid,
|
||||
whitelist_ukey=whitelist_ukey,
|
||||
)
|
||||
else:
|
||||
self.proxy_manager = None
|
||||
|
||||
self._current_proxy_url: Optional[str] = None
|
||||
self._setup_session()
|
||||
|
||||
def _setup_session(self) -> None:
|
||||
@@ -93,13 +107,14 @@ class DouyuLogin:
|
||||
self._apply_proxy()
|
||||
|
||||
def _apply_proxy(self, proxy: str = None) -> None:
|
||||
"""应用代理到Session"""
|
||||
"""应用代理到Session,并记录当前代理URL供刷新时mark_bad"""
|
||||
if proxy:
|
||||
# 使用指定的代理
|
||||
self.session.proxies = {
|
||||
'http': proxy,
|
||||
'https': proxy,
|
||||
}
|
||||
self._current_proxy_url = proxy
|
||||
elif self.proxy:
|
||||
# 使用配置的代理
|
||||
if isinstance(self.proxy, str):
|
||||
@@ -107,12 +122,14 @@ class DouyuLogin:
|
||||
'http': self.proxy,
|
||||
'https': self.proxy,
|
||||
}
|
||||
self._current_proxy_url = self.proxy
|
||||
else:
|
||||
self.session.proxies = {
|
||||
scheme: url
|
||||
for scheme, url in self.proxy.items()
|
||||
if url
|
||||
}
|
||||
self._current_proxy_url = self.proxy.get('http') or self.proxy.get('https')
|
||||
else:
|
||||
# 从代理管理器获取代理
|
||||
if self.proxy_manager:
|
||||
@@ -122,12 +139,17 @@ class DouyuLogin:
|
||||
'http': new_proxy,
|
||||
'https': new_proxy,
|
||||
}
|
||||
self._current_proxy_url = new_proxy
|
||||
|
||||
def _refresh_proxy(self) -> Optional[str]:
|
||||
"""刷新代理IP"""
|
||||
"""刷新代理IP:先标记当前代理为坏(移出代理池),再获取新代理"""
|
||||
if not self.proxy_manager:
|
||||
return None
|
||||
|
||||
# 标记当前代理为不可用,避免再被复用
|
||||
if self._current_proxy_url:
|
||||
self.proxy_manager.mark_bad(self._current_proxy_url)
|
||||
|
||||
new_proxy = self.proxy_manager.get_proxy()
|
||||
if new_proxy:
|
||||
self._apply_proxy(new_proxy)
|
||||
@@ -143,9 +165,14 @@ class DouyuLogin:
|
||||
"""
|
||||
统一发送请求,附带分段超时和更明确的错误信息。
|
||||
代理连接失败时自动重试获取新的代理IP。
|
||||
所有请求详情会记录到 HTTP 详情日志。
|
||||
"""
|
||||
timeout = kwargs.pop('timeout', self.timeout)
|
||||
safe_url = self._safe_url(url)
|
||||
req_data = kwargs.get('data')
|
||||
req_body = req_data if req_data else kwargs.get('json')
|
||||
current_proxy = self._current_proxy_url
|
||||
tag = self.account.username if self.account else ""
|
||||
|
||||
for attempt in range(max_retries):
|
||||
started = time.monotonic()
|
||||
@@ -157,28 +184,56 @@ class DouyuLogin:
|
||||
f"{method.upper()} {safe_url} -> {response.status_code} "
|
||||
f"({elapsed:.2f}s)"
|
||||
)
|
||||
# 记录请求/响应详情
|
||||
resp_body = response.text[:500] if response.text else ""
|
||||
log_http(
|
||||
category="douyu_login",
|
||||
method=method,
|
||||
url=safe_url,
|
||||
request_headers=dict(self.session.headers),
|
||||
request_body=req_body,
|
||||
status_code=response.status_code,
|
||||
response_headers=dict(response.headers),
|
||||
response_body=resp_body,
|
||||
duration=elapsed,
|
||||
proxy=current_proxy,
|
||||
tag=tag,
|
||||
)
|
||||
return response
|
||||
except requests.Timeout as exc:
|
||||
elapsed = time.monotonic() - started
|
||||
log_http(
|
||||
category="douyu_login", method=method, url=safe_url,
|
||||
request_headers=dict(self.session.headers), request_body=req_body,
|
||||
duration=elapsed, error=f"超时 timeout={timeout}: {exc}",
|
||||
proxy=current_proxy, tag=tag,
|
||||
)
|
||||
raise TimeoutError(
|
||||
f"{method.upper()} {safe_url} 超时,耗时 {elapsed:.1f}s,"
|
||||
f"timeout={timeout}"
|
||||
) from exc
|
||||
except requests.ConnectionError as exc:
|
||||
# requests 2.34+ 已移除 ProxyError,代理错误统一为 ConnectionError
|
||||
is_proxy_err = "proxy" in str(exc).lower() or "Proxy" in type(exc).__name__
|
||||
elapsed = time.monotonic() - started
|
||||
err_str = str(exc)
|
||||
is_proxy_err = "proxy" in err_str.lower() or "Proxy" in type(exc).__name__
|
||||
log_http(
|
||||
category="douyu_login", method=method, url=safe_url,
|
||||
request_headers=dict(self.session.headers), request_body=req_body,
|
||||
duration=elapsed,
|
||||
error=f"{'代理' if is_proxy_err else ''}连接失败: {err_str[:300]}",
|
||||
proxy=current_proxy, tag=tag,
|
||||
)
|
||||
if is_proxy_err:
|
||||
elapsed = time.monotonic() - started
|
||||
logger.warning(f"代理连接失败,尝试 {attempt + 1}/{max_retries}: {exc}")
|
||||
if attempt < max_retries - 1:
|
||||
self._refresh_proxy()
|
||||
current_proxy = self._current_proxy_url
|
||||
time.sleep(1)
|
||||
continue
|
||||
raise ConnectionError(
|
||||
f"{method.upper()} {safe_url} 代理连接失败,已重试 {max_retries} 次"
|
||||
) from exc
|
||||
# 非代理的 ConnectionError 也重试
|
||||
elapsed = time.monotonic() - started
|
||||
logger.warning(f"连接失败,尝试 {attempt + 1}/{max_retries}: {exc}")
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(1)
|
||||
@@ -188,6 +243,12 @@ class DouyuLogin:
|
||||
) from exc
|
||||
except requests.RequestException as exc:
|
||||
elapsed = time.monotonic() - started
|
||||
log_http(
|
||||
category="douyu_login", method=method, url=safe_url,
|
||||
request_headers=dict(self.session.headers), request_body=req_body,
|
||||
duration=elapsed, error=f"请求异常: {exc}",
|
||||
proxy=current_proxy, tag=tag,
|
||||
)
|
||||
raise ConnectionError(
|
||||
f"{method.upper()} {safe_url} 请求失败,耗时 {elapsed:.1f}s: {exc}"
|
||||
) from exc
|
||||
|
||||
Reference in New Issue
Block a user