优化代理管理与增加请求日志系统

代理优化(P0+P1):
- 修复极验失败后代理刷新空操作bug(共享ProxyManager+白名单参数)
- 极验请求超时从(3.05,12)调大到(10,30)
- 白名单sync_ip加全局锁防并发限流,保留多个出口IP应对漂移
- 代理池缓存共享:Condition防并发获取+mark_bad移除坏代理
- 适配代理API的JSON响应格式(code/data/白名单错误)
- 简化代理验证只验斗鱼主站,减少日志噪音
- 获取代理前主动同步白名单(解决ow=1模式不报白名单错误的问题)
- 每次重试重新检测出口IP并同步白名单

请求日志系统:
- 新增HttpLogger记录请求/响应详情到JSONL文件
- login.py的_request和proxy.py的verify_proxy_url接入日志
- 新增/api/logs路由查看和清空HTTP详情日志
- 前端新增请求日志页面(筛选/搜索/分页/自动刷新/详情查看)

其他:
- 添加pysocks依赖支持SOCKS5代理
- gitignore添加*.log
This commit is contained in:
yml2213
2026-06-23 00:50:57 +08:00
parent 8fdf8d4b69
commit da2fedd484
14 changed files with 1015 additions and 159 deletions
+69 -8
View File
@@ -22,6 +22,7 @@ from core.geetest.common.network import (
get_c_s,
req_fullpage_validate,
)
from utils.http_logger import log_http
class LoginResult:
@@ -62,6 +63,9 @@ class DouyuLogin:
timeout: tuple[float, float] = REQUEST_TIMEOUT,
max_geetest_retries: int = 5,
max_proxy_retries: int = 10,
whitelist_uid: str = "",
whitelist_ukey: str = "",
proxy_manager: Optional[ProxyManager] = None,
):
self.account = account
self.proxy = proxy
@@ -70,9 +74,19 @@ class DouyuLogin:
self.max_proxy_retries = max_proxy_retries # 0=无限切换直到成功
self.session = requests.Session()
# 初始化代理管理器
self.proxy_manager = get_proxy_manager(proxy_api_url) if proxy_api_url else None
# 初始化代理管理器(优先使用外部传入的共享实例,避免并发刷新冲突)
if proxy_manager:
self.proxy_manager = proxy_manager
elif proxy_api_url:
self.proxy_manager = get_proxy_manager(
proxy_api_url,
whitelist_uid=whitelist_uid,
whitelist_ukey=whitelist_ukey,
)
else:
self.proxy_manager = None
self._current_proxy_url: Optional[str] = None
self._setup_session()
def _setup_session(self) -> None:
@@ -93,13 +107,14 @@ class DouyuLogin:
self._apply_proxy()
def _apply_proxy(self, proxy: str = None) -> None:
"""应用代理到Session"""
"""应用代理到Session,并记录当前代理URL供刷新时mark_bad"""
if proxy:
# 使用指定的代理
self.session.proxies = {
'http': proxy,
'https': proxy,
}
self._current_proxy_url = proxy
elif self.proxy:
# 使用配置的代理
if isinstance(self.proxy, str):
@@ -107,12 +122,14 @@ class DouyuLogin:
'http': self.proxy,
'https': self.proxy,
}
self._current_proxy_url = self.proxy
else:
self.session.proxies = {
scheme: url
for scheme, url in self.proxy.items()
if url
}
self._current_proxy_url = self.proxy.get('http') or self.proxy.get('https')
else:
# 从代理管理器获取代理
if self.proxy_manager:
@@ -122,12 +139,17 @@ class DouyuLogin:
'http': new_proxy,
'https': new_proxy,
}
self._current_proxy_url = new_proxy
def _refresh_proxy(self) -> Optional[str]:
"""刷新代理IP"""
"""刷新代理IP:先标记当前代理为坏(移出代理池),再获取新代理"""
if not self.proxy_manager:
return None
# 标记当前代理为不可用,避免再被复用
if self._current_proxy_url:
self.proxy_manager.mark_bad(self._current_proxy_url)
new_proxy = self.proxy_manager.get_proxy()
if new_proxy:
self._apply_proxy(new_proxy)
@@ -143,9 +165,14 @@ class DouyuLogin:
"""
统一发送请求,附带分段超时和更明确的错误信息。
代理连接失败时自动重试获取新的代理IP。
所有请求详情会记录到 HTTP 详情日志。
"""
timeout = kwargs.pop('timeout', self.timeout)
safe_url = self._safe_url(url)
req_data = kwargs.get('data')
req_body = req_data if req_data else kwargs.get('json')
current_proxy = self._current_proxy_url
tag = self.account.username if self.account else ""
for attempt in range(max_retries):
started = time.monotonic()
@@ -157,28 +184,56 @@ class DouyuLogin:
f"{method.upper()} {safe_url} -> {response.status_code} "
f"({elapsed:.2f}s)"
)
# 记录请求/响应详情
resp_body = response.text[:500] if response.text else ""
log_http(
category="douyu_login",
method=method,
url=safe_url,
request_headers=dict(self.session.headers),
request_body=req_body,
status_code=response.status_code,
response_headers=dict(response.headers),
response_body=resp_body,
duration=elapsed,
proxy=current_proxy,
tag=tag,
)
return response
except requests.Timeout as exc:
elapsed = time.monotonic() - started
log_http(
category="douyu_login", method=method, url=safe_url,
request_headers=dict(self.session.headers), request_body=req_body,
duration=elapsed, error=f"超时 timeout={timeout}: {exc}",
proxy=current_proxy, tag=tag,
)
raise TimeoutError(
f"{method.upper()} {safe_url} 超时,耗时 {elapsed:.1f}s"
f"timeout={timeout}"
) from exc
except requests.ConnectionError as exc:
# requests 2.34+ 已移除 ProxyError,代理错误统一为 ConnectionError
is_proxy_err = "proxy" in str(exc).lower() or "Proxy" in type(exc).__name__
elapsed = time.monotonic() - started
err_str = str(exc)
is_proxy_err = "proxy" in err_str.lower() or "Proxy" in type(exc).__name__
log_http(
category="douyu_login", method=method, url=safe_url,
request_headers=dict(self.session.headers), request_body=req_body,
duration=elapsed,
error=f"{'代理' if is_proxy_err else ''}连接失败: {err_str[:300]}",
proxy=current_proxy, tag=tag,
)
if is_proxy_err:
elapsed = time.monotonic() - started
logger.warning(f"代理连接失败,尝试 {attempt + 1}/{max_retries}: {exc}")
if attempt < max_retries - 1:
self._refresh_proxy()
current_proxy = self._current_proxy_url
time.sleep(1)
continue
raise ConnectionError(
f"{method.upper()} {safe_url} 代理连接失败,已重试 {max_retries}"
) from exc
# 非代理的 ConnectionError 也重试
elapsed = time.monotonic() - started
logger.warning(f"连接失败,尝试 {attempt + 1}/{max_retries}: {exc}")
if attempt < max_retries - 1:
time.sleep(1)
@@ -188,6 +243,12 @@ class DouyuLogin:
) from exc
except requests.RequestException as exc:
elapsed = time.monotonic() - started
log_http(
category="douyu_login", method=method, url=safe_url,
request_headers=dict(self.session.headers), request_body=req_body,
duration=elapsed, error=f"请求异常: {exc}",
proxy=current_proxy, tag=tag,
)
raise ConnectionError(
f"{method.upper()} {safe_url} 请求失败,耗时 {elapsed:.1f}s: {exc}"
) from exc