yml/live-hub-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化代理管理与增加请求日志系统

Browse Source 
代理优化（P0+P1）:
- 修复极验失败后代理刷新空操作bug（共享ProxyManager+白名单参数）
- 极验请求超时从(3.05,12)调大到(10,30)
- 白名单sync_ip加全局锁防并发限流，保留多个出口IP应对漂移
- 代理池缓存共享：Condition防并发获取+mark_bad移除坏代理
- 适配代理API的JSON响应格式（code/data/白名单错误）
- 简化代理验证只验斗鱼主站，减少日志噪音
- 获取代理前主动同步白名单（解决ow=1模式不报白名单错误的问题）
- 每次重试重新检测出口IP并同步白名单

请求日志系统:
- 新增HttpLogger记录请求/响应详情到JSONL文件
- login.py的_request和proxy.py的verify_proxy_url接入日志
- 新增/api/logs路由查看和清空HTTP详情日志
- 前端新增请求日志页面（筛选/搜索/分页/自动刷新/详情查看）

其他:
- 添加pysocks依赖支持SOCKS5代理
- gitignore添加*.log
This commit is contained in:
yml2213
2026-06-23 00:50:57 +08:00
parent 8fdf8d4b69
commit da2fedd484
14 changed files with 1015 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/douyu/proxy.py
+257 -91
View File
@@ -1,6 +1,8 @@
"""代理管理模块"""
import json
import re
import threading
import time
import requests
from typing import Optional
@@ -8,56 +10,152 @@ from loguru import logger
class ProxyManager:
"""代理管理器"""
"""代理管理器（带已验证代理池缓存，批次内共享复用）"""
def __init__(self, api_url: str = ""):
def __init__(self, api_url: str = "", whitelist_uid: str = "", whitelist_ukey: str = ""):
self.api_url = api_url
self.whitelist_uid = whitelist_uid
self.whitelist_ukey = whitelist_ukey
self.current_proxy: Optional[str] = None
self._cond = threading.Condition()
# 已验证可用的代理池: {proxy_url: validated_timestamp}
self._verified_pool: dict[str, float] = {}
self._pool_ttl = 90 # 代理验证后90秒内可复用
self._fetching = False # 是否有线程正在获取代理
def get_proxy(self) -> Optional[str]:
def _pick_from_pool_locked(self) -> Optional[str]:
"""从池中取一个未过期的代理（调用前需持有锁）"""
now = time.time()
# 清理过期代理
expired = [p for p, t in self._verified_pool.items() if now - t > self._pool_ttl]
for p in expired:
del self._verified_pool[p]
# 取第一个可用的
for proxy in self._verified_pool:
self.current_proxy = proxy
return proxy
return None
def get_proxy(self, max_attempts: int = 5) -> Optional[str]:
"""
从代理API获取代理IP，支持API返回多个IP（多行格式），
逐一验证返回第一个可用的。
获取可用代理IP，优先从已验证代理池复用。
线程安全：池空时只有一个线程调API获取并验证所有代理入池，其他线程等待后复用。
Returns:
代理URL，格式: http://ip:port
"""
try:
logger.info("获取代理IP...")
response = requests.get(self.api_url, timeout=10)
response.raise_for_status()
text = response.text.strip()
logger.debug(f"代理API响应: {text}")
proxy_urls, _ = parse_proxy_response(text)
if not proxy_urls:
logger.warning(f"无法解析代理地址: {text}")
return None
if len(proxy_urls) == 1:
proxy = proxy_urls[0]
self.current_proxy = proxy
logger.info(f"获取到代理: {proxy}")
with self._cond:
# 1. 优先从池中取未过期的
proxy = self._pick_from_pool_locked()
if proxy:
logger.debug(f"从代理池复用: {proxy} (池剩余 {len(self._verified_pool)})")
return proxy
# 多个代理逐一验证，返回第一个可用的
logger.info(f"获取到 {len(proxy_urls)} 个代理，逐一验证")
for proxy in proxy_urls:
if self.verify_proxy(proxy):
self.current_proxy = proxy
logger.success(f"可用代理: {proxy}")
# 2. 已有线程在获取，等待结果
if self._fetching:
logger.debug("代理池空，等待其他线程获取...")
self._cond.wait(timeout=60)
proxy = self._pick_from_pool_locked()
if proxy:
logger.debug(f"等待后从代理池复用: {proxy} (池剩余 {len(self._verified_pool)})")
return proxy
logger.warning(f"代理 {proxy} 不可用，尝试下一个")
return None
# 3. 自己去获取
self._fetching = True
logger.warning("所有代理均不可用")
self.current_proxy = None
return None
# 释放锁后执行耗时的API调用+验证
try:
proxies_all, msg = self._fetch_and_verify_all(max_attempts)
finally:
with self._cond:
self._fetching = False
self._cond.notify_all()
except Exception as e:
logger.error(f"获取代理失败: {e}")
return None
if proxies_all:
with self._cond:
now = time.time()
for p in proxies_all:
self._verified_pool[p] = now
first = proxies_all[0]
self.current_proxy = first
logger.info(f"代理池充盈 {len(proxies_all)} 个可用代理: {first} ...")
return first
logger.warning(f"获取代理失败: {msg}")
return None
def _fetch_and_verify_all(self, max_attempts: int) -> tuple[Optional[list[str]], str]:
"""调代理API获取一批代理，并发验证所有可用代理，自动处理白名单同步。"""
last_error = ""
last_synced_ip: Optional[str] = None
for attempt in range(1, max_attempts + 1):
if attempt > 1:
time.sleep(min(attempt - 1, 2))
# 每次尝试前主动同步白名单（出口IP可能漂移，需重新同步）
if self.whitelist_uid and self.whitelist_ukey:
from core.douyu.whitelist import get_local_exit_ip, WhitelistManager
local_ip = get_local_exit_ip()
if local_ip and local_ip != last_synced_ip:
logger.info(f"[尝试 {attempt}] 检测出口IP: {local_ip}，同步白名单...")
manager = WhitelistManager(self.whitelist_uid, self.whitelist_ukey)
ok, sync_msg = manager.sync_ip(local_ip)
if ok:
last_synced_ip = local_ip
logger.info(f"白名单同步成功: {sync_msg}")
# 等待白名单生效
time.sleep(2)
else:
logger.warning(f"白名单同步失败: {sync_msg}")
elif local_ip == last_synced_ip:
logger.debug(f"[尝试 {attempt}] 出口IP未变: {local_ip}")
try:
response = requests.get(self.api_url, timeout=10)
response.raise_for_status()
text = response.text.strip()
proxy_urls, whitelist_ip = parse_proxy_response(text)
if proxy_urls:
logger.info(f"获取到 {len(proxy_urls)} 个代理，并发验证所有")
available, msg = verify_proxies_concurrent(proxy_urls, return_all=True)
if available:
return available, msg
last_error = msg
logger.warning(f"代理预检 {attempt}/{max_attempts}: {last_error}")
continue
# 代理API返回白名单错误
if whitelist_ip and self.whitelist_uid and self.whitelist_ukey:
logger.warning(f"代理需要白名单IP: {whitelist_ip}，自动同步...")
from core.douyu.whitelist import WhitelistManager
manager = WhitelistManager(self.whitelist_uid, self.whitelist_ukey)
ok, sync_msg = manager.sync_ip(whitelist_ip)
if ok:
last_synced_ip = whitelist_ip
logger.info("白名单已更新，等待2秒后重试...")
time.sleep(2)
continue
return None, f'白名单同步失败: {sync_msg}'
last_error = '代理API响应无法解析'
logger.warning(f"代理预检 {attempt}/{max_attempts}: {last_error}: {text[:80]}")
except Exception as exc:
last_error = f'代理API请求失败: {exc}'
logger.warning(f"代理预检 {attempt}/{max_attempts}: {last_error}")
return None, f'代理预检失败（{max_attempts}次尝试均失败）: {last_error}'
def mark_bad(self, proxy_url: str) -> None:
"""标记代理为不可用，从池中移除（极验失败/代理连接失败时调用）"""
with self._cond:
removed = self._verified_pool.pop(proxy_url, None)
if self.current_proxy == proxy_url:
self.current_proxy = None
if removed:
logger.info(f"代理标记为不可用并移出池: {proxy_url} (池剩余 {len(self._verified_pool)})")
self._cond.notify_all()
def get_proxies_dict(self, proxy: str = None) -> dict:
"""获取requests使用的proxies字典"""
@@ -87,94 +185,142 @@ class ProxyManager:
return False
def get_proxy_manager(api_url: str = "") -> ProxyManager:
"""获取代理管理器实例（每次传入 api_url 时创建新实例，避免全局状态污染）"""
return ProxyManager(api_url)
def get_proxy_manager(
api_url: str = "",
whitelist_uid: str = "",
whitelist_ukey: str = "",
) -> ProxyManager:
"""获取代理管理器实例"""
return ProxyManager(api_url, whitelist_uid=whitelist_uid, whitelist_ukey=whitelist_ukey)
def parse_proxy_response(text: str) -> tuple[list[str], Optional[str]]:
"""
解析代理API响应，支持返回多个代理地址。
解析代理API响应，支持 JSON 格式与旧版纯文本格式。
JSON 格式示例：
正常: {"code":0,"success":"true","msg":"","data":[{"IP":"1.2.3.4","Port":5791,...}]}
白名单错误: {"code":-1,"success":"true","msg":"51.请先添加白名单：39.144.109.21","data":""}
旧版纯文本格式（兼容）：
正常: 1.2.3.4:5791\\n5.6.7.8:8080
白名单错误: 请先添加白名单：39.144.109.21
Returns:
(proxy_urls, whitelist_ip)
- proxy_urls: 解析到的所有代理地址列表（http://ip:port）
- whitelist_ip: 需要添加到白名单的IP（当API返回白名单错误时），无错误时为 None
"""
text = text.strip()
text = (text or "").strip()
if not text:
return [], None
# 白名单错误优先检测
# 优先尝试 JSON 解析
try:
data = json.loads(text)
if isinstance(data, dict):
# 白名单错误：code != 0 且 msg 含白名单提示
code = data.get("code")
msg = data.get("msg", "") or ""
if code != 0 and ("白名单" in msg or "添加白名单" in msg):
ip_match = re.search(r'(\d+\.\d+\.\d+\.\d+)', msg)
if ip_match:
return [], ip_match.group(1)
# 正常返回：从 data 数组提取 IP/Port
data_field = data.get("data")
proxies: list[str] = []
if isinstance(data_field, list):
for item in data_field:
if not isinstance(item, dict):
continue
ip = item.get("IP") or item.get("ip")
port = item.get("Port") or item.get("port")
if ip and port:
proxies.append(f"http://{ip}:{port}")
if proxies:
return proxies, None
# data 为空但 code==0，可能代理暂时不可用
if code == 0 and not proxies:
return [], None
except (json.JSONDecodeError, ValueError):
# 不是 JSON，回退到文本解析
pass
# 旧版文本格式：白名单错误优先检测
if '添加白名单' in text or '白名单' in text:
ip_match = re.search(r'(\d+\.\d+\.\d+\.\d+)', text)
if ip_match:
return [], ip_match.group(1)
# 解析所有 ip:port，支持多行格式
# 旧版文本格式：解析所有 ip:port
matches = re.findall(r'(\d+\.\d+\.\d+\.\d+):(\d+)', text)
proxies = []
for ip, port in matches:
# 排除白名单提示中误匹配的
proxy = f"http://{ip}:{port}"
proxies.append(proxy)
proxies = [f"http://{ip}:{port}" for ip, port in matches]
if proxies:
return proxies, None
return [], None
def verify_proxy_url(proxy_url: str, timeout: tuple = (4, 6)) -> tuple[bool, str]:
def verify_proxy_url(proxy_url: str, timeout: tuple = (5, 8)) -> tuple[bool, str]:
"""
验证代理是否可用。
验证优先级：
1. 斗鱼主站（最相关，能访问斗鱼才是最终目的）
2. myip（快速 IP 验证）
3. 百度 IP 查询（备用）
验证代理是否可用，只验证斗鱼主站（登录的最终目标）。
Returns:
(是否可用, 消息)
"""
from utils.http_logger import log_http
import time as _time
proxies = {'http': proxy_url, 'https': proxy_url}
targets = [
('https://www.douyu.com', '斗鱼主站'),
('https://myip.ipip.net', 'IP验证'),
('https://qifu-api.baidubce.com/ip/local/geo/v1/district', '百度IP查询'),
]
for url, label in targets:
try:
response = requests.get(
url, proxies=proxies, timeout=timeout,
headers={'User-Agent': 'Mozilla/5.0'},
)
response.raise_for_status()
return True, f'代理可用 → {label}'
except Exception as e:
err_msg = str(e)
if 'Tunnel connection failed' in err_msg or '503' in err_msg:
detail = '代理拒绝连接（白名单可能未生效）'
elif 'timed out' in err_msg.lower():
detail = '连接超时'
else:
detail = type(e).__name__
logger.debug(f"代理验证 {label} 失败: {detail}")
continue
return False, '代理验证失败（所有目标不可达）'
started = _time.monotonic()
try:
response = requests.get(
'https://www.douyu.com',
proxies=proxies,
timeout=timeout,
headers={'User-Agent': 'Mozilla/5.0'},
)
elapsed = _time.monotonic() - started
response.raise_for_status()
log_http(
category="proxy_verify", method="GET", url="https://www.douyu.com",
status_code=response.status_code, response_body=f"代理可用: {proxy_url}",
duration=elapsed, proxy=proxy_url, tag="proxy_verify",
)
return True, '代理可用 → 斗鱼主站'
except Exception as e:
elapsed = _time.monotonic() - started
err_msg = str(e)
if 'Tunnel connection failed' in err_msg or '503' in err_msg:
detail = '代理拒绝连接（白名单可能未生效）'
elif 'timed out' in err_msg.lower():
detail = '连接超时'
else:
detail = type(e).__name__
logger.debug(f"代理验证失败 [{proxy_url}]: {detail}")
log_http(
category="proxy_verify", method="GET", url="https://www.douyu.com",
duration=elapsed, error=f"[{proxy_url}] {detail}: {err_msg[:200]}",
proxy=proxy_url, tag="proxy_verify",
)
return False, detail
def verify_proxies_concurrent(proxy_urls: list[str], timeout: tuple = (4, 6), max_workers: int = 5) -> tuple[Optional[str], str]:
def verify_proxies_concurrent(proxy_urls: list[str], timeout: tuple = (5, 8), max_workers: int = 5, return_all: bool = False) -> tuple[Optional[str | list[str]], str]:
"""
并发验证多个代理URL，返回第一个可用的。
并发验证多个代理URL。
Args:
proxy_urls: 代理URL列表
timeout: 验证超时
max_workers: 最大并发数
return_all: True 时返回所有可用代理列表；False（默认）返回第一个可用的
Returns:
(可用的代理URL, 消息)
return_all=False: (可用代理URL或None, 消息)
return_all=True: (可用代理URL列表或None, 消息)
"""
if not proxy_urls:
return None, '无代理可验证'
@@ -182,11 +328,32 @@ def verify_proxies_concurrent(proxy_urls: list[str], timeout: tuple = (4, 6), ma
if len(proxy_urls) == 1:
ok, msg = verify_proxy_url(proxy_urls[0], timeout)
if ok:
return proxy_urls[0], msg
return (proxy_urls if return_all else proxy_urls[0]), msg
return None, msg
from concurrent.futures import ThreadPoolExecutor, as_completed
if return_all:
# 收集所有可用代理（不取消任何任务）
available: list[str] = []
with ThreadPoolExecutor(max_workers=min(max_workers, len(proxy_urls))) as executor:
future_map = {
executor.submit(verify_proxy_url, p, timeout): p
for p in proxy_urls
}
for future in as_completed(future_map):
try:
ok, _ = future.result()
if ok:
available.append(future_map[future])
except Exception:
continue
if available:
logger.success(f"并发验证找到 {len(available)}/{len(proxy_urls)} 个可用代理")
return available, f'找到 {len(available)} 个可用代理'
return None, f'共 {len(proxy_urls)} 个代理均不可用'
# 默认：返回第一个可用的，取消其余
with ThreadPoolExecutor(max_workers=min(max_workers, len(proxy_urls))) as executor:
future_map = {
executor.submit(verify_proxy_url, p, timeout): p
@@ -198,7 +365,6 @@ def verify_proxies_concurrent(proxy_urls: list[str], timeout: tuple = (4, 6), ma
ok, msg = future.result()
if ok:
logger.success(f"并发验证找到可用代理: {proxy_url}")
# 取消剩余任务
for f in future_map:
if f != future:
f.cancel()
@@ -213,7 +379,7 @@ def resolve_working_proxy(
api_url: str,
whitelist_uid: str = "",
whitelist_ukey: str = "",
max_attempts: int = 3,
max_attempts: int = 4,
log_func=None,
) -> tuple[Optional[str], str]:
"""
@@ -224,7 +390,7 @@ def resolve_working_proxy(
api_url: 代理API地址
whitelist_uid: 白名单UID（启用白名单时传入）
whitelist_ukey: 白名单UKEY
max_attempts: 最大获取尝试次数（默认3次）
max_attempts: 最大获取尝试次数（默认4次）
log_func: 日志回调函数 (level, message)
Returns:
@@ -242,7 +408,7 @@ def resolve_working_proxy(
for attempt in range(1, max_attempts + 1):
# 重试之间增加退避延迟，避免代理API返回同一个不可用IP
if attempt > 1:
delay = min(attempt, 3)
delay = min(attempt - 1, 2)
log('info', f'等待 {delay}s 后重试...')
time.sleep(delay)