diff --git a/core/douyu/login.py b/core/douyu/login.py index a1e4805..f688b6b 100644 --- a/core/douyu/login.py +++ b/core/douyu/login.py @@ -22,7 +22,6 @@ from core.geetest.common.network import ( get_c_s, req_fullpage_validate, ) -from utils.http_logger import log_http class AccountLike(Protocol): @@ -246,14 +245,9 @@ class DouyuLogin: """ 统一发送请求，附带分段超时和更明确的错误信息。 代理连接失败时自动重试获取新的代理IP。 - 所有请求详情会记录到 HTTP 详情日志。 """ timeout = kwargs.pop('timeout', self.timeout) safe_url = self._safe_url(url) - req_data = kwargs.get('data') - req_body = req_data if req_data else kwargs.get('json') - current_proxy = self._current_proxy_url - tag = self.account.username if self.account else "" for attempt in range(max_retries): self._ensure_not_stopped() @@ -266,30 +260,9 @@ class DouyuLogin: f"{method.upper()} {safe_url} -> {response.status_code} " f"({elapsed:.2f}s)" ) - # 记录请求/响应详情 - resp_body = response.text[:500] if response.text else "" - log_http( - category="douyu_login", - method=method, - url=safe_url, - request_headers=dict(self.session.headers), - request_body=req_body, - status_code=response.status_code, - response_headers=dict(response.headers), - response_body=resp_body, - duration=elapsed, - proxy=current_proxy, - tag=tag, - ) return response except requests.Timeout as exc: elapsed = time.monotonic() - started - log_http( - category="douyu_login", method=method, url=safe_url, - request_headers=dict(self.session.headers), request_body=req_body, - duration=elapsed, error=f"超时 timeout={timeout}: {exc}", - proxy=current_proxy, tag=tag, - ) raise TimeoutError( f"{method.upper()} {safe_url} 超时，耗时 {elapsed:.1f}s，" f"timeout={timeout}" @@ -298,18 +271,10 @@ class DouyuLogin: elapsed = time.monotonic() - started err_str = str(exc) is_proxy_err = "proxy" in err_str.lower() or "Proxy" in type(exc).__name__ - log_http( - category="douyu_login", method=method, url=safe_url, - request_headers=dict(self.session.headers), request_body=req_body, - duration=elapsed, - error=f"{'代理' if is_proxy_err else ''}连接失败: {err_str[:300]}", - proxy=current_proxy, tag=tag, - ) if is_proxy_err: logger.warning(f"代理连接失败，尝试 {attempt + 1}/{max_retries}: {exc}") if attempt < max_retries - 1: self._refresh_proxy() - current_proxy = self._current_proxy_url self._sleep_interruptible(1) continue raise ConnectionError( @@ -325,12 +290,6 @@ class DouyuLogin: ) from exc except requests.RequestException as exc: elapsed = time.monotonic() - started - log_http( - category="douyu_login", method=method, url=safe_url, - request_headers=dict(self.session.headers), request_body=req_body, - duration=elapsed, error=f"请求异常: {exc}", - proxy=current_proxy, tag=tag, - ) raise ConnectionError( f"{method.upper()} {safe_url} 请求失败，耗时 {elapsed:.1f}s: {exc}" ) from exc diff --git a/core/douyu/proxy_verifier.py b/core/douyu/proxy_verifier.py index 6316566..b486e1f 100644 --- a/core/douyu/proxy_verifier.py +++ b/core/douyu/proxy_verifier.py @@ -2,7 +2,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Optional -import time as _time import requests from loguru import logger @@ -19,12 +18,9 @@ def verify_proxy_url(proxy_url: str, timeout: tuple = (5, 8)) -> tuple[bool, str Returns: (是否可用, 消息) """ - from utils.http_logger import log_http - proxies = {'http': proxy_url, 'https': proxy_url} # ── 第1关：斗鱼主站 ── - started = _time.monotonic() try: response = requests.get( 'https://www.douyu.com', @@ -32,15 +28,8 @@ def verify_proxy_url(proxy_url: str, timeout: tuple = (5, 8)) -> tuple[bool, str timeout=timeout, headers={'User-Agent': 'Mozilla/5.0'}, ) - elapsed = _time.monotonic() - started response.raise_for_status() - log_http( - category="proxy_verify", method="GET", url="https://www.douyu.com", - status_code=response.status_code, response_body=f"斗鱼主站可达: {proxy_url}", - duration=elapsed, proxy=proxy_url, tag="proxy_verify", - ) except Exception as exc: - elapsed = _time.monotonic() - started err_msg = str(exc) if 'Tunnel connection failed' in err_msg or '503' in err_msg: detail = '代理拒绝连接（白名单可能未生效）' @@ -49,15 +38,9 @@ def verify_proxy_url(proxy_url: str, timeout: tuple = (5, 8)) -> tuple[bool, str else: detail = type(exc).__name__ logger.debug(f"代理验证失败 [{proxy_url}]: 斗鱼主站不可达: {detail}") - log_http( - category="proxy_verify", method="GET", url="https://www.douyu.com", - duration=elapsed, error=f"[{proxy_url}] {detail}: {err_msg[:200]}", - proxy=proxy_url, tag="proxy_verify", - ) return False, detail # ── 第2关：极验接口 ── - started = _time.monotonic() try: response = requests.get( 'https://api.geetest.com', @@ -67,26 +50,14 @@ def verify_proxy_url(proxy_url: str, timeout: tuple = (5, 8)) -> tuple[bool, str # geetest 首页可能返回 4xx，只要能连上就算通 allow_redirects=True, ) - elapsed = _time.monotonic() - started - log_http( - category="proxy_verify", method="GET", url="https://api.geetest.com", - status_code=response.status_code, response_body=f"代理可用: {proxy_url}", - duration=elapsed, proxy=proxy_url, tag="proxy_verify", - ) return True, '代理可用 → 斗鱼+极验' except Exception as exc: - elapsed = _time.monotonic() - started err_msg = str(exc) if 'timed out' in err_msg.lower(): detail = '极验接口超时' else: detail = f'极验不可达: {type(exc).__name__}' logger.debug(f"代理验证失败 [{proxy_url}]: 斗鱼可达但{detail}") - log_http( - category="proxy_verify", method="GET", url="https://api.geetest.com", - duration=elapsed, error=f"[{proxy_url}] {detail}: {err_msg[:200]}", - proxy=proxy_url, tag="proxy_verify", - ) return False, detail diff --git a/utils/http_logger.py b/utils/http_logger.py deleted file mode 100644 index f39b57a..0000000 --- a/utils/http_logger.py +++ /dev/null @@ -1,241 +0,0 @@ -"""HTTP 请求/响应详情日志记录器 - -将关键 HTTP 请求的完整详情（method、url、headers、body、status、response、耗时） -以 JSON Lines 格式写入日志文件，便于在 Web 界面查看和排查问题。 -""" - -import json -import threading -import time -from datetime import datetime -from pathlib import Path -from typing import Any, Optional - -from loguru import logger - -# 日志文件路径 -_LOG_DIR = Path("logs") -_LOG_FILE = _LOG_DIR / "http_detail.jsonl" -_MAX_BODY_LEN = 2000 # 单个 body 最大记录长度，避免过大 - -# 线程安全写锁 -_write_lock = threading.Lock() - -# 确保日志目录存在 -_LOG_DIR.mkdir(parents=True, exist_ok=True) - - -def _truncate(text: Any, max_len: int = _MAX_BODY_LEN) -> str: - """截断过长的文本""" - if text is None: - return "" - s = text if isinstance(text, str) else str(text) - if len(s) > max_len: - return s[:max_len] + f"...[truncated {len(s) - max_len} chars]" - return s - - -def _safe_headers(headers: Any) -> dict: - """清理 headers 中的敏感信息""" - if not headers: - return {} - if hasattr(headers, 'items'): - headers = dict(headers) - safe = {} - sensitive = {'authorization', 'cookie', 'set-cookie', 'password'} - for k, v in headers.items(): - if k.lower() in sensitive: - safe[k] = '***' - else: - safe[k] = v - return safe - - -# 请求体中需要脱敏的字段名（小写匹配，包含即遮罩） -_SENSITIVE_BODY_KEYS = { - 'password', 'pwd', 'passwd', 'secret', 'token', 'apikey', 'api_key', - 'email_password', 'mm', 'authorization', 'credential', -} - - -def _safe_body(body: Any) -> str: - """清理请求体中的敏感字段值。 - - 支持 dict、JSON 字符串、其他类型。 - 敏感字段的值会被替换为 ***，其余内容保留（仍受 _truncate 限制）。 - """ - if body is None: - return "" - - # 尝试解析 JSON 字符串 - parsed = body - if isinstance(body, str): - try: - parsed = json.loads(body) - except (json.JSONDecodeError, ValueError): - # 非 JSON 字符串，直接截断 - return _truncate(body) - - # dict 类型：遮罩敏感字段 - if isinstance(parsed, dict): - safe = {} - for k, v in parsed.items(): - if any(s in k.lower() for s in _SENSITIVE_BODY_KEYS): - safe[k] = '***' - elif isinstance(v, (dict, list)): - safe[k] = _safe_body(v) if isinstance(v, dict) else _truncate(str(v)) - else: - safe[k] = v - return _truncate(json.dumps(safe, ensure_ascii=False)) - - # 其他类型：直接截断 - return _truncate(str(body)) - - -def log_http( - category: str, - method: str, - url: str, - *, - request_headers: Any = None, - request_body: Any = None, - status_code: Optional[int] = None, - response_headers: Any = None, - response_body: Any = None, - duration: Optional[float] = None, - error: Optional[str] = None, - proxy: Optional[str] = None, - tag: str = "", -) -> None: - """ - 记录一条 HTTP 请求/响应详情日志。 - - Args: - category: 分类（如 'douyu_login', 'geetest', 'proxy_verify', 'whitelist'） - method: HTTP 方法 - url: 请求 URL（会被脱敏，移除敏感查询参数） - request_headers: 请求头 - request_body: 请求体 - status_code: 响应状态码 - response_headers: 响应头 - response_body: 响应体 - duration: 耗时（秒） - error: 错误信息 - proxy: 使用的代理 - tag: 额外标签（如账号名） - """ - entry = { - "timestamp": datetime.now().isoformat(timespec="milliseconds"), - "ts": time.time(), - "category": category, - "tag": tag, - "method": method.upper(), - "url": _truncate(url, 500), - "proxy": proxy, - "request": { - "headers": _safe_headers(request_headers), - "body": _safe_body(request_body), - }, - "response": { - "status_code": status_code, - "headers": _safe_headers(response_headers), - "body": _truncate(response_body), - }, - "duration_ms": round(duration * 1000, 1) if duration is not None else None, - "error": _truncate(error, 500) if error else None, - } - - # 判断级别 - if error or (status_code and status_code >= 400): - entry["level"] = "error" - elif status_code and status_code >= 300: - entry["level"] = "warning" - else: - entry["level"] = "info" - - try: - line = json.dumps(entry, ensure_ascii=False) - with _write_lock: - with open(_LOG_FILE, "a", encoding="utf-8") as f: - f.write(line + "\n") - except Exception as e: - logger.debug(f"写入HTTP详情日志失败: {e}") - - -def read_http_logs( - *, - limit: int = 100, - offset: int = 0, - category: Optional[str] = None, - level: Optional[str] = None, - keyword: Optional[str] = None, -) -> tuple[list[dict], int]: - """ - 读取 HTTP 详情日志，支持筛选和分页。 - - Args: - limit: 返回条数上限 - offset: 偏移量（从最新往前数） - category: 按分类筛选 - level: 按级别筛选（info/warning/error） - keyword: 关键词搜索（url、tag、error） - - Returns: - (日志条目列表, 总匹配条数)，列表按时间倒序（最新在前） - """ - if not _LOG_FILE.exists(): - return [], 0 - - entries: list[dict] = [] - try: - with open(_LOG_FILE, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - - # 筛选 - if category and entry.get("category") != category: - continue - if level and entry.get("level") != level: - continue - if keyword: - kw = keyword.lower() - searchable = " ".join([ - str(entry.get("url", "")), - str(entry.get("tag", "")), - str(entry.get("error", "")), - str(entry.get("method", "")), - ]).lower() - if kw not in searchable: - continue - - entries.append(entry) - except Exception as e: - logger.error(f"读取HTTP详情日志失败: {e}") - return [], 0 - - # 按时间倒序 - entries.sort(key=lambda x: x.get("ts", 0), reverse=True) - total = len(entries) - # 分页（offset 从最新开始算） - page = entries[offset:offset + limit] - return page, total - - -def clear_http_logs() -> int: - """清空 HTTP 详情日志，返回清空的条数""" - count = 0 - with _write_lock: - if _LOG_FILE.exists(): - try: - with open(_LOG_FILE, "r", encoding="utf-8") as f: - count = sum(1 for line in f if line.strip()) - except Exception: - pass - _LOG_FILE.write_text("", encoding="utf-8") - return count diff --git a/web/backend/main.py b/web/backend/main.py index e7073de..12e94db 100644 --- a/web/backend/main.py +++ b/web/backend/main.py @@ -11,7 +11,7 @@ from fastapi.responses import FileResponse from starlette.middleware.base import BaseHTTPMiddleware from .database import init_db -from .routers import auth, users, accounts, login, proxy, cookies, logs +from .routers import auth, users, accounts, login, proxy, cookies from utils import setup_logger @@ -67,7 +67,6 @@ app.include_router(accounts.router) app.include_router(login.router) app.include_router(proxy.router) app.include_router(cookies.router) -app.include_router(logs.router) @app.get("/api/health") diff --git a/web/backend/routers/logs.py b/web/backend/routers/logs.py deleted file mode 100644 index 39dffaf..0000000 --- a/web/backend/routers/logs.py +++ /dev/null @@ -1,46 +0,0 @@ -"""请求日志路由 - 查看 HTTP 请求/响应详情日志""" - -from fastapi import APIRouter, Depends, HTTPException, Query -from typing import Optional - -from ..deps import get_current_user -from ..permissions import user_has_permission -from utils.http_logger import read_http_logs, clear_http_logs - -router = APIRouter(prefix="/api/logs", tags=["日志"]) - - -@router.get("/http") -def list_http_logs( - limit: int = Query(50, ge=1, le=500), - offset: int = Query(0, ge=0), - category: Optional[str] = None, - level: Optional[str] = None, - keyword: Optional[str] = None, - current=Depends(get_current_user), -): - """查看 HTTP 请求/响应详情日志（需要审计日志查看权限）""" - if not user_has_permission(current, "audit:view"): - # 运营也可以查看请求日志（用于排查登录问题） - if not user_has_permission(current, "login:batch"): - raise HTTPException(status_code=403, detail="无权限") - - items, total = read_http_logs( - limit=limit, - offset=offset, - category=category, - level=level, - keyword=keyword, - ) - return {"items": items, "total": total} - - -@router.delete("/http") -def clear_http_logs_api(current=Depends(get_current_user)): - """清空 HTTP 请求/响应详情日志""" - if not user_has_permission(current, "audit:view"): - if not user_has_permission(current, "login:batch"): - raise HTTPException(status_code=403, detail="无权限") - - count = clear_http_logs() - return {"success": True, "cleared": count} diff --git a/web/frontend/src/App.tsx b/web/frontend/src/App.tsx index 1e15c44..ce9926d 100644 --- a/web/frontend/src/App.tsx +++ b/web/frontend/src/App.tsx @@ -11,7 +11,6 @@ import LoginTasksPage from './pages/LoginTasksPage'; import ProxyPage from './pages/ProxyPage'; import UsersPage from './pages/UsersPage'; import CookiePage from './pages/CookiePage'; -import HttpLogsPage from './pages/HttpLogsPage'; import { getUser } from './store/auth'; import { ThemeProvider, useTheme } from './store/theme'; @@ -45,7 +44,6 @@ function AppContent() { } /> } /> } /> - } /> } /> } /> diff --git a/web/frontend/src/api/logs.ts b/web/frontend/src/api/logs.ts deleted file mode 100644 index 29db8f2..0000000 --- a/web/frontend/src/api/logs.ts +++ /dev/null @@ -1,8 +0,0 @@ -import api from './client'; -import type { HttpLogClearResult, HttpLogListResult } from './types'; - -export const logApi = { - listHttp: (params?: { limit?: number; offset?: number; category?: string; level?: string; keyword?: string }) => - api.get('/logs/http', { params }), - clearHttp: () => api.delete('/logs/http'), -}; diff --git a/web/frontend/src/api/modules.ts b/web/frontend/src/api/modules.ts index 5669239..9a8ac5b 100644 --- a/web/frontend/src/api/modules.ts +++ b/web/frontend/src/api/modules.ts @@ -2,7 +2,6 @@ export * from './types'; export { accountApi } from './accounts'; export { authApi } from './auth'; export { cookieApi } from './cookies'; -export { logApi } from './logs'; export { loginApi } from './login'; export { proxyApi } from './proxy'; export { userApi } from './users'; diff --git a/web/frontend/src/layouts/MainLayout.tsx b/web/frontend/src/layouts/MainLayout.tsx index 1ef01e0..d20a863 100644 --- a/web/frontend/src/layouts/MainLayout.tsx +++ b/web/frontend/src/layouts/MainLayout.tsx @@ -4,7 +4,7 @@ import { DashboardOutlined, UserOutlined, LogoutOutlined, CloudServerOutlined, TeamOutlined, ApiOutlined, KeyOutlined, MenuFoldOutlined, MenuUnfoldOutlined, SwapOutlined, - SunOutlined, MoonOutlined, DesktopOutlined, FileTextOutlined, + SunOutlined, MoonOutlined, DesktopOutlined, } from '@ant-design/icons'; import { useNavigate, useLocation, Outlet } from 'react-router-dom'; import { getUser, clearAuth, type AuthUser } from '../store/auth'; @@ -71,11 +71,6 @@ export default function MainLayout({ onLogout }: { onLogout?: () => void }) { menuItems.push({ key: '/proxy', label: '代理配置', icon: }); } - // 请求日志（运营和管理员可见） - if (canAny(['audit:view', 'login:batch'])) { - menuItems.push({ key: '/http-logs', label: '请求日志', icon: }); - } - // 用户管理 if (can('user:view')) { menuItems.push({ key: '/users', label: '用户管理', icon: }); diff --git a/web/frontend/src/pages/HttpLogsPage.tsx b/web/frontend/src/pages/HttpLogsPage.tsx deleted file mode 100644 index 42ee2ae..0000000 --- a/web/frontend/src/pages/HttpLogsPage.tsx +++ /dev/null @@ -1,316 +0,0 @@ -import { useState, useEffect, useCallback } from 'react'; -import { - Card, Table, Tag, Space, Button, Input, Select, Tooltip, Drawer, - Typography, message, Popconfirm, Empty, Segmented, -} from 'antd'; -import { - ReloadOutlined, DeleteOutlined, SearchOutlined, EyeOutlined, -} from '@ant-design/icons'; -import { logApi, type HttpLogEntry } from '../api/modules'; -import { usePermissions } from '../hooks/usePermissions'; -import { getErrorMessage } from '../utils/error'; - -const { Text, Paragraph } = Typography; - -const CATEGORY_LABELS: Record = { - douyu_login: '斗鱼登录', - geetest: '极验', - proxy_verify: '代理验证', - whitelist: '白名单', -}; - -const LEVEL_COLORS: Record = { - info: 'green', - warning: 'orange', - error: 'red', -}; - -export default function HttpLogsPage() { - const [logs, setLogs] = useState([]); - const [total, setTotal] = useState(0); - const [loading, setLoading] = useState(false); - const [page, setPage] = useState(1); - const [pageSize, setPageSize] = useState(50); - const [category, setCategory] = useState(undefined); - const [level, setLevel] = useState(undefined); - const [keyword, setKeyword] = useState(''); - const [detailEntry, setDetailEntry] = useState(null); - const { canAny } = usePermissions(); - - const fetchLogs = useCallback(async () => { - setLoading(true); - try { - const res = await logApi.listHttp({ - limit: pageSize, - offset: (page - 1) * pageSize, - category, - level, - keyword: keyword || undefined, - }); - setLogs(res.items || []); - setTotal(res.total || 0); - } catch (e: unknown) { - message.error(getErrorMessage(e) || '获取日志失败'); - } finally { - setLoading(false); - } - }, [page, pageSize, category, level, keyword]); - - useEffect(() => { - fetchLogs(); - }, [fetchLogs]); - - // 自动刷新 - useEffect(() => { - const timer = setInterval(() => { - if (!detailEntry) fetchLogs(); - }, 5000); - return () => clearInterval(timer); - }, [fetchLogs, detailEntry]); - - const handleClear = async () => { - try { - const res = await logApi.clearHttp(); - message.success(`已清空 ${res.cleared} 条日志`); - fetchLogs(); - } catch (e: unknown) { - message.error(getErrorMessage(e) || '清空失败'); - } - }; - - const canManage = canAny(['audit:view', 'login:batch']); - - const columns = [ - { - title: '时间', - dataIndex: 'timestamp', - width: 180, - render: (v: string) => {v}, - }, - { - title: '级别', - dataIndex: 'level', - width: 70, - render: (v: string) => {v}, - }, - { - title: '分类', - dataIndex: 'category', - width: 100, - render: (v: string) => CATEGORY_LABELS[v] || v, - }, - { - title: '方法', - dataIndex: 'method', - width: 60, - render: (v: string) => {v}, - }, - { - title: 'URL', - dataIndex: 'url', - ellipsis: true, - render: (v: string) => ( - - {v} - - ), - }, - { - title: '状态', - dataIndex: ['response', 'status_code'], - width: 70, - render: (v: number | null) => v ? ( - {v} - ) : -, - }, - { - title: '耗时', - dataIndex: 'duration_ms', - width: 80, - render: (v: number | null) => v != null ? ( - 5000 ? 'red' : v > 2000 ? 'orange' : undefined }}> - {v > 1000 ? `${(v / 1000).toFixed(1)}s` : `${v}ms`} - - ) : '-', - }, - { - title: '代理', - dataIndex: 'proxy', - width: 140, - ellipsis: true, - render: (v: string | null) => v ? ( - {v} - ) : null, - }, - { - title: '标签', - dataIndex: 'tag', - width: 100, - ellipsis: true, - render: (v: string) => v ? {v} : null, - }, - { - title: '操作', - width: 60, - render: (_: unknown, record: HttpLogEntry) => ( - - - - - - } - > - `${r.ts}-${r.url}`} - size="small" - loading={loading} - pagination={{ - current: page, - pageSize, - total, - showSizeChanger: true, - showTotal: (t) => `共 ${t} 条`, - onChange: (p, ps) => { setPage(p); setPageSize(ps); }, - }} - scroll={{ x: 1000 }} - /> - - - setDetailEntry(null)} - width={700} - > - {detailEntry && ( - -
- 时间: - {detailEntry.timestamp} -
-
- 级别: - {detailEntry.level} - 分类: - {CATEGORY_LABELS[detailEntry.category] || detailEntry.category} -
-
- 请求: - {detailEntry.method} - {detailEntry.url} -
- {detailEntry.proxy && ( -
- 代理: - {detailEntry.proxy} -
- )} - {detailEntry.tag && ( -
- 标签: - {detailEntry.tag} -
- )} -
- 耗时: - {detailEntry.duration_ms != null ? `${detailEntry.duration_ms}ms` : '-'} -
- - {detailEntry.error ? ( - - - {detailEntry.error} - - - ) : ( - -
- 状态码: - {detailEntry.response.status_code ? ( - - {detailEntry.response.status_code} - - ) : -} -
-
- 响应体: - - {detailEntry.response.body || '(空)'} - -
- - )} - - - 
-                {JSON.stringify(detailEntry.request.headers, null, 2)}
-
- - - {detailEntry.request.body && ( - - - {typeof detailEntry.request.body === 'object' - ? JSON.stringify(detailEntry.request.body, null, 2) - : detailEntry.request.body} - - - )} - - {Object.keys(detailEntry.response.headers || {}).length > 0 && ( - - 
-                  {JSON.stringify(detailEntry.response.headers, null, 2)}
-
- - )} - - )} - - - ); -}