优化代理管理与增加请求日志系统
代理优化（P0+P1）: - 修复极验失败后代理刷新空操作bug（共享ProxyManager+白名单参数） - 极验请求超时从(3.05,12)调大到(10,30) - 白名单sync_ip加全局锁防并发限流，保留多个出口IP应对漂移 - 代理池缓存共享：Condition防并发获取+mark_bad移除坏代理 - 适配代理API的JSON响应格式（code/data/白名单错误） - 简化代理验证只验斗鱼主站，减少日志噪音 - 获取代理前主动同步白名单（解决ow=1模式不报白名单错误的问题） - 每次重试重新检测出口IP并同步白名单 请求日志系统: - 新增HttpLogger记录请求/响应详情到JSONL文件 - login.py的_request和proxy.py的verify_proxy_url接入日志 - 新增/api/logs路由查看和清空HTTP详情日志 - 前端新增请求日志页面（筛选/搜索/分页/自动刷新/详情查看） 其他: - 添加pysocks依赖支持SOCKS5代理 - gitignore添加*.log
This commit is contained in:
yml2213
+2
-1
@@ -6,7 +6,7 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from .database import init_db
|
||||
from .routers import auth, users, accounts, login, proxy, cookies
|
||||
from .routers import auth, users, accounts, login, proxy, cookies, logs
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -37,6 +37,7 @@ app.include_router(accounts.router)
|
||||
app.include_router(login.router)
|
||||
app.include_router(proxy.router)
|
||||
app.include_router(cookies.router)
|
||||
app.include_router(logs.router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""请求日志路由 - 查看 HTTP 请求/响应详情日志"""
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from typing import Optional
|
||||
|
||||
from ..deps import get_current_user
|
||||
from ..permissions import has_permission
|
||||
from utils.http_logger import read_http_logs, clear_http_logs
|
||||
|
||||
router = APIRouter(prefix="/api/logs", tags=["日志"])
|
||||
|
||||
|
||||
@router.get("/http")
|
||||
def list_http_logs(
|
||||
limit: int = Query(50, ge=1, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
category: Optional[str] = None,
|
||||
level: Optional[str] = None,
|
||||
keyword: Optional[str] = None,
|
||||
current=Depends(get_current_user),
|
||||
):
|
||||
"""查看 HTTP 请求/响应详情日志（需要审计日志查看权限）"""
|
||||
if not has_permission(current.role, "audit:view"):
|
||||
# 运营也可以查看请求日志（用于排查登录问题）
|
||||
if not has_permission(current.role, "login:batch"):
|
||||
return {"items": [], "total": 0, "message": "无权限"}
|
||||
|
||||
items, total = read_http_logs(
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
category=category,
|
||||
level=level,
|
||||
keyword=keyword,
|
||||
)
|
||||
return {"items": items, "total": total}
|
||||
|
||||
|
||||
@router.delete("/http")
|
||||
def clear_http_logs_api(current=Depends(get_current_user)):
|
||||
"""清空 HTTP 请求/响应详情日志"""
|
||||
if not has_permission(current.role, "audit:view"):
|
||||
if not has_permission(current.role, "login:batch"):
|
||||
return {"success": False, "message": "无权限"}
|
||||
|
||||
count = clear_http_logs()
|
||||
return {"success": True, "cleared": count}
|
||||
@@ -11,7 +11,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from core.douyu import DouyuLogin
|
||||
from core.models import Account, ProxyConfig as DouyuProxyConfig
|
||||
from core.douyu.proxy import resolve_working_proxy
|
||||
from core.douyu.proxy import resolve_working_proxy, get_proxy_manager
|
||||
from ..models import Account as AccountModel, LoginTask, ProxyConfig as ProxyConfigModel
|
||||
from ..permissions import has_permission
|
||||
|
||||
@@ -47,6 +47,17 @@ class LoginBatchRunner:
|
||||
self._counter_lock = threading.Lock()
|
||||
self._completed = 0
|
||||
|
||||
# 共享代理管理器（带锁，避免并发白名单限流；极验失败时可刷新代理）
|
||||
self._shared_proxy_manager = None
|
||||
if proxy_config and proxy_config.enabled and proxy_config.api_url:
|
||||
wl_uid = proxy_config.whitelist_uid or "" if proxy_config.whitelist_enabled else ""
|
||||
wl_ukey = proxy_config.whitelist_ukey or "" if proxy_config.whitelist_enabled else ""
|
||||
self._shared_proxy_manager = get_proxy_manager(
|
||||
proxy_config.api_url,
|
||||
whitelist_uid=wl_uid,
|
||||
whitelist_ukey=wl_ukey,
|
||||
)
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
|
||||
@@ -62,7 +73,7 @@ class LoginBatchRunner:
|
||||
解析代理配置，返回 (proxy_dict, message)。
|
||||
|
||||
- 静态代理：直接返回 dict
|
||||
- API代理：调用 resolve_working_proxy 预检，自动同步白名单
|
||||
- API代理：通过共享 ProxyManager（带已验证代理池缓存）获取，自动同步白名单
|
||||
- 无代理：返回 (None, '')
|
||||
"""
|
||||
if not self.proxy_config or not self.proxy_config.enabled:
|
||||
@@ -73,23 +84,12 @@ class LoginBatchRunner:
|
||||
proxy_url = self.proxy_config.http or self.proxy_config.https
|
||||
return {'http': proxy_url, 'https': proxy_url}, f'使用静态代理: {proxy_url}'
|
||||
|
||||
# API代理：预检获取可用代理
|
||||
if self.proxy_config.api_url:
|
||||
whitelist_uid = ''
|
||||
whitelist_ukey = ''
|
||||
if self.proxy_config.whitelist_enabled:
|
||||
whitelist_uid = self.proxy_config.whitelist_uid or ''
|
||||
whitelist_ukey = self.proxy_config.whitelist_ukey or ''
|
||||
|
||||
proxy_url, msg = resolve_working_proxy(
|
||||
api_url=self.proxy_config.api_url,
|
||||
whitelist_uid=whitelist_uid,
|
||||
whitelist_ukey=whitelist_ukey,
|
||||
log_func=self._push_log,
|
||||
)
|
||||
# API代理：通过共享代理管理器获取（优先从已验证代理池复用）
|
||||
if self._shared_proxy_manager:
|
||||
proxy_url = self._shared_proxy_manager.get_proxy()
|
||||
if proxy_url:
|
||||
return {'http': proxy_url, 'https': proxy_url}, msg
|
||||
return None, msg
|
||||
return {'http': proxy_url, 'https': proxy_url}, f'使用API代理: {proxy_url}'
|
||||
return None, '代理不可用'
|
||||
|
||||
return None, ''
|
||||
|
||||
@@ -144,6 +144,7 @@ class LoginBatchRunner:
|
||||
proxy=proxy_dict,
|
||||
max_geetest_retries=self.max_geetest_retries,
|
||||
max_proxy_retries=self.max_proxy_retries,
|
||||
proxy_manager=self._shared_proxy_manager,
|
||||
)
|
||||
result = loginer.login()
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import LoginTasksPage from './pages/LoginTasksPage';
|
||||
import ProxyPage from './pages/ProxyPage';
|
||||
import UsersPage from './pages/UsersPage';
|
||||
import CookiePage from './pages/CookiePage';
|
||||
import HttpLogsPage from './pages/HttpLogsPage';
|
||||
import { getToken } from './store/auth';
|
||||
import { ThemeProvider, useTheme } from './store/theme';
|
||||
|
||||
@@ -44,6 +45,7 @@ function AppContent() {
|
||||
<Route path="login-tasks" element={<LoginTasksPage />} />
|
||||
<Route path="cookies" element={<CookiePage />} />
|
||||
<Route path="proxy" element={<ProxyPage />} />
|
||||
<Route path="http-logs" element={<HttpLogsPage />} />
|
||||
<Route path="users" element={<UsersPage />} />
|
||||
</Route>
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
|
||||
@@ -77,3 +77,9 @@ export const proxyApi = {
|
||||
test: () => api.post<any, any>('/proxy/test'),
|
||||
testWhitelist: () => api.post<any, any>('/proxy/whitelist/test'),
|
||||
};
|
||||
|
||||
export const logApi = {
|
||||
listHttp: (params?: { limit?: number; offset?: number; category?: string; level?: string; keyword?: string }) =>
|
||||
api.get<any, { items: any[]; total: number }>('/logs/http', { params }),
|
||||
clearHttp: () => api.delete<any, { success: boolean; cleared: number }>('/logs/http'),
|
||||
};
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
DashboardOutlined, UserOutlined, LogoutOutlined,
|
||||
CloudServerOutlined, TeamOutlined, ApiOutlined, KeyOutlined,
|
||||
MenuFoldOutlined, MenuUnfoldOutlined, SwapOutlined,
|
||||
SunOutlined, MoonOutlined, DesktopOutlined,
|
||||
SunOutlined, MoonOutlined, DesktopOutlined, FileTextOutlined,
|
||||
} from '@ant-design/icons';
|
||||
import { useNavigate, useLocation, Outlet } from 'react-router-dom';
|
||||
import { getUser, clearAuth, hasPerm, type AuthUser } from '../store/auth';
|
||||
@@ -69,6 +69,11 @@ export default function MainLayout({ onLogout }: { onLogout?: () => void }) {
|
||||
menuItems.push({ key: '/proxy', label: '代理配置', icon: <CloudServerOutlined /> });
|
||||
}
|
||||
|
||||
// 请求日志（运营和管理员可见）
|
||||
if (hasPerm(user, 'audit:view') || hasPerm(user, 'login:batch')) {
|
||||
menuItems.push({ key: '/http-logs', label: '请求日志', icon: <FileTextOutlined /> });
|
||||
}
|
||||
|
||||
// 用户管理
|
||||
if (hasPerm(user, 'user:view')) {
|
||||
menuItems.push({ key: '/users', label: '用户管理', icon: <TeamOutlined /> });
|
||||
|
||||
@@ -0,0 +1,330 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import {
|
||||
Card, Table, Tag, Space, Button, Input, Select, Tooltip, Drawer,
|
||||
Typography, message, Popconfirm, Empty, Segmented,
|
||||
} from 'antd';
|
||||
import {
|
||||
ReloadOutlined, DeleteOutlined, SearchOutlined, EyeOutlined,
|
||||
} from '@ant-design/icons';
|
||||
import { logApi } from '../api/modules';
|
||||
import { hasPerm, getUser } from '../store/auth';
|
||||
|
||||
const { Text, Paragraph } = Typography;
|
||||
|
||||
interface HttpLogEntry {
|
||||
timestamp: string;
|
||||
ts: number;
|
||||
category: string;
|
||||
tag: string;
|
||||
method: string;
|
||||
url: string;
|
||||
proxy: string | null;
|
||||
request: { headers: Record<string, string>; body: string };
|
||||
response: { status_code: number | null; headers: Record<string, string>; body: string };
|
||||
duration_ms: number | null;
|
||||
error: string | null;
|
||||
level: string;
|
||||
}
|
||||
|
||||
const CATEGORY_LABELS: Record<string, string> = {
|
||||
douyu_login: '斗鱼登录',
|
||||
geetest: '极验',
|
||||
proxy_verify: '代理验证',
|
||||
whitelist: '白名单',
|
||||
};
|
||||
|
||||
const LEVEL_COLORS: Record<string, string> = {
|
||||
info: 'green',
|
||||
warning: 'orange',
|
||||
error: 'red',
|
||||
};
|
||||
|
||||
export default function HttpLogsPage() {
|
||||
const [logs, setLogs] = useState<HttpLogEntry[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [page, setPage] = useState(1);
|
||||
const [pageSize, setPageSize] = useState(50);
|
||||
const [category, setCategory] = useState<string | undefined>(undefined);
|
||||
const [level, setLevel] = useState<string | undefined>(undefined);
|
||||
const [keyword, setKeyword] = useState('');
|
||||
const [detailEntry, setDetailEntry] = useState<HttpLogEntry | null>(null);
|
||||
const user = getUser();
|
||||
|
||||
const fetchLogs = useCallback(async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const res = await logApi.listHttp({
|
||||
limit: pageSize,
|
||||
offset: (page - 1) * pageSize,
|
||||
category,
|
||||
level,
|
||||
keyword: keyword || undefined,
|
||||
});
|
||||
setLogs(res.items || []);
|
||||
setTotal(res.total || 0);
|
||||
} catch (e: any) {
|
||||
message.error(e.message || '获取日志失败');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [page, pageSize, category, level, keyword]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchLogs();
|
||||
}, [fetchLogs]);
|
||||
|
||||
// 自动刷新
|
||||
useEffect(() => {
|
||||
const timer = setInterval(() => {
|
||||
if (!detailEntry) fetchLogs();
|
||||
}, 5000);
|
||||
return () => clearInterval(timer);
|
||||
}, [fetchLogs, detailEntry]);
|
||||
|
||||
const handleClear = async () => {
|
||||
try {
|
||||
const res = await logApi.clearHttp();
|
||||
message.success(`已清空 ${res.cleared} 条日志`);
|
||||
fetchLogs();
|
||||
} catch (e: any) {
|
||||
message.error(e.message || '清空失败');
|
||||
}
|
||||
};
|
||||
|
||||
const canManage = user && (hasPerm(user, 'audit:view') || hasPerm(user, 'login:batch'));
|
||||
|
||||
const columns = [
|
||||
{
|
||||
title: '时间',
|
||||
dataIndex: 'timestamp',
|
||||
width: 180,
|
||||
render: (v: string) => <Text style={{ fontSize: 12 }}>{v}</Text>,
|
||||
},
|
||||
{
|
||||
title: '级别',
|
||||
dataIndex: 'level',
|
||||
width: 70,
|
||||
render: (v: string) => <Tag color={LEVEL_COLORS[v] || 'default'}>{v}</Tag>,
|
||||
},
|
||||
{
|
||||
title: '分类',
|
||||
dataIndex: 'category',
|
||||
width: 100,
|
||||
render: (v: string) => CATEGORY_LABELS[v] || v,
|
||||
},
|
||||
{
|
||||
title: '方法',
|
||||
dataIndex: 'method',
|
||||
width: 60,
|
||||
render: (v: string) => <Tag>{v}</Tag>,
|
||||
},
|
||||
{
|
||||
title: 'URL',
|
||||
dataIndex: 'url',
|
||||
ellipsis: true,
|
||||
render: (v: string) => (
|
||||
<Tooltip title={v}>
|
||||
<Text style={{ fontSize: 12 }} ellipsis>{v}</Text>
|
||||
</Tooltip>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: '状态',
|
||||
dataIndex: ['response', 'status_code'],
|
||||
width: 70,
|
||||
render: (v: number | null) => v ? (
|
||||
<Tag color={v < 300 ? 'green' : v < 400 ? 'blue' : 'red'}>{v}</Tag>
|
||||
) : <Tag>-</Tag>,
|
||||
},
|
||||
{
|
||||
title: '耗时',
|
||||
dataIndex: 'duration_ms',
|
||||
width: 80,
|
||||
render: (v: number | null) => v != null ? (
|
||||
<Text style={{ fontSize: 12, color: v > 5000 ? 'red' : v > 2000 ? 'orange' : undefined }}>
|
||||
{v > 1000 ? `${(v / 1000).toFixed(1)}s` : `${v}ms`}
|
||||
</Text>
|
||||
) : '-',
|
||||
},
|
||||
{
|
||||
title: '代理',
|
||||
dataIndex: 'proxy',
|
||||
width: 140,
|
||||
ellipsis: true,
|
||||
render: (v: string | null) => v ? (
|
||||
<Text style={{ fontSize: 12 }} type="secondary">{v}</Text>
|
||||
) : null,
|
||||
},
|
||||
{
|
||||
title: '标签',
|
||||
dataIndex: 'tag',
|
||||
width: 100,
|
||||
ellipsis: true,
|
||||
render: (v: string) => v ? <Text style={{ fontSize: 12 }}>{v}</Text> : null,
|
||||
},
|
||||
{
|
||||
title: '操作',
|
||||
width: 60,
|
||||
render: (_: any, record: HttpLogEntry) => (
|
||||
<Button type="link" size="small" icon={<EyeOutlined />} onClick={() => setDetailEntry(record)} />
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
if (!canManage) {
|
||||
return <Empty description="无权限查看" />;
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Card
|
||||
title="请求日志"
|
||||
extra={
|
||||
<Space>
|
||||
<Segmented
|
||||
options={[
|
||||
{ label: '全部', value: '' },
|
||||
{ label: '信息', value: 'info' },
|
||||
{ label: '警告', value: 'warning' },
|
||||
{ label: '错误', value: 'error' },
|
||||
]}
|
||||
value={level || ''}
|
||||
onChange={(v) => { setLevel(v as string || undefined); setPage(1); }}
|
||||
/>
|
||||
<Select
|
||||
placeholder="分类"
|
||||
allowClear
|
||||
style={{ width: 130 }}
|
||||
value={category}
|
||||
onChange={(v) => { setCategory(v); setPage(1); }}
|
||||
options={Object.entries(CATEGORY_LABELS).map(([k, v]) => ({ value: k, label: v }))}
|
||||
/>
|
||||
<Input
|
||||
placeholder="搜索关键词"
|
||||
allowClear
|
||||
style={{ width: 180 }}
|
||||
prefix={<SearchOutlined />}
|
||||
value={keyword}
|
||||
onChange={(e) => setKeyword(e.target.value)}
|
||||
onPressEnter={() => { setPage(1); fetchLogs(); }}
|
||||
/>
|
||||
<Button icon={<ReloadOutlined />} onClick={fetchLogs} loading={loading}>刷新</Button>
|
||||
<Popconfirm title="确定清空所有日志？" onConfirm={handleClear}>
|
||||
<Button danger icon={<DeleteOutlined />}>清空</Button>
|
||||
</Popconfirm>
|
||||
</Space>
|
||||
}
|
||||
>
|
||||
<Table
|
||||
dataSource={logs}
|
||||
columns={columns}
|
||||
rowKey={(r) => `${r.ts}-${r.url}`}
|
||||
size="small"
|
||||
loading={loading}
|
||||
pagination={{
|
||||
current: page,
|
||||
pageSize,
|
||||
total,
|
||||
showSizeChanger: true,
|
||||
showTotal: (t) => `共 ${t} 条`,
|
||||
onChange: (p, ps) => { setPage(p); setPageSize(ps); },
|
||||
}}
|
||||
scroll={{ x: 1000 }}
|
||||
/>
|
||||
</Card>
|
||||
|
||||
<Drawer
|
||||
title="请求详情"
|
||||
open={!!detailEntry}
|
||||
onClose={() => setDetailEntry(null)}
|
||||
width={700}
|
||||
>
|
||||
{detailEntry && (
|
||||
<Space direction="vertical" style={{ width: '100%' }} size="middle">
|
||||
<div>
|
||||
<Text strong>时间: </Text>
|
||||
<Text>{detailEntry.timestamp}</Text>
|
||||
</div>
|
||||
<div>
|
||||
<Text strong>级别: </Text>
|
||||
<Tag color={LEVEL_COLORS[detailEntry.level]}>{detailEntry.level}</Tag>
|
||||
<Text strong style={{ marginLeft: 16 }}>分类: </Text>
|
||||
<Tag>{CATEGORY_LABELS[detailEntry.category] || detailEntry.category}</Tag>
|
||||
</div>
|
||||
<div>
|
||||
<Text strong>请求: </Text>
|
||||
<Tag color="blue">{detailEntry.method}</Tag>
|
||||
<Text copyable style={{ fontSize: 13 }}>{detailEntry.url}</Text>
|
||||
</div>
|
||||
{detailEntry.proxy && (
|
||||
<div>
|
||||
<Text strong>代理: </Text>
|
||||
<Text code>{detailEntry.proxy}</Text>
|
||||
</div>
|
||||
)}
|
||||
{detailEntry.tag && (
|
||||
<div>
|
||||
<Text strong>标签: </Text>
|
||||
<Text>{detailEntry.tag}</Text>
|
||||
</div>
|
||||
)}
|
||||
<div>
|
||||
<Text strong>耗时: </Text>
|
||||
<Text>{detailEntry.duration_ms != null ? `${detailEntry.duration_ms}ms` : '-'}</Text>
|
||||
</div>
|
||||
|
||||
{detailEntry.error ? (
|
||||
<Card title="错误" size="small" style={{ borderColor: '#ff4d4f' }}>
|
||||
<Paragraph type="danger" style={{ margin: 0, whiteSpace: 'pre-wrap', fontSize: 13 }}>
|
||||
{detailEntry.error}
|
||||
</Paragraph>
|
||||
</Card>
|
||||
) : (
|
||||
<Card title="响应" size="small">
|
||||
<div style={{ marginBottom: 8 }}>
|
||||
<Text strong>状态码: </Text>
|
||||
{detailEntry.response.status_code ? (
|
||||
<Tag color={detailEntry.response.status_code < 300 ? 'green' : 'red'}>
|
||||
{detailEntry.response.status_code}
|
||||
</Tag>
|
||||
) : <Text type="secondary">-</Text>}
|
||||
</div>
|
||||
<div>
|
||||
<Text strong>响应体:</Text>
|
||||
<Paragraph style={{ background: 'rgba(0,0,0,0.04)', padding: 8, borderRadius: 4, whiteSpace: 'pre-wrap', fontSize: 12, margin: '4px 0 0' }}>
|
||||
{detailEntry.response.body || '(空)'}
|
||||
</Paragraph>
|
||||
</div>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
<Card title="请求头" size="small">
|
||||
<pre style={{ fontSize: 12, margin: 0, maxHeight: 200, overflow: 'auto' }}>
|
||||
{JSON.stringify(detailEntry.request.headers, null, 2)}
|
||||
</pre>
|
||||
</Card>
|
||||
|
||||
{detailEntry.request.body && (
|
||||
<Card title="请求体" size="small">
|
||||
<Paragraph style={{ background: 'rgba(0,0,0,0.04)', padding: 8, borderRadius: 4, whiteSpace: 'pre-wrap', fontSize: 12, margin: 0 }}>
|
||||
{typeof detailEntry.request.body === 'object'
|
||||
? JSON.stringify(detailEntry.request.body, null, 2)
|
||||
: detailEntry.request.body}
|
||||
</Paragraph>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{Object.keys(detailEntry.response.headers || {}).length > 0 && (
|
||||
<Card title="响应头" size="small">
|
||||
<pre style={{ fontSize: 12, margin: 0, maxHeight: 200, overflow: 'auto' }}>
|
||||
{JSON.stringify(detailEntry.response.headers, null, 2)}
|
||||
</pre>
|
||||
</Card>
|
||||
)}
|
||||
</Space>
|
||||
)}
|
||||
</Drawer>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user