yml2213
453a637480
安全修复: - WebSocket 端点添加认证（cookie/token），防止未授权窃听日志 - SPA serve_spa 添加路径遍历防护（resolve + relative_to 检查） - Token 改用 httpOnly Cookie 存储，移除前端 localStorage token（防 XSS 窃取） - 添加安全响应头中间件（X-Content-Type-Options/X-Frame-Options/Referrer-Policy） - HTTP 请求日志脱敏请求体中的 password/secret/token 等敏感字段 - 权限检查统一使用 user_has_permission（考虑自定义权限，修复 has_permission 忽略 custom_permissions 的缺陷） 性能与稳定性: - cookies.py 列表接口修复 N+1 查询（改为批量查询 Account） - login_service.py run() 结束时关闭 DB Session（防止连接泄漏） - _active_batches/_active_tests 全局字典添加 threading.Lock（防止并发竞态） 配置优化: - CORS 源支持环境变量 CORS_ORIGINS 配置 - Uvicorn reload 支持环境变量 UVICORN_RELOAD 控制（生产环境默认关闭） - Cookie 安全标志支持环境变量 COOKIE_SECURE 配置（HTTPS 部署时启用） - logs.py 权限不足返回 HTTP 403（原来返回 200 + message）
242 lines
7.2 KiB
Python
242 lines
7.2 KiB
Python
"""HTTP 请求/响应详情日志记录器
|
||
|
||
将关键 HTTP 请求的完整详情（method、url、headers、body、status、response、耗时）
|
||
以 JSON Lines 格式写入日志文件，便于在 Web 界面查看和排查问题。
|
||
"""
|
||
|
||
import json
|
||
import threading
|
||
import time
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Optional
|
||
|
||
from loguru import logger
|
||
|
||
# 日志文件路径
|
||
_LOG_DIR = Path("logs")
|
||
_LOG_FILE = _LOG_DIR / "http_detail.jsonl"
|
||
_MAX_BODY_LEN = 2000 # 单个 body 最大记录长度，避免过大
|
||
|
||
# 线程安全写锁
|
||
_write_lock = threading.Lock()
|
||
|
||
# 确保日志目录存在
|
||
_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
|
||
def _truncate(text: Any, max_len: int = _MAX_BODY_LEN) -> str:
|
||
"""截断过长的文本"""
|
||
if text is None:
|
||
return ""
|
||
s = text if isinstance(text, str) else str(text)
|
||
if len(s) > max_len:
|
||
return s[:max_len] + f"...[truncated {len(s) - max_len} chars]"
|
||
return s
|
||
|
||
|
||
def _safe_headers(headers: Any) -> dict:
|
||
"""清理 headers 中的敏感信息"""
|
||
if not headers:
|
||
return {}
|
||
if hasattr(headers, 'items'):
|
||
headers = dict(headers)
|
||
safe = {}
|
||
sensitive = {'authorization', 'cookie', 'set-cookie', 'password'}
|
||
for k, v in headers.items():
|
||
if k.lower() in sensitive:
|
||
safe[k] = '***'
|
||
else:
|
||
safe[k] = v
|
||
return safe
|
||
|
||
|
||
# 请求体中需要脱敏的字段名（小写匹配，包含即遮罩）
|
||
_SENSITIVE_BODY_KEYS = {
|
||
'password', 'pwd', 'passwd', 'secret', 'token', 'apikey', 'api_key',
|
||
'email_password', 'mm', 'authorization', 'credential',
|
||
}
|
||
|
||
|
||
def _safe_body(body: Any) -> str:
|
||
"""清理请求体中的敏感字段值。
|
||
|
||
支持 dict、JSON 字符串、其他类型。
|
||
敏感字段的值会被替换为 ***，其余内容保留（仍受 _truncate 限制）。
|
||
"""
|
||
if body is None:
|
||
return ""
|
||
|
||
# 尝试解析 JSON 字符串
|
||
parsed = body
|
||
if isinstance(body, str):
|
||
try:
|
||
parsed = json.loads(body)
|
||
except (json.JSONDecodeError, ValueError):
|
||
# 非 JSON 字符串，直接截断
|
||
return _truncate(body)
|
||
|
||
# dict 类型：遮罩敏感字段
|
||
if isinstance(parsed, dict):
|
||
safe = {}
|
||
for k, v in parsed.items():
|
||
if any(s in k.lower() for s in _SENSITIVE_BODY_KEYS):
|
||
safe[k] = '***'
|
||
elif isinstance(v, (dict, list)):
|
||
safe[k] = _safe_body(v) if isinstance(v, dict) else _truncate(str(v))
|
||
else:
|
||
safe[k] = v
|
||
return _truncate(json.dumps(safe, ensure_ascii=False))
|
||
|
||
# 其他类型：直接截断
|
||
return _truncate(str(body))
|
||
|
||
|
||
def log_http(
|
||
category: str,
|
||
method: str,
|
||
url: str,
|
||
*,
|
||
request_headers: Any = None,
|
||
request_body: Any = None,
|
||
status_code: Optional[int] = None,
|
||
response_headers: Any = None,
|
||
response_body: Any = None,
|
||
duration: Optional[float] = None,
|
||
error: Optional[str] = None,
|
||
proxy: Optional[str] = None,
|
||
tag: str = "",
|
||
) -> None:
|
||
"""
|
||
记录一条 HTTP 请求/响应详情日志。
|
||
|
||
Args:
|
||
category: 分类（如 'douyu_login', 'geetest', 'proxy_verify', 'whitelist'）
|
||
method: HTTP 方法
|
||
url: 请求 URL（会被脱敏，移除敏感查询参数）
|
||
request_headers: 请求头
|
||
request_body: 请求体
|
||
status_code: 响应状态码
|
||
response_headers: 响应头
|
||
response_body: 响应体
|
||
duration: 耗时（秒）
|
||
error: 错误信息
|
||
proxy: 使用的代理
|
||
tag: 额外标签（如账号名）
|
||
"""
|
||
entry = {
|
||
"timestamp": datetime.now().isoformat(timespec="milliseconds"),
|
||
"ts": time.time(),
|
||
"category": category,
|
||
"tag": tag,
|
||
"method": method.upper(),
|
||
"url": _truncate(url, 500),
|
||
"proxy": proxy,
|
||
"request": {
|
||
"headers": _safe_headers(request_headers),
|
||
"body": _safe_body(request_body),
|
||
},
|
||
"response": {
|
||
"status_code": status_code,
|
||
"headers": _safe_headers(response_headers),
|
||
"body": _truncate(response_body),
|
||
},
|
||
"duration_ms": round(duration * 1000, 1) if duration is not None else None,
|
||
"error": _truncate(error, 500) if error else None,
|
||
}
|
||
|
||
# 判断级别
|
||
if error or (status_code and status_code >= 400):
|
||
entry["level"] = "error"
|
||
elif status_code and status_code >= 300:
|
||
entry["level"] = "warning"
|
||
else:
|
||
entry["level"] = "info"
|
||
|
||
try:
|
||
line = json.dumps(entry, ensure_ascii=False)
|
||
with _write_lock:
|
||
with open(_LOG_FILE, "a", encoding="utf-8") as f:
|
||
f.write(line + "\n")
|
||
except Exception as e:
|
||
logger.debug(f"写入HTTP详情日志失败: {e}")
|
||
|
||
|
||
def read_http_logs(
|
||
*,
|
||
limit: int = 100,
|
||
offset: int = 0,
|
||
category: Optional[str] = None,
|
||
level: Optional[str] = None,
|
||
keyword: Optional[str] = None,
|
||
) -> tuple[list[dict], int]:
|
||
"""
|
||
读取 HTTP 详情日志，支持筛选和分页。
|
||
|
||
Args:
|
||
limit: 返回条数上限
|
||
offset: 偏移量（从最新往前数）
|
||
category: 按分类筛选
|
||
level: 按级别筛选（info/warning/error）
|
||
keyword: 关键词搜索（url、tag、error）
|
||
|
||
Returns:
|
||
(日志条目列表, 总匹配条数)，列表按时间倒序（最新在前）
|
||
"""
|
||
if not _LOG_FILE.exists():
|
||
return [], 0
|
||
|
||
entries: list[dict] = []
|
||
try:
|
||
with open(_LOG_FILE, "r", encoding="utf-8") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
entry = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
|
||
# 筛选
|
||
if category and entry.get("category") != category:
|
||
continue
|
||
if level and entry.get("level") != level:
|
||
continue
|
||
if keyword:
|
||
kw = keyword.lower()
|
||
searchable = " ".join([
|
||
str(entry.get("url", "")),
|
||
str(entry.get("tag", "")),
|
||
str(entry.get("error", "")),
|
||
str(entry.get("method", "")),
|
||
]).lower()
|
||
if kw not in searchable:
|
||
continue
|
||
|
||
entries.append(entry)
|
||
except Exception as e:
|
||
logger.error(f"读取HTTP详情日志失败: {e}")
|
||
return [], 0
|
||
|
||
# 按时间倒序
|
||
entries.sort(key=lambda x: x.get("ts", 0), reverse=True)
|
||
total = len(entries)
|
||
# 分页（offset 从最新开始算）
|
||
page = entries[offset:offset + limit]
|
||
return page, total
|
||
|
||
|
||
def clear_http_logs() -> int:
|
||
"""清空 HTTP 详情日志，返回清空的条数"""
|
||
count = 0
|
||
with _write_lock:
|
||
if _LOG_FILE.exists():
|
||
try:
|
||
with open(_LOG_FILE, "r", encoding="utf-8") as f:
|
||
count = sum(1 for line in f if line.strip())
|
||
except Exception:
|
||
pass
|
||
_LOG_FILE.write_text("", encoding="utf-8")
|
||
return count
|