feat(logging): improve local and docker log handling
This commit is contained in:
yml2213
+156
-31
@@ -1,15 +1,23 @@
|
||||
"""日志配置模块"""
|
||||
"""应用日志配置：统一文件位置、脱敏与按日期/大小轮转。"""
|
||||
|
||||
import sys
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from logging.handlers import TimedRotatingFileHandler
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from logging.handlers import BaseRotatingHandler
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
_LOG_LEVELS = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}
|
||||
_DEFAULT_RETENTION_DAYS = 14
|
||||
_DEFAULT_ROTATION_SIZE = 50 * 1024 * 1024
|
||||
|
||||
|
||||
def _normalize_log_level(level: str | None) -> str:
|
||||
@@ -18,6 +26,59 @@ def _normalize_log_level(level: str | None) -> str:
|
||||
return normalized if normalized in _LOG_LEVELS else "INFO"
|
||||
|
||||
|
||||
def _parse_positive_int(value: str | None, default: int) -> int:
|
||||
try:
|
||||
parsed = int(str(value or default).strip())
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return parsed if parsed > 0 else default
|
||||
|
||||
|
||||
def _parse_size(value: str | None, default: int = _DEFAULT_ROTATION_SIZE) -> int:
|
||||
"""解析 50M、1GiB 等易读大小；非法值保持安全默认值。"""
|
||||
matched = re.fullmatch(r"\s*(\d+)\s*([kmgt]?i?b?)?\s*", str(value or ""), re.I)
|
||||
if not matched:
|
||||
return default
|
||||
amount = int(matched.group(1))
|
||||
if amount <= 0:
|
||||
return default
|
||||
unit = (matched.group(2) or "b").lower().rstrip("b").rstrip("i")
|
||||
power = {"": 0, "k": 1, "m": 2, "g": 3, "t": 4}.get(unit)
|
||||
return amount * (1024**power) if power is not None else default
|
||||
|
||||
|
||||
class _SensitiveDataFilter(logging.Filter):
|
||||
"""在日志离开进程前移除 Cookie、令牌和常见密钥值。"""
|
||||
|
||||
_HEADER_RE = re.compile(
|
||||
r"\b(cookie|set-cookie|authorization|proxy-authorization)\s*([:=])\s*[^\r\n]+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_KEY_VALUE_RE = re.compile(
|
||||
r"(?P<prefix>(?:[?&,\s{\[]|\")"
|
||||
r"(?:access[_-]?token|refresh[_-]?token|id[_-]?token|token|password|passwd|"
|
||||
r"secret|api[_-]?key|app[_-]?key|cookie)"
|
||||
r"(?:\"|\])?\s*[=:]\s*(?:\"|')?)"
|
||||
r"(?P<value>[^&\s,}\]\\\"']+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
|
||||
_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b")
|
||||
|
||||
@classmethod
|
||||
def redact(cls, message: str) -> str:
|
||||
message = cls._HEADER_RE.sub(lambda m: f"{m.group(1)}{m.group(2)} [REDACTED]", message)
|
||||
message = cls._KEY_VALUE_RE.sub(lambda m: f"{m.group('prefix')}[REDACTED]", message)
|
||||
message = cls._BEARER_RE.sub("Bearer [REDACTED]", message)
|
||||
return cls._JWT_RE.sub("[REDACTED_JWT]", message)
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
# 先展开 args，再清空 args，确保 Formatter 不会重新格式化出敏感值。
|
||||
record.msg = self.redact(record.getMessage())
|
||||
record.args = ()
|
||||
return True
|
||||
|
||||
|
||||
class _RelevantAccessFilter(logging.Filter):
|
||||
"""仅保留异常 HTTP 请求，避免前端轮询淹没业务日志。"""
|
||||
|
||||
@@ -30,67 +91,131 @@ class _RelevantAccessFilter(logging.Filter):
|
||||
return bool(matched and int(matched.group(1)) >= 400)
|
||||
|
||||
|
||||
def _configure_standard_logging(level: str, log_dir: Path) -> logging.Logger:
|
||||
"""将业务日志和关键框架日志收敛到单一轮转文件。"""
|
||||
class _SizeAndDayRotatingFileHandler(BaseRotatingHandler):
|
||||
"""写入日期文件；跨日切换文件，按大小压缩归档。"""
|
||||
|
||||
def __init__(self, filename: Path, max_bytes: int, retention_days: int) -> None:
|
||||
super().__init__(str(filename), "a", encoding="utf-8", delay=True)
|
||||
self.max_bytes = max_bytes
|
||||
self.retention_days = retention_days
|
||||
self._active_day = datetime.now().date()
|
||||
path = Path(filename)
|
||||
self._log_dir = path.parent
|
||||
self._suffix = path.suffix
|
||||
self._filename_prefix = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", path.stem)
|
||||
|
||||
def _path_for_day(self, day) -> Path:
|
||||
return self._log_dir / f"{self._filename_prefix}-{day.isoformat()}{self._suffix}"
|
||||
|
||||
def shouldRollover(self, record: logging.LogRecord) -> bool: # noqa: N802
|
||||
if datetime.now().date() != self._active_day:
|
||||
return True
|
||||
if self.stream is None:
|
||||
self.stream = self._open()
|
||||
self.stream.seek(0, os.SEEK_END)
|
||||
message = f"{self.format(record)}\n"
|
||||
return self.stream.tell() + len(message.encode("utf-8")) >= self.max_bytes
|
||||
|
||||
def doRollover(self) -> None: # noqa: N802
|
||||
if self.stream is not None:
|
||||
self.stream.close()
|
||||
self.stream = None
|
||||
source = Path(self.baseFilename)
|
||||
current_day = datetime.now().date()
|
||||
if current_day != self._active_day:
|
||||
# 每日文件本身已带日期，跨日时直接切换到新文件，无需再移动旧文件。
|
||||
self.baseFilename = os.fspath(self._path_for_day(current_day).resolve())
|
||||
self._active_day = current_day
|
||||
self._delete_expired_archives()
|
||||
return
|
||||
if source.exists() and source.stat().st_size:
|
||||
stamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
archive = source.with_name(f"{source.name}.{stamp}.{os.getpid()}.gz")
|
||||
sequence = 1
|
||||
while archive.exists():
|
||||
archive = source.with_name(f"{source.name}.{stamp}.{os.getpid()}.{sequence}.gz")
|
||||
sequence += 1
|
||||
with source.open("rb") as raw, gzip.open(archive, "wb") as compressed:
|
||||
shutil.copyfileobj(raw, compressed)
|
||||
source.unlink()
|
||||
self._delete_expired_archives()
|
||||
|
||||
def _delete_expired_archives(self) -> None:
|
||||
cutoff = datetime.now() - timedelta(days=self.retention_days)
|
||||
for archive in self._log_dir.glob(f"{self._filename_prefix}-*.log*.gz"):
|
||||
try:
|
||||
if datetime.fromtimestamp(archive.stat().st_mtime) < cutoff:
|
||||
archive.unlink()
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
|
||||
def _close_handlers(target: logging.Logger) -> None:
|
||||
for handler in target.handlers[:]:
|
||||
target.removeHandler(handler)
|
||||
handler.close()
|
||||
|
||||
|
||||
def _configure_standard_logging(level: str, log_path: Path) -> logging.Logger:
|
||||
"""将业务日志和关键框架日志收敛到单一安全轮转文件。"""
|
||||
log_level = getattr(logging, level.upper(), logging.INFO)
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s | %(levelname)-8s | %(name)s | %(message)s"
|
||||
)
|
||||
framework_level = max(log_level, logging.INFO)
|
||||
formatter = logging.Formatter("%(asctime)s | %(levelname)-8s | %(name)s | %(message)s")
|
||||
redaction_filter = _SensitiveDataFilter()
|
||||
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setLevel(log_level)
|
||||
console_handler.setFormatter(formatter)
|
||||
console_handler.addFilter(redaction_filter)
|
||||
|
||||
file_handler = TimedRotatingFileHandler(
|
||||
log_dir / "app.log", when="midnight", backupCount=7, encoding="utf-8"
|
||||
)
|
||||
retention_days = _parse_positive_int(os.getenv("LOG_RETENTION_DAYS"), _DEFAULT_RETENTION_DAYS)
|
||||
rotation_size = _parse_size(os.getenv("LOG_ROTATION_SIZE"))
|
||||
file_handler = _SizeAndDayRotatingFileHandler(log_path, rotation_size, retention_days)
|
||||
file_handler.setLevel(log_level)
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.addFilter(redaction_filter)
|
||||
|
||||
app_logger = logging.getLogger("app")
|
||||
app_logger.handlers.clear()
|
||||
_close_handlers(app_logger)
|
||||
app_logger.handlers = [console_handler, file_handler]
|
||||
app_logger.setLevel(log_level)
|
||||
app_logger.propagate = False
|
||||
|
||||
for name in ("uvicorn", "uvicorn.error", "fastapi", "starlette"):
|
||||
standard_logger = logging.getLogger(name)
|
||||
_close_handlers(standard_logger)
|
||||
standard_logger.handlers = [console_handler, file_handler]
|
||||
standard_logger.setLevel(log_level)
|
||||
# 应用 DEBUG 不应打开 Uvicorn 的 WebSocket 协议/请求头细节。
|
||||
standard_logger.setLevel(framework_level)
|
||||
standard_logger.propagate = False
|
||||
|
||||
access_logger = logging.getLogger("uvicorn.access")
|
||||
_close_handlers(access_logger)
|
||||
access_logger.handlers = [console_handler, file_handler]
|
||||
access_logger.filters = [_RelevantAccessFilter()]
|
||||
access_logger.filters.clear()
|
||||
access_logger.addFilter(_RelevantAccessFilter())
|
||||
access_logger.setLevel(logging.INFO)
|
||||
access_logger.propagate = False
|
||||
return app_logger
|
||||
|
||||
|
||||
def setup_logger(level: str = "INFO", log_dir: str = None, log_file: str = None) -> None:
|
||||
"""
|
||||
配置日志
|
||||
def setup_logger(level: str = "INFO", log_dir: str | None = None, log_file: str | None = None) -> None:
|
||||
"""配置控制台和文件日志。
|
||||
|
||||
Args:
|
||||
level: 日志级别
|
||||
log_dir: 日志目录路径（写入 app.log，按日轮转）
|
||||
log_file: 日志文件路径（兼容旧接口，优先级低于 log_dir）
|
||||
``log_dir`` 默认写入 ``app-YYYY-MM-DD.log``；``log_file`` 可指定完整文件名。环境变量
|
||||
``LOG_ROTATION_SIZE``（默认 50M）和 ``LOG_RETENTION_DAYS``（默认 14）控制归档。
|
||||
"""
|
||||
level = _normalize_log_level(level)
|
||||
# 移除默认 handler，由标准 logging 统一处理控制台和文件输出。
|
||||
logger.remove()
|
||||
|
||||
# 确定日志文件路径
|
||||
if log_dir:
|
||||
dir_path = Path(log_dir)
|
||||
dir_path.mkdir(parents=True, exist_ok=True)
|
||||
app_logger = _configure_standard_logging(level, dir_path)
|
||||
elif log_file:
|
||||
file_path = Path(log_file)
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
app_logger = _configure_standard_logging(level, file_path.parent)
|
||||
if log_file:
|
||||
file_path = Path(log_file).expanduser()
|
||||
elif log_dir:
|
||||
file_path = Path(log_dir).expanduser() / f"app-{datetime.now():%Y-%m-%d}.log"
|
||||
else:
|
||||
return
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
app_logger = _configure_standard_logging(level, file_path)
|
||||
|
||||
def forward_to_standard(message) -> None:
|
||||
record = message.record
|
||||
|
||||
Reference in New Issue
Block a user