241 lines
9.0 KiB
Python
241 lines
9.0 KiB
Python
"""应用日志配置：统一文件位置、脱敏与按日期/大小轮转。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import gzip
|
||
import logging
|
||
import os
|
||
import re
|
||
import shutil
|
||
import sys
|
||
from datetime import datetime, timedelta
|
||
from logging.handlers import BaseRotatingHandler
|
||
from pathlib import Path
|
||
|
||
from loguru import logger
|
||
|
||
|
||
_LOG_LEVELS = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}
|
||
_DEFAULT_RETENTION_DAYS = 14
|
||
_DEFAULT_ROTATION_SIZE = 50 * 1024 * 1024
|
||
|
||
|
||
def _normalize_log_level(level: str | None) -> str:
|
||
"""规范化环境变量日志等级，非法值回退到 INFO。"""
|
||
normalized = str(level or "INFO").strip().upper()
|
||
return normalized if normalized in _LOG_LEVELS else "INFO"
|
||
|
||
|
||
def _parse_positive_int(value: str | None, default: int) -> int:
|
||
try:
|
||
parsed = int(str(value or default).strip())
|
||
except (TypeError, ValueError):
|
||
return default
|
||
return parsed if parsed > 0 else default
|
||
|
||
|
||
def _parse_size(value: str | None, default: int = _DEFAULT_ROTATION_SIZE) -> int:
|
||
"""解析 50M、1GiB 等易读大小；非法值保持安全默认值。"""
|
||
matched = re.fullmatch(r"\s*(\d+)\s*([kmgt]?i?b?)?\s*", str(value or ""), re.I)
|
||
if not matched:
|
||
return default
|
||
amount = int(matched.group(1))
|
||
if amount <= 0:
|
||
return default
|
||
unit = (matched.group(2) or "b").lower().rstrip("b").rstrip("i")
|
||
power = {"": 0, "k": 1, "m": 2, "g": 3, "t": 4}.get(unit)
|
||
return amount * (1024**power) if power is not None else default
|
||
|
||
|
||
class _SensitiveDataFilter(logging.Filter):
|
||
"""在日志离开进程前移除 Cookie、令牌和常见密钥值。"""
|
||
|
||
_HEADER_RE = re.compile(
|
||
r"\b(cookie|set-cookie|authorization|proxy-authorization)\s*([:=])\s*[^\r\n]+",
|
||
re.IGNORECASE,
|
||
)
|
||
_KEY_VALUE_RE = re.compile(
|
||
r"(?P<prefix>(?:[?&,\s{\[]|\")"
|
||
r"(?:access[_-]?token|refresh[_-]?token|id[_-]?token|token|password|passwd|"
|
||
r"secret|api[_-]?key|app[_-]?key|cookie)"
|
||
r"(?:\"|\])?\s*[=:]\s*(?:\"|')?)"
|
||
r"(?P<value>[^&\s,}\]\\\"']+)",
|
||
re.IGNORECASE,
|
||
)
|
||
_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
|
||
_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b")
|
||
|
||
@classmethod
|
||
def redact(cls, message: str) -> str:
|
||
message = cls._HEADER_RE.sub(
|
||
lambda m: f"{m.group(1)}{m.group(2)} [REDACTED]", message
|
||
)
|
||
message = cls._KEY_VALUE_RE.sub(
|
||
lambda m: f"{m.group('prefix')}[REDACTED]", message
|
||
)
|
||
message = cls._BEARER_RE.sub("Bearer [REDACTED]", message)
|
||
return cls._JWT_RE.sub("[REDACTED_JWT]", message)
|
||
|
||
def filter(self, record: logging.LogRecord) -> bool:
|
||
# 先展开 args，再清空 args，确保 Formatter 不会重新格式化出敏感值。
|
||
record.msg = self.redact(record.getMessage())
|
||
record.args = ()
|
||
return True
|
||
|
||
|
||
class _RelevantAccessFilter(logging.Filter):
|
||
"""仅保留异常 HTTP 请求，避免前端轮询淹没业务日志。"""
|
||
|
||
_STATUS_RE = re.compile(r'"\s+(\d{3})\b')
|
||
|
||
def filter(self, record: logging.LogRecord) -> bool:
|
||
if record.levelno >= logging.WARNING:
|
||
return True
|
||
matched = self._STATUS_RE.search(record.getMessage())
|
||
return bool(matched and int(matched.group(1)) >= 400)
|
||
|
||
|
||
class _SizeAndDayRotatingFileHandler(BaseRotatingHandler):
|
||
"""写入日期文件；跨日切换文件，按大小压缩归档。"""
|
||
|
||
def __init__(self, filename: Path, max_bytes: int, retention_days: int) -> None:
|
||
super().__init__(str(filename), "a", encoding="utf-8", delay=True)
|
||
self.max_bytes = max_bytes
|
||
self.retention_days = retention_days
|
||
self._active_day = datetime.now().date()
|
||
path = Path(filename)
|
||
self._log_dir = path.parent
|
||
self._suffix = path.suffix
|
||
self._filename_prefix = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", path.stem)
|
||
|
||
def _path_for_day(self, day) -> Path:
|
||
return (
|
||
self._log_dir / f"{self._filename_prefix}-{day.isoformat()}{self._suffix}"
|
||
)
|
||
|
||
def shouldRollover(self, record: logging.LogRecord) -> bool: # noqa: N802
|
||
if datetime.now().date() != self._active_day:
|
||
return True
|
||
if self.stream is None:
|
||
self.stream = self._open()
|
||
self.stream.seek(0, os.SEEK_END)
|
||
message = f"{self.format(record)}\n"
|
||
return self.stream.tell() + len(message.encode("utf-8")) >= self.max_bytes
|
||
|
||
def doRollover(self) -> None: # noqa: N802
|
||
if self.stream is not None:
|
||
self.stream.close()
|
||
self.stream = None
|
||
source = Path(self.baseFilename)
|
||
current_day = datetime.now().date()
|
||
if current_day != self._active_day:
|
||
# 每日文件本身已带日期，跨日时直接切换到新文件，无需再移动旧文件。
|
||
self.baseFilename = os.fspath(self._path_for_day(current_day).resolve())
|
||
self._active_day = current_day
|
||
self._delete_expired_archives()
|
||
return
|
||
if source.exists() and source.stat().st_size:
|
||
stamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||
archive = source.with_name(f"{source.name}.{stamp}.{os.getpid()}.gz")
|
||
sequence = 1
|
||
while archive.exists():
|
||
archive = source.with_name(
|
||
f"{source.name}.{stamp}.{os.getpid()}.{sequence}.gz"
|
||
)
|
||
sequence += 1
|
||
with source.open("rb") as raw, gzip.open(archive, "wb") as compressed:
|
||
shutil.copyfileobj(raw, compressed)
|
||
source.unlink()
|
||
self._delete_expired_archives()
|
||
|
||
def _delete_expired_archives(self) -> None:
|
||
cutoff = datetime.now() - timedelta(days=self.retention_days)
|
||
for archive in self._log_dir.glob(f"{self._filename_prefix}-*.log*.gz"):
|
||
try:
|
||
if datetime.fromtimestamp(archive.stat().st_mtime) < cutoff:
|
||
archive.unlink()
|
||
except OSError:
|
||
continue
|
||
|
||
|
||
def _close_handlers(target: logging.Logger) -> None:
|
||
for handler in target.handlers[:]:
|
||
target.removeHandler(handler)
|
||
handler.close()
|
||
|
||
|
||
def _configure_standard_logging(level: str, log_path: Path) -> logging.Logger:
|
||
"""将业务日志和关键框架日志收敛到单一安全轮转文件。"""
|
||
log_level = getattr(logging, level.upper(), logging.INFO)
|
||
framework_level = max(log_level, logging.INFO)
|
||
formatter = logging.Formatter(
|
||
"%(asctime)s | %(levelname)-8s | %(name)s | %(message)s"
|
||
)
|
||
redaction_filter = _SensitiveDataFilter()
|
||
|
||
console_handler = logging.StreamHandler(sys.stdout)
|
||
console_handler.setLevel(log_level)
|
||
console_handler.setFormatter(formatter)
|
||
console_handler.addFilter(redaction_filter)
|
||
|
||
retention_days = _parse_positive_int(
|
||
os.getenv("LOG_RETENTION_DAYS"), _DEFAULT_RETENTION_DAYS
|
||
)
|
||
rotation_size = _parse_size(os.getenv("LOG_ROTATION_SIZE"))
|
||
file_handler = _SizeAndDayRotatingFileHandler(
|
||
log_path, rotation_size, retention_days
|
||
)
|
||
file_handler.setLevel(log_level)
|
||
file_handler.setFormatter(formatter)
|
||
file_handler.addFilter(redaction_filter)
|
||
|
||
app_logger = logging.getLogger("app")
|
||
_close_handlers(app_logger)
|
||
app_logger.handlers = [console_handler, file_handler]
|
||
app_logger.setLevel(log_level)
|
||
app_logger.propagate = False
|
||
|
||
for name in ("uvicorn", "uvicorn.error", "fastapi", "starlette"):
|
||
standard_logger = logging.getLogger(name)
|
||
_close_handlers(standard_logger)
|
||
standard_logger.handlers = [console_handler, file_handler]
|
||
# 应用 DEBUG 不应打开 Uvicorn 的 WebSocket 协议/请求头细节。
|
||
standard_logger.setLevel(framework_level)
|
||
standard_logger.propagate = False
|
||
|
||
access_logger = logging.getLogger("uvicorn.access")
|
||
_close_handlers(access_logger)
|
||
access_logger.handlers = [console_handler, file_handler]
|
||
access_logger.filters.clear()
|
||
access_logger.addFilter(_RelevantAccessFilter())
|
||
access_logger.setLevel(logging.INFO)
|
||
access_logger.propagate = False
|
||
return app_logger
|
||
|
||
|
||
def setup_logger(
|
||
level: str = "INFO", log_dir: str | None = None, log_file: str | None = None
|
||
) -> None:
|
||
"""配置控制台和文件日志。
|
||
|
||
``log_dir`` 默认写入 ``app-YYYY-MM-DD.log``；``log_file`` 可指定完整文件名。环境变量
|
||
``LOG_ROTATION_SIZE``（默认 50M）和 ``LOG_RETENTION_DAYS``（默认 14）控制归档。
|
||
"""
|
||
level = _normalize_log_level(level)
|
||
logger.remove()
|
||
|
||
if log_file:
|
||
file_path = Path(log_file).expanduser()
|
||
elif log_dir:
|
||
file_path = Path(log_dir).expanduser() / f"app-{datetime.now():%Y-%m-%d}.log"
|
||
else:
|
||
return
|
||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||
app_logger = _configure_standard_logging(level, file_path)
|
||
|
||
def forward_to_standard(message) -> None:
|
||
record = message.record
|
||
app_logger.log(record["level"].no, f"{record['name']} | {record['message']}")
|
||
|
||
logger.add(forward_to_standard, level=level, format="{message}")
|