优化生产日志体积:访问降噪、压缩清理与退款告警节流
跳过轮询/封面等热路径成功请求,按日 gzip 并保留可配天数,退款 max-retry 汇总告警。
This commit is contained in:
@@ -1,27 +1,35 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type dailyWriter struct {
|
||||
mu sync.Mutex
|
||||
dir string
|
||||
prefix string
|
||||
location *time.Location
|
||||
day string
|
||||
file *os.File
|
||||
mu sync.Mutex
|
||||
dir string
|
||||
prefix string
|
||||
location *time.Location
|
||||
retainDays int
|
||||
day string
|
||||
file *os.File
|
||||
}
|
||||
|
||||
func newDailyWriter(dir string, prefix string, location *time.Location) *dailyWriter {
|
||||
func newDailyWriter(dir string, prefix string, location *time.Location, retainDays int) *dailyWriter {
|
||||
if retainDays < 0 {
|
||||
retainDays = 0
|
||||
}
|
||||
return &dailyWriter{
|
||||
dir: dir,
|
||||
prefix: prefix,
|
||||
location: location,
|
||||
dir: dir,
|
||||
prefix: prefix,
|
||||
location: location,
|
||||
retainDays: retainDays,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +63,7 @@ func (w *dailyWriter) rotateIfNeeded(now time.Time) error {
|
||||
return err
|
||||
}
|
||||
|
||||
prevDay := w.day
|
||||
if w.file != nil {
|
||||
_ = w.file.Close()
|
||||
w.file = nil
|
||||
@@ -67,5 +76,94 @@ func (w *dailyWriter) rotateIfNeeded(now time.Time) error {
|
||||
}
|
||||
w.file = file
|
||||
w.day = day
|
||||
|
||||
// 异步压缩昨日日志并清理过期文件,避免阻塞业务写路径
|
||||
if prevDay != "" && prevDay != day {
|
||||
go w.maintain(prevDay, now)
|
||||
} else {
|
||||
// 进程启动或首写:尝试压缩昨天 + 清理过期
|
||||
yesterday := now.AddDate(0, 0, -1).Format("2006-01-02")
|
||||
go w.maintain(yesterday, now)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *dailyWriter) maintain(prevDay string, now time.Time) {
|
||||
if prevDay != "" {
|
||||
_ = w.compressDay(prevDay)
|
||||
}
|
||||
if w.retainDays > 0 {
|
||||
_ = w.purgeOlderThan(now)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *dailyWriter) compressDay(day string) error {
|
||||
srcPath := filepath.Join(w.dir, fmt.Sprintf("%s-%s.log", w.prefix, day))
|
||||
dstPath := srcPath + ".gz"
|
||||
if _, err := os.Stat(dstPath); err == nil {
|
||||
// 已压缩则删除明文(若仍存在)
|
||||
_ = os.Remove(srcPath)
|
||||
return nil
|
||||
}
|
||||
src, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gz := gzip.NewWriter(dst)
|
||||
if _, err := io.Copy(gz, src); err != nil {
|
||||
_ = gz.Close()
|
||||
_ = dst.Close()
|
||||
_ = os.Remove(dstPath)
|
||||
return err
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
_ = dst.Close()
|
||||
_ = os.Remove(dstPath)
|
||||
return err
|
||||
}
|
||||
if err := dst.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Remove(srcPath)
|
||||
}
|
||||
|
||||
func (w *dailyWriter) purgeOlderThan(now time.Time) error {
|
||||
// 按「日历天」比较:保留最近 retainDays 天(含当天),更早的 .log / .log.gz 删除。
|
||||
today := time.Date(now.In(w.location).Year(), now.In(w.location).Month(), now.In(w.location).Day(), 0, 0, 0, 0, w.location)
|
||||
cutoff := today.AddDate(0, 0, -w.retainDays) // day < cutoff 才删除;retainDays=2 且今天 12 号 → 删除 10 号之前
|
||||
entries, err := os.ReadDir(w.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prefix := w.prefix + "-"
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if !strings.HasPrefix(name, prefix) {
|
||||
continue
|
||||
}
|
||||
// app-2026-07-11.log 或 app-2026-07-11.log.gz
|
||||
rest := strings.TrimPrefix(name, prefix)
|
||||
rest = strings.TrimSuffix(rest, ".gz")
|
||||
rest = strings.TrimSuffix(rest, ".log")
|
||||
day, err := time.ParseInLocation("2006-01-02", rest, w.location)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if day.Before(cutoff) {
|
||||
_ = os.Remove(filepath.Join(w.dir, name))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCompressDayAndPurge(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*60*60)
|
||||
w := newDailyWriter(dir, "app", loc, 2)
|
||||
|
||||
// 准备三天明文日志
|
||||
days := []string{"2026-07-01", "2026-07-10", "2026-07-11"}
|
||||
for _, day := range days {
|
||||
path := filepath.Join(dir, "app-"+day+".log")
|
||||
if err := os.WriteFile(path, []byte(`{"msg":"hello `+day+`"}`+"\n"), 0o644); err != nil {
|
||||
t.Fatalf("write log: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 压缩 7-10
|
||||
if err := w.compressDay("2026-07-10"); err != nil {
|
||||
t.Fatalf("compressDay: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "app-2026-07-10.log")); !os.IsNotExist(err) {
|
||||
t.Fatalf("plain log should be removed after gzip, err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "app-2026-07-10.log.gz")); err != nil {
|
||||
t.Fatalf("gzip missing: %v", err)
|
||||
}
|
||||
|
||||
// 以 7-12 为「现在」,保留 2 天 → cutoff=7-10,删除 7-10 之前(仅 7-01)
|
||||
now := time.Date(2026, 7, 12, 12, 0, 0, 0, loc)
|
||||
if err := w.purgeOlderThan(now); err != nil {
|
||||
t.Fatalf("purgeOlderThan: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "app-2026-07-01.log")); !os.IsNotExist(err) {
|
||||
t.Fatal("expired log should be purged")
|
||||
}
|
||||
// 7-10 / 7-11 仍应存在(cutoff 当天起保留)
|
||||
if _, err := os.Stat(filepath.Join(dir, "app-2026-07-10.log.gz")); err != nil {
|
||||
t.Fatalf("recent gzip should remain: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "app-2026-07-11.log")); err != nil {
|
||||
t.Fatalf("recent plain should remain: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressDayIdempotent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
loc := time.UTC
|
||||
w := newDailyWriter(dir, "app", loc, 0)
|
||||
path := filepath.Join(dir, "app-2026-07-11.log")
|
||||
if err := os.WriteFile(path, []byte("line\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.compressDay("2026-07-11"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.compressDay("2026-07-11"); err != nil {
|
||||
t.Fatalf("second compress should be no-op: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,7 @@ func New(cfg config.LogConfig) (*zap.Logger, error) {
|
||||
))
|
||||
}
|
||||
if cfg.EnableFile {
|
||||
writer := newDailyWriter(cfg.Dir, "app", location)
|
||||
writer := newDailyWriter(cfg.Dir, "app", location, cfg.RetainDays)
|
||||
cores = append(cores, zapcore.NewCore(
|
||||
zapcore.NewJSONEncoder(encoderConfig),
|
||||
writer,
|
||||
|
||||
Reference in New Issue
Block a user