Files

283 lines
7.3 KiB
Go

package fileuploadcleanup
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"net/url"
"strings"
"time"
"hfb_sys/backend/internal/model"
"github.com/redis/go-redis/v9"
"go.uber.org/zap"
"gorm.io/gorm"
)
const (
cleanupLockKey = "hfb:job:file-upload-cleanup:lock"
cleanupInterval = 24 * time.Hour
cleanupRetention = 30 * 24 * time.Hour
cleanupBatchSize = 200
)
// Job 清理长期未关联业务记录的上传归属,避免临时草稿记录无限增长。
type Job struct {
db *gorm.DB
redis *redis.Client
logger *zap.Logger
instanceID string
}
func New(db *gorm.DB, redisClient *redis.Client, logger *zap.Logger) *Job {
if logger == nil {
logger = zap.NewNop()
}
return &Job{db: db, redis: redisClient, logger: logger, instanceID: newInstanceID()}
}
func (j *Job) Start(ctx context.Context) {
if j == nil || j.db == nil {
return
}
go j.loop(ctx)
}
func (j *Job) loop(ctx context.Context) {
j.run(ctx, time.Now())
ticker := time.NewTicker(cleanupInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
j.logger.Debug("临时文件归属清理任务已停止")
return
case now := <-ticker.C:
j.run(ctx, now)
}
}
}
func (j *Job) run(ctx context.Context, now time.Time) {
release, ok := j.acquireLock(ctx)
if !ok {
return
}
defer release()
deleted, err := j.cleanup(ctx, now)
if err != nil {
j.logger.Warn("临时文件归属清理失败", zap.Error(err))
return
}
if deleted > 0 {
j.logger.Info("已清理未关联临时文件归属", zap.Int("count", deleted))
}
}
func (j *Job) acquireLock(ctx context.Context) (func(), bool) {
if j.redis == nil {
return func() {}, true
}
ok, err := j.redis.SetNX(ctx, cleanupLockKey, j.instanceID, 10*time.Minute).Result()
if err != nil {
j.logger.Warn("临时文件归属清理任务获取锁失败", zap.Error(err))
return func() {}, true
}
if !ok {
return nil, false
}
return func() {
releaseCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
script := redis.NewScript(`if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end`)
if err := script.Run(releaseCtx, j.redis, []string{cleanupLockKey}, j.instanceID).Err(); err != nil {
j.logger.Warn("临时文件归属清理任务释放锁失败", zap.Error(err))
}
}, true
}
func (j *Job) cleanup(ctx context.Context, now time.Time) (int, error) {
cutoff := now.Add(-cleanupRetention)
lastID := uint64(0)
deleted := 0
for {
var records []model.FileUploadOwner
if err := j.db.WithContext(ctx).
Where("id > ? AND created_at < ?", lastID, cutoff).
Order("id ASC").
Limit(cleanupBatchSize).
Find(&records).Error; err != nil {
return deleted, err
}
if len(records) == 0 {
return deleted, nil
}
for _, record := range records {
lastID = record.ID
referenced, err := j.isReferenced(ctx, record.ObjectKey)
if err != nil {
return deleted, err
}
if referenced {
continue
}
result := j.db.WithContext(ctx).
Where("id = ? AND created_at < ?", record.ID, cutoff).
Delete(&model.FileUploadOwner{})
if result.Error != nil {
return deleted, result.Error
}
deleted += int(result.RowsAffected)
}
}
}
func (j *Job) isReferenced(ctx context.Context, key string) (bool, error) {
values := referenceValues(key)
payload, err := json.Marshal(values)
if err != nil {
return false, err
}
queries := make([]struct {
sql string
args []any
}, 0, 2)
jsonArgs := []any{string(payload)}
switch strings.SplitN(key, "/", 2)[0] {
case "listing", "misc":
queries = append(queries, struct {
sql string
args []any
}{"SELECT COUNT(1) FROM game_accounts WHERE JSON_OVERLAPS(screenshot_urls, CAST(? AS JSON))", jsonArgs})
case "chat":
queries = append(queries, struct {
sql string
args []any
}{"SELECT COUNT(1) FROM chat_messages WHERE JSON_OVERLAPS(attachment_urls, CAST(? AS JSON))", jsonArgs})
case "qrcode", "support-qrcode":
clause, args := scalarIn(values)
queries = append(queries, struct {
sql string
args []any
}{"SELECT COUNT(1) FROM chat_qrcode_pool WHERE image_url IN (" + clause + ")", args})
case "handoff":
queries = append(queries, struct {
sql string
args []any
}{"SELECT COUNT(1) FROM handoff_records WHERE JSON_OVERLAPS(attachment_urls, CAST(? AS JSON))", jsonArgs})
case "dispute":
queries = append(queries,
struct {
sql string
args []any
}{"SELECT COUNT(1) FROM disputes WHERE JSON_OVERLAPS(evidence_urls, CAST(? AS JSON))", jsonArgs},
struct {
sql string
args []any
}{"SELECT COUNT(1) FROM order_checkouts WHERE JSON_OVERLAPS(evidence_urls, CAST(? AS JSON))", jsonArgs},
)
case "payment-cert":
queries = append(queries, struct {
sql string
args []any
}{"SELECT COUNT(1) FROM user_payment_accounts WHERE JSON_OVERLAPS(certificate_urls, CAST(? AS JSON))", jsonArgs})
case "payment-proof":
clause, args := scalarIn(values)
queries = append(queries, struct {
sql string
args []any
}{"SELECT COUNT(1) FROM withdrawal_requests WHERE payment_proof_url IN (" + clause + ")", args})
case "operating-expense":
clause, args := scalarIn(values)
queries = append(queries, struct {
sql string
args []any
}{"SELECT COUNT(1) FROM operating_expenses WHERE voucher_url IN (" + clause + ")", args})
case "manual-disbursement":
clause, args := scalarIn(values)
queries = append(queries, struct {
sql string
args []any
}{"SELECT COUNT(1) FROM manual_disbursements WHERE voucher_url IN (" + clause + ")", args})
default:
return false, nil
}
for _, query := range queries {
var count int64
if err := j.db.WithContext(ctx).Raw(query.sql, query.args...).Scan(&count).Error; err != nil {
return false, err
}
if count > 0 {
return true, nil
}
}
return false, nil
}
func referenceValues(key string) []string {
basePrefix := key
for _, suffix := range []string{".medium.jpg", ".thumb.jpg"} {
if strings.HasSuffix(basePrefix, suffix) {
basePrefix = strings.TrimSuffix(basePrefix, suffix)
break
}
}
candidates := []string{key}
if basePrefix != key {
candidates = append(candidates, basePrefix)
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
candidates = append(candidates, basePrefix+ext)
}
}
seen := make(map[string]struct{}, len(candidates)*8)
values := make([]string, 0, len(candidates)*8)
add := func(value string) {
if value == "" {
return
}
if _, ok := seen[value]; ok {
return
}
seen[value] = struct{}{}
values = append(values, value)
}
for _, candidate := range candidates {
encoded := url.QueryEscape(candidate)
add(candidate)
add(encoded)
for _, prefix := range []string{
"/api/files/object?key=",
"/api/admin/files/object?key=",
"/api/public/files/object?key=",
} {
add(prefix + encoded)
add(prefix + candidate)
}
}
return values
}
func scalarIn(values []string) (string, []any) {
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(values)), ",")
args := make([]any, 0, len(values))
for _, value := range values {
args = append(args, value)
}
return placeholders, args
}
func newInstanceID() string {
value := make([]byte, 8)
if _, err := rand.Read(value); err != nil {
return fmt.Sprintf("file-cleanup-%d", time.Now().UnixNano())
}
return hex.EncodeToString(value)
}