283 lines
7.3 KiB
Go
283 lines
7.3 KiB
Go
package fileuploadcleanup
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"hfb_sys/backend/internal/model"
|
|
|
|
"github.com/redis/go-redis/v9"
|
|
"go.uber.org/zap"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
const (
|
|
cleanupLockKey = "hfb:job:file-upload-cleanup:lock"
|
|
cleanupInterval = 24 * time.Hour
|
|
cleanupRetention = 30 * 24 * time.Hour
|
|
cleanupBatchSize = 200
|
|
)
|
|
|
|
// Job 清理长期未关联业务记录的上传归属,避免临时草稿记录无限增长。
|
|
type Job struct {
|
|
db *gorm.DB
|
|
redis *redis.Client
|
|
logger *zap.Logger
|
|
instanceID string
|
|
}
|
|
|
|
func New(db *gorm.DB, redisClient *redis.Client, logger *zap.Logger) *Job {
|
|
if logger == nil {
|
|
logger = zap.NewNop()
|
|
}
|
|
return &Job{db: db, redis: redisClient, logger: logger, instanceID: newInstanceID()}
|
|
}
|
|
|
|
func (j *Job) Start(ctx context.Context) {
|
|
if j == nil || j.db == nil {
|
|
return
|
|
}
|
|
go j.loop(ctx)
|
|
}
|
|
|
|
func (j *Job) loop(ctx context.Context) {
|
|
j.run(ctx, time.Now())
|
|
ticker := time.NewTicker(cleanupInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
j.logger.Debug("临时文件归属清理任务已停止")
|
|
return
|
|
case now := <-ticker.C:
|
|
j.run(ctx, now)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (j *Job) run(ctx context.Context, now time.Time) {
|
|
release, ok := j.acquireLock(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
defer release()
|
|
deleted, err := j.cleanup(ctx, now)
|
|
if err != nil {
|
|
j.logger.Warn("临时文件归属清理失败", zap.Error(err))
|
|
return
|
|
}
|
|
if deleted > 0 {
|
|
j.logger.Info("已清理未关联临时文件归属", zap.Int("count", deleted))
|
|
}
|
|
}
|
|
|
|
func (j *Job) acquireLock(ctx context.Context) (func(), bool) {
|
|
if j.redis == nil {
|
|
return func() {}, true
|
|
}
|
|
ok, err := j.redis.SetNX(ctx, cleanupLockKey, j.instanceID, 10*time.Minute).Result()
|
|
if err != nil {
|
|
j.logger.Warn("临时文件归属清理任务获取锁失败", zap.Error(err))
|
|
return func() {}, true
|
|
}
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
return func() {
|
|
releaseCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
script := redis.NewScript(`if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end`)
|
|
if err := script.Run(releaseCtx, j.redis, []string{cleanupLockKey}, j.instanceID).Err(); err != nil {
|
|
j.logger.Warn("临时文件归属清理任务释放锁失败", zap.Error(err))
|
|
}
|
|
}, true
|
|
}
|
|
|
|
func (j *Job) cleanup(ctx context.Context, now time.Time) (int, error) {
|
|
cutoff := now.Add(-cleanupRetention)
|
|
lastID := uint64(0)
|
|
deleted := 0
|
|
for {
|
|
var records []model.FileUploadOwner
|
|
if err := j.db.WithContext(ctx).
|
|
Where("id > ? AND created_at < ?", lastID, cutoff).
|
|
Order("id ASC").
|
|
Limit(cleanupBatchSize).
|
|
Find(&records).Error; err != nil {
|
|
return deleted, err
|
|
}
|
|
if len(records) == 0 {
|
|
return deleted, nil
|
|
}
|
|
for _, record := range records {
|
|
lastID = record.ID
|
|
referenced, err := j.isReferenced(ctx, record.ObjectKey)
|
|
if err != nil {
|
|
return deleted, err
|
|
}
|
|
if referenced {
|
|
continue
|
|
}
|
|
result := j.db.WithContext(ctx).
|
|
Where("id = ? AND created_at < ?", record.ID, cutoff).
|
|
Delete(&model.FileUploadOwner{})
|
|
if result.Error != nil {
|
|
return deleted, result.Error
|
|
}
|
|
deleted += int(result.RowsAffected)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (j *Job) isReferenced(ctx context.Context, key string) (bool, error) {
|
|
values := referenceValues(key)
|
|
payload, err := json.Marshal(values)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
queries := make([]struct {
|
|
sql string
|
|
args []any
|
|
}, 0, 2)
|
|
jsonArgs := []any{string(payload)}
|
|
switch strings.SplitN(key, "/", 2)[0] {
|
|
case "listing", "misc":
|
|
queries = append(queries, struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM game_accounts WHERE JSON_OVERLAPS(screenshot_urls, CAST(? AS JSON))", jsonArgs})
|
|
case "chat":
|
|
queries = append(queries, struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM chat_messages WHERE JSON_OVERLAPS(attachment_urls, CAST(? AS JSON))", jsonArgs})
|
|
case "qrcode", "support-qrcode":
|
|
clause, args := scalarIn(values)
|
|
queries = append(queries, struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM chat_qrcode_pool WHERE image_url IN (" + clause + ")", args})
|
|
case "handoff":
|
|
queries = append(queries, struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM handoff_records WHERE JSON_OVERLAPS(attachment_urls, CAST(? AS JSON))", jsonArgs})
|
|
case "dispute":
|
|
queries = append(queries,
|
|
struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM disputes WHERE JSON_OVERLAPS(evidence_urls, CAST(? AS JSON))", jsonArgs},
|
|
struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM order_checkouts WHERE JSON_OVERLAPS(evidence_urls, CAST(? AS JSON))", jsonArgs},
|
|
)
|
|
case "payment-cert":
|
|
queries = append(queries, struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM user_payment_accounts WHERE JSON_OVERLAPS(certificate_urls, CAST(? AS JSON))", jsonArgs})
|
|
case "payment-proof":
|
|
clause, args := scalarIn(values)
|
|
queries = append(queries, struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM withdrawal_requests WHERE payment_proof_url IN (" + clause + ")", args})
|
|
case "operating-expense":
|
|
clause, args := scalarIn(values)
|
|
queries = append(queries, struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM operating_expenses WHERE voucher_url IN (" + clause + ")", args})
|
|
case "manual-disbursement":
|
|
clause, args := scalarIn(values)
|
|
queries = append(queries, struct {
|
|
sql string
|
|
args []any
|
|
}{"SELECT COUNT(1) FROM manual_disbursements WHERE voucher_url IN (" + clause + ")", args})
|
|
default:
|
|
return false, nil
|
|
}
|
|
|
|
for _, query := range queries {
|
|
var count int64
|
|
if err := j.db.WithContext(ctx).Raw(query.sql, query.args...).Scan(&count).Error; err != nil {
|
|
return false, err
|
|
}
|
|
if count > 0 {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func referenceValues(key string) []string {
|
|
basePrefix := key
|
|
for _, suffix := range []string{".medium.jpg", ".thumb.jpg"} {
|
|
if strings.HasSuffix(basePrefix, suffix) {
|
|
basePrefix = strings.TrimSuffix(basePrefix, suffix)
|
|
break
|
|
}
|
|
}
|
|
|
|
candidates := []string{key}
|
|
if basePrefix != key {
|
|
candidates = append(candidates, basePrefix)
|
|
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
|
|
candidates = append(candidates, basePrefix+ext)
|
|
}
|
|
}
|
|
|
|
seen := make(map[string]struct{}, len(candidates)*8)
|
|
values := make([]string, 0, len(candidates)*8)
|
|
add := func(value string) {
|
|
if value == "" {
|
|
return
|
|
}
|
|
if _, ok := seen[value]; ok {
|
|
return
|
|
}
|
|
seen[value] = struct{}{}
|
|
values = append(values, value)
|
|
}
|
|
for _, candidate := range candidates {
|
|
encoded := url.QueryEscape(candidate)
|
|
add(candidate)
|
|
add(encoded)
|
|
for _, prefix := range []string{
|
|
"/api/files/object?key=",
|
|
"/api/admin/files/object?key=",
|
|
"/api/public/files/object?key=",
|
|
} {
|
|
add(prefix + encoded)
|
|
add(prefix + candidate)
|
|
}
|
|
}
|
|
return values
|
|
}
|
|
|
|
func scalarIn(values []string) (string, []any) {
|
|
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(values)), ",")
|
|
args := make([]any, 0, len(values))
|
|
for _, value := range values {
|
|
args = append(args, value)
|
|
}
|
|
return placeholders, args
|
|
}
|
|
|
|
func newInstanceID() string {
|
|
value := make([]byte, 8)
|
|
if _, err := rand.Read(value); err != nil {
|
|
return fmt.Sprintf("file-cleanup-%d", time.Now().UnixNano())
|
|
}
|
|
return hex.EncodeToString(value)
|
|
}
|