// file-object-bindings-backfill 回填私有文件对象与业务资源的绑定关系。 // // 默认只预演。实际写入时按主键游标分批提交,可重复执行;指定 -checkpoint // 后会记录每个来源已完成的最后主键,进程中断后可从断点继续。 package main import ( "context" "encoding/json" "flag" "fmt" "log" "os" "path/filepath" "sort" "strings" "time" "hfb_sys/backend/internal/config" "hfb_sys/backend/internal/database" "hfb_sys/backend/internal/filekey" "hfb_sys/backend/internal/logging" "hfb_sys/backend/internal/model" "gorm.io/datatypes" "gorm.io/gorm" "gorm.io/gorm/clause" ) type sourceSpec struct { Name string SQL string } type sourceRow struct { ID uint64 `gorm:"column:id"` TenantID uint64 `gorm:"column:tenant_id"` OwnerUserID uint64 `gorm:"column:owner_user_id"` ResourceID uint64 `gorm:"column:resource_id"` ConversationID *uint64 `gorm:"column:conversation_id"` Keys datatypes.JSON `gorm:"column:object_keys"` ScalarKey string `gorm:"column:scalar_key"` } type sourceStats struct { Rows int64 `json:"rows"` References int64 `json:"references"` InvalidKeys int64 `json:"invalid_keys"` MalformedJSON int64 `json:"malformed_json"` NoResource int64 `json:"no_resource"` InsertedBindings int64 `json:"inserted_bindings"` ExistingBindings int64 `json:"existing_bindings"` } type checkpointFile struct { UpdatedAt string `json:"updated_at"` Sources map[string]uint64 `json:"sources"` } var sources = []sourceSpec{ { Name: "game_accounts", SQL: `SELECT id, source_tenant_id AS tenant_id, owner_id AS owner_user_id, id AS resource_id, NULL AS conversation_id, screenshot_urls AS object_keys, '' AS scalar_key FROM game_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "chat_messages", SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, conversation_id AS resource_id, NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key FROM chat_messages WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "handoff_records", SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id, NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key FROM handoff_records WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "disputes", SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id, NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key FROM disputes WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "order_checkouts", SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id, NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key FROM order_checkouts WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "user_payment_accounts", SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id, NULL AS conversation_id, certificate_urls AS object_keys, '' AS scalar_key FROM user_payment_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "withdrawal_requests", SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id, NULL AS conversation_id, NULL AS object_keys, payment_proof_url AS scalar_key FROM withdrawal_requests WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "chat_qrcode_pool", SQL: `SELECT id, tenant_id, 0 AS owner_user_id, id AS resource_id, conversation_id, NULL AS object_keys, image_url AS scalar_key FROM chat_qrcode_pool WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "operating_expenses", SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id, NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key FROM operating_expenses WHERE id > ? ORDER BY id ASC LIMIT ?`, }, { Name: "manual_disbursements", SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id, NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key FROM manual_disbursements WHERE id > ? ORDER BY id ASC LIMIT ?`, }, } func main() { dryRun := flag.Bool("dry-run", true, "只统计不写库;实际回填时传 -dry-run=false") batchSize := flag.Int("batch-size", 500, "每批主键记录数") onlySource := flag.String("source", "", "只处理指定来源,例如 chat_messages") afterID := flag.Uint64("after-id", 0, "所有来源从该主键之后开始,仅用于手动续跑") checkpointPath := flag.String("checkpoint", "", "断点 JSON 文件路径;为空时不持久化断点") flag.Parse() if *batchSize < 1 || *batchSize > 5000 { log.Fatal("-batch-size 必须在 1 到 5000 之间") } selected := selectSources(*onlySource) if len(selected) == 0 { log.Fatalf("未知来源 %q", *onlySource) } cfg := config.Load() logger, err := logging.New(cfg.Log) if err != nil { log.Fatalf("初始化日志失败: %v", err) } defer func() { _ = logger.Sync() }() db, err := database.OpenMySQLWithOptions(cfg.MySQLDSN, cfg.Log.Level, logger, database.MySQLOptions{}) if err != nil { log.Fatalf("连接数据库失败: %v", err) } checkpoint := loadCheckpoint(*checkpointPath) mode := "预演" if !*dryRun { mode = "实际回填" } fmt.Printf("模式=%s 批次=%d 来源=%d 断点=%s\n", mode, *batchSize, len(selected), emptyAsNone(*checkpointPath)) for _, source := range selected { after := checkpoint.Sources[source.Name] if *afterID > after { after = *afterID } stats, lastID, err := runSource(db, source, after, *batchSize, *dryRun, func(next uint64) error { if *dryRun || *checkpointPath == "" { return nil } checkpoint.Sources[source.Name] = next return saveCheckpoint(*checkpointPath, checkpoint) }) if err != nil { log.Fatalf("来源 %s 回填失败(最后主键 %d): %v", source.Name, lastID, err) } fmt.Printf( "[%s] 扫描记录=%d 对象引用=%d 新增绑定=%d 已存在=%d 无效key=%d JSON解析失败=%d 无资源关联=%d 最后主键=%d\n", source.Name, stats.Rows, stats.References, stats.InsertedBindings, stats.ExistingBindings, stats.InvalidKeys, stats.MalformedJSON, stats.NoResource, lastID, ) } } func selectSources(name string) []sourceSpec { name = strings.TrimSpace(name) if name == "" { return sources } for _, source := range sources { if source.Name == name { return []sourceSpec{source} } } return nil } func runSource( db *gorm.DB, source sourceSpec, afterID uint64, batchSize int, dryRun bool, checkpoint func(uint64) error, ) (sourceStats, uint64, error) { var stats sourceStats cursor := afterID for { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) var rows []sourceRow err := db.WithContext(ctx).Raw(source.SQL, cursor, batchSize).Scan(&rows).Error cancel() if err != nil { return stats, cursor, err } if len(rows) == 0 { return stats, cursor, nil } bindings := make([]model.FileObjectBinding, 0, len(rows)*2) seen := make(map[string]struct{}) for _, row := range rows { stats.Rows++ cursor = row.ID resourceType, resourceID, scope, ownerUserID, tenantID := bindingTarget(source.Name, row) keys, malformed, invalid := canonicalKeys(row) stats.MalformedJSON += int64(malformed) stats.InvalidKeys += int64(invalid) if resourceID == 0 { if len(keys) > 0 || malformed > 0 || invalid > 0 { stats.NoResource++ } continue } for _, key := range keys { stats.References++ identity := key + "\x00" + resourceType + "\x00" + scope if _, exists := seen[identity]; exists { stats.ExistingBindings++ continue } seen[identity] = struct{}{} bindings = append(bindings, model.FileObjectBinding{ TenantID: tenantID, CanonicalKey: key, ResourceType: resourceType, ResourceID: resourceID, AccessScope: scope, OwnerUserID: ownerUserID, }) } } if !dryRun && len(bindings) > 0 { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { result := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&bindings) if result.Error != nil { return result.Error } stats.InsertedBindings += result.RowsAffected stats.ExistingBindings += int64(len(bindings)) - result.RowsAffected return nil }) cancel() if err != nil { return stats, cursor, err } } else { stats.InsertedBindings += int64(len(bindings)) } if err := checkpoint(cursor); err != nil { return stats, cursor, err } if len(rows) < batchSize { return stats, cursor, nil } } } func bindingTarget(source string, row sourceRow) (resourceType string, resourceID uint64, scope string, ownerUserID, tenantID uint64) { switch source { case "game_accounts": return model.FileResourceGameAccount, row.ResourceID, model.FileScopeListingAsset, row.OwnerUserID, row.TenantID case "chat_messages": return model.FileResourceChatConversation, row.ResourceID, model.FileScopeChatParticipant, 0, 0 case "handoff_records", "disputes", "order_checkouts": return model.FileResourceOrder, row.ResourceID, model.FileScopeOrderParticipant, 0, 0 case "user_payment_accounts": return model.FileResourcePaymentAccount, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID case "withdrawal_requests": return model.FileResourceWithdrawal, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID case "chat_qrcode_pool": if row.ConversationID != nil && *row.ConversationID > 0 { return model.FileResourceChatConversation, *row.ConversationID, model.FileScopeQrCodeParticipant, 0, row.TenantID } return model.FileResourceChatQrCode, row.ResourceID, model.FileScopeQrCodeTenant, 0, row.TenantID case "operating_expenses": return model.FileResourceOperatingExpense, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0 case "manual_disbursements": return model.FileResourceManualDisbursement, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0 default: return "", 0, "", 0, 0 } } func canonicalKeys(row sourceRow) ([]string, int, int) { keys := make([]string, 0, 2) malformed := 0 invalid := 0 if len(row.Keys) > 0 { var rawKeys []string if err := json.Unmarshal(row.Keys, &rawKeys); err != nil { malformed++ } else { keys = append(keys, rawKeys...) } } if strings.TrimSpace(row.ScalarKey) != "" { keys = append(keys, row.ScalarKey) } seen := make(map[string]struct{}, len(keys)) canonical := make([]string, 0, len(keys)) for _, raw := range keys { key := filekey.Canonical(raw) if key == "" { invalid++ continue } if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} canonical = append(canonical, key) } return canonical, malformed, invalid } func loadCheckpoint(path string) checkpointFile { checkpoint := checkpointFile{Sources: make(map[string]uint64)} if strings.TrimSpace(path) == "" { return checkpoint } payload, err := os.ReadFile(path) if err != nil { return checkpoint } if err := json.Unmarshal(payload, &checkpoint); err != nil || checkpoint.Sources == nil { checkpoint.Sources = make(map[string]uint64) } return checkpoint } func saveCheckpoint(path string, checkpoint checkpointFile) error { checkpoint.UpdatedAt = time.Now().Format(time.RFC3339) payload, err := json.MarshalIndent(checkpoint, "", " ") if err != nil { return err } dir := filepath.Dir(path) if dir != "." { if err := os.MkdirAll(dir, 0o755); err != nil { return err } } temp := path + ".tmp" if err := os.WriteFile(temp, payload, 0o600); err != nil { return err } return os.Rename(temp, path) } func emptyAsNone(value string) string { if strings.TrimSpace(value) == "" { return "未启用" } return value } func init() { // 保证来源顺序稳定,便于运维日志对比。 sort.SliceStable(sources, func(i, j int) bool { return sources[i].Name < sources[j].Name }) }