392 lines
12 KiB
Go
392 lines
12 KiB
Go
// file-object-bindings-backfill 回填私有文件对象与业务资源的绑定关系。
|
||
//
|
||
// 默认只预演。实际写入时按主键游标分批提交,可重复执行;指定 -checkpoint
|
||
// 后会记录每个来源已完成的最后主键,进程中断后可从断点继续。
|
||
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"flag"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"hfb_sys/backend/internal/config"
|
||
"hfb_sys/backend/internal/database"
|
||
"hfb_sys/backend/internal/filekey"
|
||
"hfb_sys/backend/internal/logging"
|
||
"hfb_sys/backend/internal/model"
|
||
|
||
"gorm.io/datatypes"
|
||
"gorm.io/gorm"
|
||
"gorm.io/gorm/clause"
|
||
)
|
||
|
||
type sourceSpec struct {
|
||
Name string
|
||
SQL string
|
||
}
|
||
|
||
type sourceRow struct {
|
||
ID uint64 `gorm:"column:id"`
|
||
TenantID uint64 `gorm:"column:tenant_id"`
|
||
OwnerUserID uint64 `gorm:"column:owner_user_id"`
|
||
ResourceID uint64 `gorm:"column:resource_id"`
|
||
ConversationID *uint64 `gorm:"column:conversation_id"`
|
||
Keys datatypes.JSON `gorm:"column:object_keys"`
|
||
ScalarKey string `gorm:"column:scalar_key"`
|
||
}
|
||
|
||
type sourceStats struct {
|
||
Rows int64 `json:"rows"`
|
||
References int64 `json:"references"`
|
||
InvalidKeys int64 `json:"invalid_keys"`
|
||
MalformedJSON int64 `json:"malformed_json"`
|
||
NoResource int64 `json:"no_resource"`
|
||
InsertedBindings int64 `json:"inserted_bindings"`
|
||
ExistingBindings int64 `json:"existing_bindings"`
|
||
}
|
||
|
||
type checkpointFile struct {
|
||
UpdatedAt string `json:"updated_at"`
|
||
Sources map[string]uint64 `json:"sources"`
|
||
}
|
||
|
||
var sources = []sourceSpec{
|
||
{
|
||
Name: "game_accounts",
|
||
SQL: `SELECT id, source_tenant_id AS tenant_id, owner_id AS owner_user_id,
|
||
id AS resource_id, NULL AS conversation_id, screenshot_urls AS object_keys,
|
||
'' AS scalar_key
|
||
FROM game_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "chat_messages",
|
||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, conversation_id AS resource_id,
|
||
NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key
|
||
FROM chat_messages WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "handoff_records",
|
||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
|
||
NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key
|
||
FROM handoff_records WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "disputes",
|
||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
|
||
NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key
|
||
FROM disputes WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "order_checkouts",
|
||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
|
||
NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key
|
||
FROM order_checkouts WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "user_payment_accounts",
|
||
SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id,
|
||
NULL AS conversation_id, certificate_urls AS object_keys, '' AS scalar_key
|
||
FROM user_payment_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "withdrawal_requests",
|
||
SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id,
|
||
NULL AS conversation_id, NULL AS object_keys, payment_proof_url AS scalar_key
|
||
FROM withdrawal_requests WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "chat_qrcode_pool",
|
||
SQL: `SELECT id, tenant_id, 0 AS owner_user_id, id AS resource_id,
|
||
conversation_id, NULL AS object_keys, image_url AS scalar_key
|
||
FROM chat_qrcode_pool WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "operating_expenses",
|
||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id,
|
||
NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key
|
||
FROM operating_expenses WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
{
|
||
Name: "manual_disbursements",
|
||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id,
|
||
NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key
|
||
FROM manual_disbursements WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||
},
|
||
}
|
||
|
||
func main() {
|
||
dryRun := flag.Bool("dry-run", true, "只统计不写库;实际回填时传 -dry-run=false")
|
||
batchSize := flag.Int("batch-size", 500, "每批主键记录数")
|
||
onlySource := flag.String("source", "", "只处理指定来源,例如 chat_messages")
|
||
afterID := flag.Uint64("after-id", 0, "所有来源从该主键之后开始,仅用于手动续跑")
|
||
checkpointPath := flag.String("checkpoint", "", "断点 JSON 文件路径;为空时不持久化断点")
|
||
flag.Parse()
|
||
|
||
if *batchSize < 1 || *batchSize > 5000 {
|
||
log.Fatal("-batch-size 必须在 1 到 5000 之间")
|
||
}
|
||
selected := selectSources(*onlySource)
|
||
if len(selected) == 0 {
|
||
log.Fatalf("未知来源 %q", *onlySource)
|
||
}
|
||
|
||
cfg := config.Load()
|
||
logger, err := logging.New(cfg.Log)
|
||
if err != nil {
|
||
log.Fatalf("初始化日志失败: %v", err)
|
||
}
|
||
defer func() {
|
||
_ = logger.Sync()
|
||
}()
|
||
db, err := database.OpenMySQLWithOptions(cfg.MySQLDSN, cfg.Log.Level, logger, database.MySQLOptions{})
|
||
if err != nil {
|
||
log.Fatalf("连接数据库失败: %v", err)
|
||
}
|
||
|
||
checkpoint := loadCheckpoint(*checkpointPath)
|
||
mode := "预演"
|
||
if !*dryRun {
|
||
mode = "实际回填"
|
||
}
|
||
fmt.Printf("模式=%s 批次=%d 来源=%d 断点=%s\n", mode, *batchSize, len(selected), emptyAsNone(*checkpointPath))
|
||
|
||
for _, source := range selected {
|
||
after := checkpoint.Sources[source.Name]
|
||
if *afterID > after {
|
||
after = *afterID
|
||
}
|
||
stats, lastID, err := runSource(db, source, after, *batchSize, *dryRun, func(next uint64) error {
|
||
if *dryRun || *checkpointPath == "" {
|
||
return nil
|
||
}
|
||
checkpoint.Sources[source.Name] = next
|
||
return saveCheckpoint(*checkpointPath, checkpoint)
|
||
})
|
||
if err != nil {
|
||
log.Fatalf("来源 %s 回填失败(最后主键 %d): %v", source.Name, lastID, err)
|
||
}
|
||
fmt.Printf(
|
||
"[%s] 扫描记录=%d 对象引用=%d 新增绑定=%d 已存在=%d 无效key=%d JSON解析失败=%d 无资源关联=%d 最后主键=%d\n",
|
||
source.Name,
|
||
stats.Rows,
|
||
stats.References,
|
||
stats.InsertedBindings,
|
||
stats.ExistingBindings,
|
||
stats.InvalidKeys,
|
||
stats.MalformedJSON,
|
||
stats.NoResource,
|
||
lastID,
|
||
)
|
||
}
|
||
}
|
||
|
||
func selectSources(name string) []sourceSpec {
|
||
name = strings.TrimSpace(name)
|
||
if name == "" {
|
||
return sources
|
||
}
|
||
for _, source := range sources {
|
||
if source.Name == name {
|
||
return []sourceSpec{source}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func runSource(
|
||
db *gorm.DB,
|
||
source sourceSpec,
|
||
afterID uint64,
|
||
batchSize int,
|
||
dryRun bool,
|
||
checkpoint func(uint64) error,
|
||
) (sourceStats, uint64, error) {
|
||
var stats sourceStats
|
||
cursor := afterID
|
||
for {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||
var rows []sourceRow
|
||
err := db.WithContext(ctx).Raw(source.SQL, cursor, batchSize).Scan(&rows).Error
|
||
cancel()
|
||
if err != nil {
|
||
return stats, cursor, err
|
||
}
|
||
if len(rows) == 0 {
|
||
return stats, cursor, nil
|
||
}
|
||
|
||
bindings := make([]model.FileObjectBinding, 0, len(rows)*2)
|
||
seen := make(map[string]struct{})
|
||
for _, row := range rows {
|
||
stats.Rows++
|
||
cursor = row.ID
|
||
resourceType, resourceID, scope, ownerUserID, tenantID := bindingTarget(source.Name, row)
|
||
keys, malformed, invalid := canonicalKeys(row)
|
||
stats.MalformedJSON += int64(malformed)
|
||
stats.InvalidKeys += int64(invalid)
|
||
if resourceID == 0 {
|
||
if len(keys) > 0 || malformed > 0 || invalid > 0 {
|
||
stats.NoResource++
|
||
}
|
||
continue
|
||
}
|
||
for _, key := range keys {
|
||
stats.References++
|
||
identity := key + "\x00" + resourceType + "\x00" + scope
|
||
if _, exists := seen[identity]; exists {
|
||
stats.ExistingBindings++
|
||
continue
|
||
}
|
||
seen[identity] = struct{}{}
|
||
bindings = append(bindings, model.FileObjectBinding{
|
||
TenantID: tenantID,
|
||
CanonicalKey: key,
|
||
ResourceType: resourceType,
|
||
ResourceID: resourceID,
|
||
AccessScope: scope,
|
||
OwnerUserID: ownerUserID,
|
||
})
|
||
}
|
||
}
|
||
|
||
if !dryRun && len(bindings) > 0 {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||
result := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&bindings)
|
||
if result.Error != nil {
|
||
return result.Error
|
||
}
|
||
stats.InsertedBindings += result.RowsAffected
|
||
stats.ExistingBindings += int64(len(bindings)) - result.RowsAffected
|
||
return nil
|
||
})
|
||
cancel()
|
||
if err != nil {
|
||
return stats, cursor, err
|
||
}
|
||
} else {
|
||
stats.InsertedBindings += int64(len(bindings))
|
||
}
|
||
if err := checkpoint(cursor); err != nil {
|
||
return stats, cursor, err
|
||
}
|
||
if len(rows) < batchSize {
|
||
return stats, cursor, nil
|
||
}
|
||
}
|
||
}
|
||
|
||
func bindingTarget(source string, row sourceRow) (resourceType string, resourceID uint64, scope string, ownerUserID, tenantID uint64) {
|
||
switch source {
|
||
case "game_accounts":
|
||
return model.FileResourceGameAccount, row.ResourceID, model.FileScopeListingAsset, row.OwnerUserID, row.TenantID
|
||
case "chat_messages":
|
||
return model.FileResourceChatConversation, row.ResourceID, model.FileScopeChatParticipant, 0, 0
|
||
case "handoff_records", "disputes", "order_checkouts":
|
||
return model.FileResourceOrder, row.ResourceID, model.FileScopeOrderParticipant, 0, 0
|
||
case "user_payment_accounts":
|
||
return model.FileResourcePaymentAccount, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID
|
||
case "withdrawal_requests":
|
||
return model.FileResourceWithdrawal, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID
|
||
case "chat_qrcode_pool":
|
||
if row.ConversationID != nil && *row.ConversationID > 0 {
|
||
return model.FileResourceChatConversation, *row.ConversationID, model.FileScopeQrCodeParticipant, 0, row.TenantID
|
||
}
|
||
return model.FileResourceChatQrCode, row.ResourceID, model.FileScopeQrCodeTenant, 0, row.TenantID
|
||
case "operating_expenses":
|
||
return model.FileResourceOperatingExpense, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0
|
||
case "manual_disbursements":
|
||
return model.FileResourceManualDisbursement, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0
|
||
default:
|
||
return "", 0, "", 0, 0
|
||
}
|
||
}
|
||
|
||
func canonicalKeys(row sourceRow) ([]string, int, int) {
|
||
keys := make([]string, 0, 2)
|
||
malformed := 0
|
||
invalid := 0
|
||
if len(row.Keys) > 0 {
|
||
var rawKeys []string
|
||
if err := json.Unmarshal(row.Keys, &rawKeys); err != nil {
|
||
malformed++
|
||
} else {
|
||
keys = append(keys, rawKeys...)
|
||
}
|
||
}
|
||
if strings.TrimSpace(row.ScalarKey) != "" {
|
||
keys = append(keys, row.ScalarKey)
|
||
}
|
||
|
||
seen := make(map[string]struct{}, len(keys))
|
||
canonical := make([]string, 0, len(keys))
|
||
for _, raw := range keys {
|
||
key := filekey.Canonical(raw)
|
||
if key == "" {
|
||
invalid++
|
||
continue
|
||
}
|
||
if _, exists := seen[key]; exists {
|
||
continue
|
||
}
|
||
seen[key] = struct{}{}
|
||
canonical = append(canonical, key)
|
||
}
|
||
return canonical, malformed, invalid
|
||
}
|
||
|
||
func loadCheckpoint(path string) checkpointFile {
|
||
checkpoint := checkpointFile{Sources: make(map[string]uint64)}
|
||
if strings.TrimSpace(path) == "" {
|
||
return checkpoint
|
||
}
|
||
payload, err := os.ReadFile(path)
|
||
if err != nil {
|
||
return checkpoint
|
||
}
|
||
if err := json.Unmarshal(payload, &checkpoint); err != nil || checkpoint.Sources == nil {
|
||
checkpoint.Sources = make(map[string]uint64)
|
||
}
|
||
return checkpoint
|
||
}
|
||
|
||
func saveCheckpoint(path string, checkpoint checkpointFile) error {
|
||
checkpoint.UpdatedAt = time.Now().Format(time.RFC3339)
|
||
payload, err := json.MarshalIndent(checkpoint, "", " ")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
dir := filepath.Dir(path)
|
||
if dir != "." {
|
||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
temp := path + ".tmp"
|
||
if err := os.WriteFile(temp, payload, 0o600); err != nil {
|
||
return err
|
||
}
|
||
return os.Rename(temp, path)
|
||
}
|
||
|
||
func emptyAsNone(value string) string {
|
||
if strings.TrimSpace(value) == "" {
|
||
return "未启用"
|
||
}
|
||
return value
|
||
}
|
||
|
||
func init() {
|
||
// 保证来源顺序稳定,便于运维日志对比。
|
||
sort.SliceStable(sources, func(i, j int) bool {
|
||
return sources[i].Name < sources[j].Name
|
||
})
|
||
}
|