Files
hfb_sys/backend/cmd/file-object-bindings-backfill/main.go
T

392 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// file-object-bindings-backfill 回填私有文件对象与业务资源的绑定关系。
//
// 默认只预演。实际写入时按主键游标分批提交,可重复执行;指定 -checkpoint
// 后会记录每个来源已完成的最后主键,进程中断后可从断点继续。
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strings"
"time"
"hfb_sys/backend/internal/config"
"hfb_sys/backend/internal/database"
"hfb_sys/backend/internal/filekey"
"hfb_sys/backend/internal/logging"
"hfb_sys/backend/internal/model"
"gorm.io/datatypes"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type sourceSpec struct {
Name string
SQL string
}
type sourceRow struct {
ID uint64 `gorm:"column:id"`
TenantID uint64 `gorm:"column:tenant_id"`
OwnerUserID uint64 `gorm:"column:owner_user_id"`
ResourceID uint64 `gorm:"column:resource_id"`
ConversationID *uint64 `gorm:"column:conversation_id"`
Keys datatypes.JSON `gorm:"column:object_keys"`
ScalarKey string `gorm:"column:scalar_key"`
}
type sourceStats struct {
Rows int64 `json:"rows"`
References int64 `json:"references"`
InvalidKeys int64 `json:"invalid_keys"`
MalformedJSON int64 `json:"malformed_json"`
NoResource int64 `json:"no_resource"`
InsertedBindings int64 `json:"inserted_bindings"`
ExistingBindings int64 `json:"existing_bindings"`
}
type checkpointFile struct {
UpdatedAt string `json:"updated_at"`
Sources map[string]uint64 `json:"sources"`
}
var sources = []sourceSpec{
{
Name: "game_accounts",
SQL: `SELECT id, source_tenant_id AS tenant_id, owner_id AS owner_user_id,
id AS resource_id, NULL AS conversation_id, screenshot_urls AS object_keys,
'' AS scalar_key
FROM game_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "chat_messages",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, conversation_id AS resource_id,
NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key
FROM chat_messages WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "handoff_records",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key
FROM handoff_records WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "disputes",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key
FROM disputes WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "order_checkouts",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key
FROM order_checkouts WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "user_payment_accounts",
SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id,
NULL AS conversation_id, certificate_urls AS object_keys, '' AS scalar_key
FROM user_payment_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "withdrawal_requests",
SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id,
NULL AS conversation_id, NULL AS object_keys, payment_proof_url AS scalar_key
FROM withdrawal_requests WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "chat_qrcode_pool",
SQL: `SELECT id, tenant_id, 0 AS owner_user_id, id AS resource_id,
conversation_id, NULL AS object_keys, image_url AS scalar_key
FROM chat_qrcode_pool WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "operating_expenses",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id,
NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key
FROM operating_expenses WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "manual_disbursements",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id,
NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key
FROM manual_disbursements WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
}
func main() {
dryRun := flag.Bool("dry-run", true, "只统计不写库;实际回填时传 -dry-run=false")
batchSize := flag.Int("batch-size", 500, "每批主键记录数")
onlySource := flag.String("source", "", "只处理指定来源,例如 chat_messages")
afterID := flag.Uint64("after-id", 0, "所有来源从该主键之后开始,仅用于手动续跑")
checkpointPath := flag.String("checkpoint", "", "断点 JSON 文件路径;为空时不持久化断点")
flag.Parse()
if *batchSize < 1 || *batchSize > 5000 {
log.Fatal("-batch-size 必须在 1 到 5000 之间")
}
selected := selectSources(*onlySource)
if len(selected) == 0 {
log.Fatalf("未知来源 %q", *onlySource)
}
cfg := config.Load()
logger, err := logging.New(cfg.Log)
if err != nil {
log.Fatalf("初始化日志失败: %v", err)
}
defer func() {
_ = logger.Sync()
}()
db, err := database.OpenMySQLWithOptions(cfg.MySQLDSN, cfg.Log.Level, logger, database.MySQLOptions{})
if err != nil {
log.Fatalf("连接数据库失败: %v", err)
}
checkpoint := loadCheckpoint(*checkpointPath)
mode := "预演"
if !*dryRun {
mode = "实际回填"
}
fmt.Printf("模式=%s 批次=%d 来源=%d 断点=%s\n", mode, *batchSize, len(selected), emptyAsNone(*checkpointPath))
for _, source := range selected {
after := checkpoint.Sources[source.Name]
if *afterID > after {
after = *afterID
}
stats, lastID, err := runSource(db, source, after, *batchSize, *dryRun, func(next uint64) error {
if *dryRun || *checkpointPath == "" {
return nil
}
checkpoint.Sources[source.Name] = next
return saveCheckpoint(*checkpointPath, checkpoint)
})
if err != nil {
log.Fatalf("来源 %s 回填失败(最后主键 %d): %v", source.Name, lastID, err)
}
fmt.Printf(
"[%s] 扫描记录=%d 对象引用=%d 新增绑定=%d 已存在=%d 无效key=%d JSON解析失败=%d 无资源关联=%d 最后主键=%d\n",
source.Name,
stats.Rows,
stats.References,
stats.InsertedBindings,
stats.ExistingBindings,
stats.InvalidKeys,
stats.MalformedJSON,
stats.NoResource,
lastID,
)
}
}
func selectSources(name string) []sourceSpec {
name = strings.TrimSpace(name)
if name == "" {
return sources
}
for _, source := range sources {
if source.Name == name {
return []sourceSpec{source}
}
}
return nil
}
func runSource(
db *gorm.DB,
source sourceSpec,
afterID uint64,
batchSize int,
dryRun bool,
checkpoint func(uint64) error,
) (sourceStats, uint64, error) {
var stats sourceStats
cursor := afterID
for {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
var rows []sourceRow
err := db.WithContext(ctx).Raw(source.SQL, cursor, batchSize).Scan(&rows).Error
cancel()
if err != nil {
return stats, cursor, err
}
if len(rows) == 0 {
return stats, cursor, nil
}
bindings := make([]model.FileObjectBinding, 0, len(rows)*2)
seen := make(map[string]struct{})
for _, row := range rows {
stats.Rows++
cursor = row.ID
resourceType, resourceID, scope, ownerUserID, tenantID := bindingTarget(source.Name, row)
keys, malformed, invalid := canonicalKeys(row)
stats.MalformedJSON += int64(malformed)
stats.InvalidKeys += int64(invalid)
if resourceID == 0 {
if len(keys) > 0 || malformed > 0 || invalid > 0 {
stats.NoResource++
}
continue
}
for _, key := range keys {
stats.References++
identity := key + "\x00" + resourceType + "\x00" + scope
if _, exists := seen[identity]; exists {
stats.ExistingBindings++
continue
}
seen[identity] = struct{}{}
bindings = append(bindings, model.FileObjectBinding{
TenantID: tenantID,
CanonicalKey: key,
ResourceType: resourceType,
ResourceID: resourceID,
AccessScope: scope,
OwnerUserID: ownerUserID,
})
}
}
if !dryRun && len(bindings) > 0 {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
result := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&bindings)
if result.Error != nil {
return result.Error
}
stats.InsertedBindings += result.RowsAffected
stats.ExistingBindings += int64(len(bindings)) - result.RowsAffected
return nil
})
cancel()
if err != nil {
return stats, cursor, err
}
} else {
stats.InsertedBindings += int64(len(bindings))
}
if err := checkpoint(cursor); err != nil {
return stats, cursor, err
}
if len(rows) < batchSize {
return stats, cursor, nil
}
}
}
func bindingTarget(source string, row sourceRow) (resourceType string, resourceID uint64, scope string, ownerUserID, tenantID uint64) {
switch source {
case "game_accounts":
return model.FileResourceGameAccount, row.ResourceID, model.FileScopeListingAsset, row.OwnerUserID, row.TenantID
case "chat_messages":
return model.FileResourceChatConversation, row.ResourceID, model.FileScopeChatParticipant, 0, 0
case "handoff_records", "disputes", "order_checkouts":
return model.FileResourceOrder, row.ResourceID, model.FileScopeOrderParticipant, 0, 0
case "user_payment_accounts":
return model.FileResourcePaymentAccount, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID
case "withdrawal_requests":
return model.FileResourceWithdrawal, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID
case "chat_qrcode_pool":
if row.ConversationID != nil && *row.ConversationID > 0 {
return model.FileResourceChatConversation, *row.ConversationID, model.FileScopeQrCodeParticipant, 0, row.TenantID
}
return model.FileResourceChatQrCode, row.ResourceID, model.FileScopeQrCodeTenant, 0, row.TenantID
case "operating_expenses":
return model.FileResourceOperatingExpense, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0
case "manual_disbursements":
return model.FileResourceManualDisbursement, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0
default:
return "", 0, "", 0, 0
}
}
func canonicalKeys(row sourceRow) ([]string, int, int) {
keys := make([]string, 0, 2)
malformed := 0
invalid := 0
if len(row.Keys) > 0 {
var rawKeys []string
if err := json.Unmarshal(row.Keys, &rawKeys); err != nil {
malformed++
} else {
keys = append(keys, rawKeys...)
}
}
if strings.TrimSpace(row.ScalarKey) != "" {
keys = append(keys, row.ScalarKey)
}
seen := make(map[string]struct{}, len(keys))
canonical := make([]string, 0, len(keys))
for _, raw := range keys {
key := filekey.Canonical(raw)
if key == "" {
invalid++
continue
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
canonical = append(canonical, key)
}
return canonical, malformed, invalid
}
func loadCheckpoint(path string) checkpointFile {
checkpoint := checkpointFile{Sources: make(map[string]uint64)}
if strings.TrimSpace(path) == "" {
return checkpoint
}
payload, err := os.ReadFile(path)
if err != nil {
return checkpoint
}
if err := json.Unmarshal(payload, &checkpoint); err != nil || checkpoint.Sources == nil {
checkpoint.Sources = make(map[string]uint64)
}
return checkpoint
}
func saveCheckpoint(path string, checkpoint checkpointFile) error {
checkpoint.UpdatedAt = time.Now().Format(time.RFC3339)
payload, err := json.MarshalIndent(checkpoint, "", " ")
if err != nil {
return err
}
dir := filepath.Dir(path)
if dir != "." {
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
}
temp := path + ".tmp"
if err := os.WriteFile(temp, payload, 0o600); err != nil {
return err
}
return os.Rename(temp, path)
}
func emptyAsNone(value string) string {
if strings.TrimSpace(value) == "" {
return "未启用"
}
return value
}
func init() {
// 保证来源顺序稳定,便于运维日志对比。
sort.SliceStable(sources, func(i, j int) bool {
return sources[i].Name < sources[j].Name
})
}