perf(files): 优化私有文件鉴权与高成本查询隔离

This commit is contained in:
yml
2026-11-20 20:12:00 +08:00
parent e01a01c1aa
commit 9d542af51f
26 changed files with 2068 additions and 42 deletions
+1
View File
@@ -110,6 +110,7 @@ func main() {
jobCtx, stopJobs := context.WithCancel(context.Background())
defer stopJobs()
if deps.DB != nil {
database.StartPoolStatsLogger(jobCtx, deps.DB, logger, 30*time.Second)
var paymentRepo *payment.Repository
if paymentConfigRepo := newPaymentConfigRepositoryForJobs(cfg, deps.DB, logger); paymentConfigRepo != nil {
// 超时取消前的渠道核对要把收款推进到订单上,所以这里必须带真实的订单仓库,
@@ -0,0 +1,391 @@
// file-object-bindings-backfill 回填私有文件对象与业务资源的绑定关系。
//
// 默认只预演。实际写入时按主键游标分批提交,可重复执行;指定 -checkpoint
// 后会记录每个来源已完成的最后主键,进程中断后可从断点继续。
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strings"
"time"
"hfb_sys/backend/internal/config"
"hfb_sys/backend/internal/database"
"hfb_sys/backend/internal/filekey"
"hfb_sys/backend/internal/logging"
"hfb_sys/backend/internal/model"
"gorm.io/datatypes"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type sourceSpec struct {
Name string
SQL string
}
type sourceRow struct {
ID uint64 `gorm:"column:id"`
TenantID uint64 `gorm:"column:tenant_id"`
OwnerUserID uint64 `gorm:"column:owner_user_id"`
ResourceID uint64 `gorm:"column:resource_id"`
ConversationID *uint64 `gorm:"column:conversation_id"`
Keys datatypes.JSON `gorm:"column:object_keys"`
ScalarKey string `gorm:"column:scalar_key"`
}
type sourceStats struct {
Rows int64 `json:"rows"`
References int64 `json:"references"`
InvalidKeys int64 `json:"invalid_keys"`
MalformedJSON int64 `json:"malformed_json"`
NoResource int64 `json:"no_resource"`
InsertedBindings int64 `json:"inserted_bindings"`
ExistingBindings int64 `json:"existing_bindings"`
}
type checkpointFile struct {
UpdatedAt string `json:"updated_at"`
Sources map[string]uint64 `json:"sources"`
}
var sources = []sourceSpec{
{
Name: "game_accounts",
SQL: `SELECT id, source_tenant_id AS tenant_id, owner_id AS owner_user_id,
id AS resource_id, NULL AS conversation_id, screenshot_urls AS object_keys,
'' AS scalar_key
FROM game_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "chat_messages",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, conversation_id AS resource_id,
NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key
FROM chat_messages WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "handoff_records",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key
FROM handoff_records WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "disputes",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key
FROM disputes WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "order_checkouts",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key
FROM order_checkouts WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "user_payment_accounts",
SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id,
NULL AS conversation_id, certificate_urls AS object_keys, '' AS scalar_key
FROM user_payment_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "withdrawal_requests",
SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id,
NULL AS conversation_id, NULL AS object_keys, payment_proof_url AS scalar_key
FROM withdrawal_requests WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "chat_qrcode_pool",
SQL: `SELECT id, tenant_id, 0 AS owner_user_id, id AS resource_id,
conversation_id, NULL AS object_keys, image_url AS scalar_key
FROM chat_qrcode_pool WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "operating_expenses",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id,
NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key
FROM operating_expenses WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
{
Name: "manual_disbursements",
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id,
NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key
FROM manual_disbursements WHERE id > ? ORDER BY id ASC LIMIT ?`,
},
}
func main() {
dryRun := flag.Bool("dry-run", true, "只统计不写库;实际回填时传 -dry-run=false")
batchSize := flag.Int("batch-size", 500, "每批主键记录数")
onlySource := flag.String("source", "", "只处理指定来源,例如 chat_messages")
afterID := flag.Uint64("after-id", 0, "所有来源从该主键之后开始,仅用于手动续跑")
checkpointPath := flag.String("checkpoint", "", "断点 JSON 文件路径;为空时不持久化断点")
flag.Parse()
if *batchSize < 1 || *batchSize > 5000 {
log.Fatal("-batch-size 必须在 1 到 5000 之间")
}
selected := selectSources(*onlySource)
if len(selected) == 0 {
log.Fatalf("未知来源 %q", *onlySource)
}
cfg := config.Load()
logger, err := logging.New(cfg.Log)
if err != nil {
log.Fatalf("初始化日志失败: %v", err)
}
defer func() {
_ = logger.Sync()
}()
db, err := database.OpenMySQLWithOptions(cfg.MySQLDSN, cfg.Log.Level, logger, database.MySQLOptions{})
if err != nil {
log.Fatalf("连接数据库失败: %v", err)
}
checkpoint := loadCheckpoint(*checkpointPath)
mode := "预演"
if !*dryRun {
mode = "实际回填"
}
fmt.Printf("模式=%s 批次=%d 来源=%d 断点=%s\n", mode, *batchSize, len(selected), emptyAsNone(*checkpointPath))
for _, source := range selected {
after := checkpoint.Sources[source.Name]
if *afterID > after {
after = *afterID
}
stats, lastID, err := runSource(db, source, after, *batchSize, *dryRun, func(next uint64) error {
if *dryRun || *checkpointPath == "" {
return nil
}
checkpoint.Sources[source.Name] = next
return saveCheckpoint(*checkpointPath, checkpoint)
})
if err != nil {
log.Fatalf("来源 %s 回填失败(最后主键 %d): %v", source.Name, lastID, err)
}
fmt.Printf(
"[%s] 扫描记录=%d 对象引用=%d 新增绑定=%d 已存在=%d 无效key=%d JSON解析失败=%d 无资源关联=%d 最后主键=%d\n",
source.Name,
stats.Rows,
stats.References,
stats.InsertedBindings,
stats.ExistingBindings,
stats.InvalidKeys,
stats.MalformedJSON,
stats.NoResource,
lastID,
)
}
}
func selectSources(name string) []sourceSpec {
name = strings.TrimSpace(name)
if name == "" {
return sources
}
for _, source := range sources {
if source.Name == name {
return []sourceSpec{source}
}
}
return nil
}
func runSource(
db *gorm.DB,
source sourceSpec,
afterID uint64,
batchSize int,
dryRun bool,
checkpoint func(uint64) error,
) (sourceStats, uint64, error) {
var stats sourceStats
cursor := afterID
for {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
var rows []sourceRow
err := db.WithContext(ctx).Raw(source.SQL, cursor, batchSize).Scan(&rows).Error
cancel()
if err != nil {
return stats, cursor, err
}
if len(rows) == 0 {
return stats, cursor, nil
}
bindings := make([]model.FileObjectBinding, 0, len(rows)*2)
seen := make(map[string]struct{})
for _, row := range rows {
stats.Rows++
cursor = row.ID
resourceType, resourceID, scope, ownerUserID, tenantID := bindingTarget(source.Name, row)
keys, malformed, invalid := canonicalKeys(row)
stats.MalformedJSON += int64(malformed)
stats.InvalidKeys += int64(invalid)
if resourceID == 0 {
if len(keys) > 0 || malformed > 0 || invalid > 0 {
stats.NoResource++
}
continue
}
for _, key := range keys {
stats.References++
identity := key + "\x00" + resourceType + "\x00" + scope
if _, exists := seen[identity]; exists {
stats.ExistingBindings++
continue
}
seen[identity] = struct{}{}
bindings = append(bindings, model.FileObjectBinding{
TenantID: tenantID,
CanonicalKey: key,
ResourceType: resourceType,
ResourceID: resourceID,
AccessScope: scope,
OwnerUserID: ownerUserID,
})
}
}
if !dryRun && len(bindings) > 0 {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
result := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&bindings)
if result.Error != nil {
return result.Error
}
stats.InsertedBindings += result.RowsAffected
stats.ExistingBindings += int64(len(bindings)) - result.RowsAffected
return nil
})
cancel()
if err != nil {
return stats, cursor, err
}
} else {
stats.InsertedBindings += int64(len(bindings))
}
if err := checkpoint(cursor); err != nil {
return stats, cursor, err
}
if len(rows) < batchSize {
return stats, cursor, nil
}
}
}
func bindingTarget(source string, row sourceRow) (resourceType string, resourceID uint64, scope string, ownerUserID, tenantID uint64) {
switch source {
case "game_accounts":
return model.FileResourceGameAccount, row.ResourceID, model.FileScopeListingAsset, row.OwnerUserID, row.TenantID
case "chat_messages":
return model.FileResourceChatConversation, row.ResourceID, model.FileScopeChatParticipant, 0, 0
case "handoff_records", "disputes", "order_checkouts":
return model.FileResourceOrder, row.ResourceID, model.FileScopeOrderParticipant, 0, 0
case "user_payment_accounts":
return model.FileResourcePaymentAccount, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID
case "withdrawal_requests":
return model.FileResourceWithdrawal, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID
case "chat_qrcode_pool":
if row.ConversationID != nil && *row.ConversationID > 0 {
return model.FileResourceChatConversation, *row.ConversationID, model.FileScopeQrCodeParticipant, 0, row.TenantID
}
return model.FileResourceChatQrCode, row.ResourceID, model.FileScopeQrCodeTenant, 0, row.TenantID
case "operating_expenses":
return model.FileResourceOperatingExpense, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0
case "manual_disbursements":
return model.FileResourceManualDisbursement, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0
default:
return "", 0, "", 0, 0
}
}
func canonicalKeys(row sourceRow) ([]string, int, int) {
keys := make([]string, 0, 2)
malformed := 0
invalid := 0
if len(row.Keys) > 0 {
var rawKeys []string
if err := json.Unmarshal(row.Keys, &rawKeys); err != nil {
malformed++
} else {
keys = append(keys, rawKeys...)
}
}
if strings.TrimSpace(row.ScalarKey) != "" {
keys = append(keys, row.ScalarKey)
}
seen := make(map[string]struct{}, len(keys))
canonical := make([]string, 0, len(keys))
for _, raw := range keys {
key := filekey.Canonical(raw)
if key == "" {
invalid++
continue
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
canonical = append(canonical, key)
}
return canonical, malformed, invalid
}
func loadCheckpoint(path string) checkpointFile {
checkpoint := checkpointFile{Sources: make(map[string]uint64)}
if strings.TrimSpace(path) == "" {
return checkpoint
}
payload, err := os.ReadFile(path)
if err != nil {
return checkpoint
}
if err := json.Unmarshal(payload, &checkpoint); err != nil || checkpoint.Sources == nil {
checkpoint.Sources = make(map[string]uint64)
}
return checkpoint
}
func saveCheckpoint(path string, checkpoint checkpointFile) error {
checkpoint.UpdatedAt = time.Now().Format(time.RFC3339)
payload, err := json.MarshalIndent(checkpoint, "", " ")
if err != nil {
return err
}
dir := filepath.Dir(path)
if dir != "." {
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
}
temp := path + ".tmp"
if err := os.WriteFile(temp, payload, 0o600); err != nil {
return err
}
return os.Rename(temp, path)
}
func emptyAsNone(value string) string {
if strings.TrimSpace(value) == "" {
return "未启用"
}
return value
}
func init() {
// 保证来源顺序稳定,便于运维日志对比。
sort.SliceStable(sources, func(i, j int) bool {
return sources[i].Name < sources[j].Name
})
}
@@ -0,0 +1,52 @@
package main
import (
"testing"
"hfb_sys/backend/internal/model"
"gorm.io/datatypes"
)
func TestCanonicalKeysHandlesJSONScalarAndInvalidReferences(t *testing.T) {
row := sourceRow{
Keys: datatypes.JSON([]byte(`[
"/api/files/object?key=listing%2Fa.webp",
"listing/a.webp",
"https://external.example.com/file.webp"
]`)),
ScalarKey: "/api/files/object?key=listing%2Fb.webp",
}
keys, malformed, invalid := canonicalKeys(row)
if malformed != 0 || invalid != 1 {
t.Fatalf("malformed=%d invalid=%d, want 0/1", malformed, invalid)
}
if len(keys) != 2 || keys[0] != "listing/a.webp" || keys[1] != "listing/b.webp" {
t.Fatalf("keys = %#v", keys)
}
}
func TestCanonicalKeysReportsMalformedJSON(t *testing.T) {
keys, malformed, invalid := canonicalKeys(sourceRow{
Keys: datatypes.JSON([]byte(`{"bad":true}`)),
})
if len(keys) != 0 || malformed != 1 || invalid != 0 {
t.Fatalf("keys=%#v malformed=%d invalid=%d", keys, malformed, invalid)
}
}
func TestBindingTargetForConversationQrCode(t *testing.T) {
conversationID := uint64(88)
resourceType, resourceID, scope, ownerUserID, tenantID := bindingTarget("chat_qrcode_pool", sourceRow{
ResourceID: 9,
TenantID: 101,
ConversationID: &conversationID,
})
if resourceType != model.FileResourceChatConversation ||
resourceID != conversationID ||
scope != model.FileScopeQrCodeParticipant ||
ownerUserID != 0 ||
tenantID != 101 {
t.Fatalf("target = %s/%d/%s owner=%d tenant=%d", resourceType, resourceID, scope, ownerUserID, tenantID)
}
}