perf(files): 优化私有文件鉴权与高成本查询隔离
This commit is contained in:
@@ -110,6 +110,7 @@ func main() {
|
||||
jobCtx, stopJobs := context.WithCancel(context.Background())
|
||||
defer stopJobs()
|
||||
if deps.DB != nil {
|
||||
database.StartPoolStatsLogger(jobCtx, deps.DB, logger, 30*time.Second)
|
||||
var paymentRepo *payment.Repository
|
||||
if paymentConfigRepo := newPaymentConfigRepositoryForJobs(cfg, deps.DB, logger); paymentConfigRepo != nil {
|
||||
// 超时取消前的渠道核对要把收款推进到订单上,所以这里必须带真实的订单仓库,
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
// file-object-bindings-backfill 回填私有文件对象与业务资源的绑定关系。
|
||||
//
|
||||
// 默认只预演。实际写入时按主键游标分批提交,可重复执行;指定 -checkpoint
|
||||
// 后会记录每个来源已完成的最后主键,进程中断后可从断点继续。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"hfb_sys/backend/internal/config"
|
||||
"hfb_sys/backend/internal/database"
|
||||
"hfb_sys/backend/internal/filekey"
|
||||
"hfb_sys/backend/internal/logging"
|
||||
"hfb_sys/backend/internal/model"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
type sourceSpec struct {
|
||||
Name string
|
||||
SQL string
|
||||
}
|
||||
|
||||
type sourceRow struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
TenantID uint64 `gorm:"column:tenant_id"`
|
||||
OwnerUserID uint64 `gorm:"column:owner_user_id"`
|
||||
ResourceID uint64 `gorm:"column:resource_id"`
|
||||
ConversationID *uint64 `gorm:"column:conversation_id"`
|
||||
Keys datatypes.JSON `gorm:"column:object_keys"`
|
||||
ScalarKey string `gorm:"column:scalar_key"`
|
||||
}
|
||||
|
||||
type sourceStats struct {
|
||||
Rows int64 `json:"rows"`
|
||||
References int64 `json:"references"`
|
||||
InvalidKeys int64 `json:"invalid_keys"`
|
||||
MalformedJSON int64 `json:"malformed_json"`
|
||||
NoResource int64 `json:"no_resource"`
|
||||
InsertedBindings int64 `json:"inserted_bindings"`
|
||||
ExistingBindings int64 `json:"existing_bindings"`
|
||||
}
|
||||
|
||||
type checkpointFile struct {
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
Sources map[string]uint64 `json:"sources"`
|
||||
}
|
||||
|
||||
var sources = []sourceSpec{
|
||||
{
|
||||
Name: "game_accounts",
|
||||
SQL: `SELECT id, source_tenant_id AS tenant_id, owner_id AS owner_user_id,
|
||||
id AS resource_id, NULL AS conversation_id, screenshot_urls AS object_keys,
|
||||
'' AS scalar_key
|
||||
FROM game_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "chat_messages",
|
||||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, conversation_id AS resource_id,
|
||||
NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key
|
||||
FROM chat_messages WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "handoff_records",
|
||||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
|
||||
NULL AS conversation_id, attachment_urls AS object_keys, '' AS scalar_key
|
||||
FROM handoff_records WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "disputes",
|
||||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
|
||||
NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key
|
||||
FROM disputes WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "order_checkouts",
|
||||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, order_id AS resource_id,
|
||||
NULL AS conversation_id, evidence_urls AS object_keys, '' AS scalar_key
|
||||
FROM order_checkouts WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "user_payment_accounts",
|
||||
SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id,
|
||||
NULL AS conversation_id, certificate_urls AS object_keys, '' AS scalar_key
|
||||
FROM user_payment_accounts WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "withdrawal_requests",
|
||||
SQL: `SELECT id, tenant_id, user_id AS owner_user_id, id AS resource_id,
|
||||
NULL AS conversation_id, NULL AS object_keys, payment_proof_url AS scalar_key
|
||||
FROM withdrawal_requests WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "chat_qrcode_pool",
|
||||
SQL: `SELECT id, tenant_id, 0 AS owner_user_id, id AS resource_id,
|
||||
conversation_id, NULL AS object_keys, image_url AS scalar_key
|
||||
FROM chat_qrcode_pool WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "operating_expenses",
|
||||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id,
|
||||
NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key
|
||||
FROM operating_expenses WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
{
|
||||
Name: "manual_disbursements",
|
||||
SQL: `SELECT id, 0 AS tenant_id, 0 AS owner_user_id, id AS resource_id,
|
||||
NULL AS conversation_id, NULL AS object_keys, voucher_url AS scalar_key
|
||||
FROM manual_disbursements WHERE id > ? ORDER BY id ASC LIMIT ?`,
|
||||
},
|
||||
}
|
||||
|
||||
func main() {
|
||||
dryRun := flag.Bool("dry-run", true, "只统计不写库;实际回填时传 -dry-run=false")
|
||||
batchSize := flag.Int("batch-size", 500, "每批主键记录数")
|
||||
onlySource := flag.String("source", "", "只处理指定来源,例如 chat_messages")
|
||||
afterID := flag.Uint64("after-id", 0, "所有来源从该主键之后开始,仅用于手动续跑")
|
||||
checkpointPath := flag.String("checkpoint", "", "断点 JSON 文件路径;为空时不持久化断点")
|
||||
flag.Parse()
|
||||
|
||||
if *batchSize < 1 || *batchSize > 5000 {
|
||||
log.Fatal("-batch-size 必须在 1 到 5000 之间")
|
||||
}
|
||||
selected := selectSources(*onlySource)
|
||||
if len(selected) == 0 {
|
||||
log.Fatalf("未知来源 %q", *onlySource)
|
||||
}
|
||||
|
||||
cfg := config.Load()
|
||||
logger, err := logging.New(cfg.Log)
|
||||
if err != nil {
|
||||
log.Fatalf("初始化日志失败: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = logger.Sync()
|
||||
}()
|
||||
db, err := database.OpenMySQLWithOptions(cfg.MySQLDSN, cfg.Log.Level, logger, database.MySQLOptions{})
|
||||
if err != nil {
|
||||
log.Fatalf("连接数据库失败: %v", err)
|
||||
}
|
||||
|
||||
checkpoint := loadCheckpoint(*checkpointPath)
|
||||
mode := "预演"
|
||||
if !*dryRun {
|
||||
mode = "实际回填"
|
||||
}
|
||||
fmt.Printf("模式=%s 批次=%d 来源=%d 断点=%s\n", mode, *batchSize, len(selected), emptyAsNone(*checkpointPath))
|
||||
|
||||
for _, source := range selected {
|
||||
after := checkpoint.Sources[source.Name]
|
||||
if *afterID > after {
|
||||
after = *afterID
|
||||
}
|
||||
stats, lastID, err := runSource(db, source, after, *batchSize, *dryRun, func(next uint64) error {
|
||||
if *dryRun || *checkpointPath == "" {
|
||||
return nil
|
||||
}
|
||||
checkpoint.Sources[source.Name] = next
|
||||
return saveCheckpoint(*checkpointPath, checkpoint)
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("来源 %s 回填失败(最后主键 %d): %v", source.Name, lastID, err)
|
||||
}
|
||||
fmt.Printf(
|
||||
"[%s] 扫描记录=%d 对象引用=%d 新增绑定=%d 已存在=%d 无效key=%d JSON解析失败=%d 无资源关联=%d 最后主键=%d\n",
|
||||
source.Name,
|
||||
stats.Rows,
|
||||
stats.References,
|
||||
stats.InsertedBindings,
|
||||
stats.ExistingBindings,
|
||||
stats.InvalidKeys,
|
||||
stats.MalformedJSON,
|
||||
stats.NoResource,
|
||||
lastID,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func selectSources(name string) []sourceSpec {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return sources
|
||||
}
|
||||
for _, source := range sources {
|
||||
if source.Name == name {
|
||||
return []sourceSpec{source}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runSource(
|
||||
db *gorm.DB,
|
||||
source sourceSpec,
|
||||
afterID uint64,
|
||||
batchSize int,
|
||||
dryRun bool,
|
||||
checkpoint func(uint64) error,
|
||||
) (sourceStats, uint64, error) {
|
||||
var stats sourceStats
|
||||
cursor := afterID
|
||||
for {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
var rows []sourceRow
|
||||
err := db.WithContext(ctx).Raw(source.SQL, cursor, batchSize).Scan(&rows).Error
|
||||
cancel()
|
||||
if err != nil {
|
||||
return stats, cursor, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return stats, cursor, nil
|
||||
}
|
||||
|
||||
bindings := make([]model.FileObjectBinding, 0, len(rows)*2)
|
||||
seen := make(map[string]struct{})
|
||||
for _, row := range rows {
|
||||
stats.Rows++
|
||||
cursor = row.ID
|
||||
resourceType, resourceID, scope, ownerUserID, tenantID := bindingTarget(source.Name, row)
|
||||
keys, malformed, invalid := canonicalKeys(row)
|
||||
stats.MalformedJSON += int64(malformed)
|
||||
stats.InvalidKeys += int64(invalid)
|
||||
if resourceID == 0 {
|
||||
if len(keys) > 0 || malformed > 0 || invalid > 0 {
|
||||
stats.NoResource++
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, key := range keys {
|
||||
stats.References++
|
||||
identity := key + "\x00" + resourceType + "\x00" + scope
|
||||
if _, exists := seen[identity]; exists {
|
||||
stats.ExistingBindings++
|
||||
continue
|
||||
}
|
||||
seen[identity] = struct{}{}
|
||||
bindings = append(bindings, model.FileObjectBinding{
|
||||
TenantID: tenantID,
|
||||
CanonicalKey: key,
|
||||
ResourceType: resourceType,
|
||||
ResourceID: resourceID,
|
||||
AccessScope: scope,
|
||||
OwnerUserID: ownerUserID,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !dryRun && len(bindings) > 0 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
result := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&bindings)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
stats.InsertedBindings += result.RowsAffected
|
||||
stats.ExistingBindings += int64(len(bindings)) - result.RowsAffected
|
||||
return nil
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return stats, cursor, err
|
||||
}
|
||||
} else {
|
||||
stats.InsertedBindings += int64(len(bindings))
|
||||
}
|
||||
if err := checkpoint(cursor); err != nil {
|
||||
return stats, cursor, err
|
||||
}
|
||||
if len(rows) < batchSize {
|
||||
return stats, cursor, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func bindingTarget(source string, row sourceRow) (resourceType string, resourceID uint64, scope string, ownerUserID, tenantID uint64) {
|
||||
switch source {
|
||||
case "game_accounts":
|
||||
return model.FileResourceGameAccount, row.ResourceID, model.FileScopeListingAsset, row.OwnerUserID, row.TenantID
|
||||
case "chat_messages":
|
||||
return model.FileResourceChatConversation, row.ResourceID, model.FileScopeChatParticipant, 0, 0
|
||||
case "handoff_records", "disputes", "order_checkouts":
|
||||
return model.FileResourceOrder, row.ResourceID, model.FileScopeOrderParticipant, 0, 0
|
||||
case "user_payment_accounts":
|
||||
return model.FileResourcePaymentAccount, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID
|
||||
case "withdrawal_requests":
|
||||
return model.FileResourceWithdrawal, row.ResourceID, model.FileScopeOwner, row.OwnerUserID, row.TenantID
|
||||
case "chat_qrcode_pool":
|
||||
if row.ConversationID != nil && *row.ConversationID > 0 {
|
||||
return model.FileResourceChatConversation, *row.ConversationID, model.FileScopeQrCodeParticipant, 0, row.TenantID
|
||||
}
|
||||
return model.FileResourceChatQrCode, row.ResourceID, model.FileScopeQrCodeTenant, 0, row.TenantID
|
||||
case "operating_expenses":
|
||||
return model.FileResourceOperatingExpense, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0
|
||||
case "manual_disbursements":
|
||||
return model.FileResourceManualDisbursement, row.ResourceID, model.FileScopePlatformSuperAdmin, 0, 0
|
||||
default:
|
||||
return "", 0, "", 0, 0
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalKeys(row sourceRow) ([]string, int, int) {
|
||||
keys := make([]string, 0, 2)
|
||||
malformed := 0
|
||||
invalid := 0
|
||||
if len(row.Keys) > 0 {
|
||||
var rawKeys []string
|
||||
if err := json.Unmarshal(row.Keys, &rawKeys); err != nil {
|
||||
malformed++
|
||||
} else {
|
||||
keys = append(keys, rawKeys...)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(row.ScalarKey) != "" {
|
||||
keys = append(keys, row.ScalarKey)
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(keys))
|
||||
canonical := make([]string, 0, len(keys))
|
||||
for _, raw := range keys {
|
||||
key := filekey.Canonical(raw)
|
||||
if key == "" {
|
||||
invalid++
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
canonical = append(canonical, key)
|
||||
}
|
||||
return canonical, malformed, invalid
|
||||
}
|
||||
|
||||
func loadCheckpoint(path string) checkpointFile {
|
||||
checkpoint := checkpointFile{Sources: make(map[string]uint64)}
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return checkpoint
|
||||
}
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return checkpoint
|
||||
}
|
||||
if err := json.Unmarshal(payload, &checkpoint); err != nil || checkpoint.Sources == nil {
|
||||
checkpoint.Sources = make(map[string]uint64)
|
||||
}
|
||||
return checkpoint
|
||||
}
|
||||
|
||||
func saveCheckpoint(path string, checkpoint checkpointFile) error {
|
||||
checkpoint.UpdatedAt = time.Now().Format(time.RFC3339)
|
||||
payload, err := json.MarshalIndent(checkpoint, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
if dir != "." {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
temp := path + ".tmp"
|
||||
if err := os.WriteFile(temp, payload, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temp, path)
|
||||
}
|
||||
|
||||
func emptyAsNone(value string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return "未启用"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func init() {
|
||||
// 保证来源顺序稳定,便于运维日志对比。
|
||||
sort.SliceStable(sources, func(i, j int) bool {
|
||||
return sources[i].Name < sources[j].Name
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"hfb_sys/backend/internal/model"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
)
|
||||
|
||||
func TestCanonicalKeysHandlesJSONScalarAndInvalidReferences(t *testing.T) {
|
||||
row := sourceRow{
|
||||
Keys: datatypes.JSON([]byte(`[
|
||||
"/api/files/object?key=listing%2Fa.webp",
|
||||
"listing/a.webp",
|
||||
"https://external.example.com/file.webp"
|
||||
]`)),
|
||||
ScalarKey: "/api/files/object?key=listing%2Fb.webp",
|
||||
}
|
||||
keys, malformed, invalid := canonicalKeys(row)
|
||||
if malformed != 0 || invalid != 1 {
|
||||
t.Fatalf("malformed=%d invalid=%d, want 0/1", malformed, invalid)
|
||||
}
|
||||
if len(keys) != 2 || keys[0] != "listing/a.webp" || keys[1] != "listing/b.webp" {
|
||||
t.Fatalf("keys = %#v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalKeysReportsMalformedJSON(t *testing.T) {
|
||||
keys, malformed, invalid := canonicalKeys(sourceRow{
|
||||
Keys: datatypes.JSON([]byte(`{"bad":true}`)),
|
||||
})
|
||||
if len(keys) != 0 || malformed != 1 || invalid != 0 {
|
||||
t.Fatalf("keys=%#v malformed=%d invalid=%d", keys, malformed, invalid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBindingTargetForConversationQrCode(t *testing.T) {
|
||||
conversationID := uint64(88)
|
||||
resourceType, resourceID, scope, ownerUserID, tenantID := bindingTarget("chat_qrcode_pool", sourceRow{
|
||||
ResourceID: 9,
|
||||
TenantID: 101,
|
||||
ConversationID: &conversationID,
|
||||
})
|
||||
if resourceType != model.FileResourceChatConversation ||
|
||||
resourceID != conversationID ||
|
||||
scope != model.FileScopeQrCodeParticipant ||
|
||||
ownerUserID != 0 ||
|
||||
tenantID != 101 {
|
||||
t.Fatalf("target = %s/%d/%s owner=%d tenant=%d", resourceType, resourceID, scope, ownerUserID, tenantID)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user