支持迁移对象存储到 OSS
This commit is contained in:
@@ -40,6 +40,16 @@ STORAGE_ENDPOINT=http://localhost:19090
|
|||||||
STORAGE_BUCKET=hfb-sys
|
STORAGE_BUCKET=hfb-sys
|
||||||
STORAGE_ACCESS_KEY_ID=minioadmin
|
STORAGE_ACCESS_KEY_ID=minioadmin
|
||||||
STORAGE_SECRET_ACCESS_KEY=minioadmin
|
STORAGE_SECRET_ACCESS_KEY=minioadmin
|
||||||
|
STORAGE_REGION=
|
||||||
|
STORAGE_BUCKET_LOOKUP=auto
|
||||||
|
|
||||||
|
# 对象存储迁移期间的镜像端;本地开发默认关闭。
|
||||||
|
STORAGE_MIRROR_ENDPOINT=
|
||||||
|
STORAGE_MIRROR_BUCKET=
|
||||||
|
STORAGE_MIRROR_ACCESS_KEY_ID=
|
||||||
|
STORAGE_MIRROR_SECRET_ACCESS_KEY=
|
||||||
|
STORAGE_MIRROR_REGION=
|
||||||
|
STORAGE_MIRROR_BUCKET_LOOKUP=auto
|
||||||
|
|
||||||
# 短信服务:本地开发使用 mock。
|
# 短信服务:本地开发使用 mock。
|
||||||
SMS_PROVIDER=mock
|
SMS_PROVIDER=mock
|
||||||
|
|||||||
@@ -48,13 +48,26 @@ LOG_ENABLE_CONSOLE=false
|
|||||||
LOG_ENABLE_FILE=true
|
LOG_ENABLE_FILE=true
|
||||||
LOG_RETAIN_DAYS=14
|
LOG_RETAIN_DAYS=14
|
||||||
|
|
||||||
# MinIO 容器初始化变量,同时供后端对象存储使用;使用内置 MinIO 时,STORAGE_* 密钥必须和 MINIO_ROOT_* 保持一致。
|
# MinIO 容器初始化变量。迁移完成前仍保留,用于回退与校验。
|
||||||
MINIO_ROOT_USER=change-minio-user
|
MINIO_ROOT_USER=change-minio-user
|
||||||
MINIO_ROOT_PASSWORD=change-minio-password
|
MINIO_ROOT_PASSWORD=change-minio-password
|
||||||
|
# 当前主对象存储。使用内置 MinIO 时,STORAGE_* 密钥必须和 MINIO_ROOT_* 保持一致。
|
||||||
STORAGE_ENDPOINT=http://minio:9000
|
STORAGE_ENDPOINT=http://minio:9000
|
||||||
STORAGE_BUCKET=hfb-sys
|
STORAGE_BUCKET=hfb-sys
|
||||||
STORAGE_ACCESS_KEY_ID=change-minio-user
|
STORAGE_ACCESS_KEY_ID=change-minio-user
|
||||||
STORAGE_SECRET_ACCESS_KEY=change-minio-password
|
STORAGE_SECRET_ACCESS_KEY=change-minio-password
|
||||||
|
STORAGE_REGION=
|
||||||
|
STORAGE_BUCKET_LOOKUP=auto
|
||||||
|
|
||||||
|
# OSS 迁移镜像端。迁移期间设置后,新上传文件会同时写入 MinIO 与 OSS。
|
||||||
|
# 杭州 ECS 应使用内网 Endpoint:https://oss-cn-hangzhou-internal.aliyuncs.com
|
||||||
|
# OSS 的 S3 兼容访问使用 cn-hangzhou 区域和 DNS Bucket 寻址。
|
||||||
|
STORAGE_MIRROR_ENDPOINT=
|
||||||
|
STORAGE_MIRROR_BUCKET=hfb-sys-assets
|
||||||
|
STORAGE_MIRROR_ACCESS_KEY_ID=
|
||||||
|
STORAGE_MIRROR_SECRET_ACCESS_KEY=
|
||||||
|
STORAGE_MIRROR_REGION=cn-hangzhou
|
||||||
|
STORAGE_MIRROR_BUCKET_LOOKUP=dns
|
||||||
|
|
||||||
# 生产环境建议接入真实短信服务;未配置时不要使用 mock 对外运营。
|
# 生产环境建议接入真实短信服务;未配置时不要使用 mock 对外运营。
|
||||||
SMS_PROVIDER=aliyun
|
SMS_PROVIDER=aliyun
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
|||||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||||
--mount=type=cache,target=/root/.cache/go-build \
|
--mount=type=cache,target=/root/.cache/go-build \
|
||||||
CGO_ENABLED=0 GOOS=linux go build -o /out/hfb-api ./cmd/api
|
CGO_ENABLED=0 GOOS=linux go build -o /out/hfb-api ./cmd/api
|
||||||
|
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||||
|
--mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
CGO_ENABLED=0 GOOS=linux go build -o /out/hfb-storage-migrate ./cmd/storage-migrate
|
||||||
|
|
||||||
FROM alpine:3.22
|
FROM alpine:3.22
|
||||||
|
|
||||||
@@ -30,6 +33,7 @@ RUN sed -i 's#https\?://dl-cdn.alpinelinux.org/alpine#https://mirrors.aliyun.com
|
|||||||
&& mkdir -p /app/logs \
|
&& mkdir -p /app/logs \
|
||||||
&& chown app:app /app/logs
|
&& chown app:app /app/logs
|
||||||
COPY --from=build /out/hfb-api /app/hfb-api
|
COPY --from=build /out/hfb-api /app/hfb-api
|
||||||
|
COPY --from=build /out/hfb-storage-migrate /app/hfb-storage-migrate
|
||||||
COPY --from=build /go/bin/goose /app/goose
|
COPY --from=build /go/bin/goose /app/goose
|
||||||
COPY --from=build /src/migrations /app/migrations
|
COPY --from=build /src/migrations /app/migrations
|
||||||
USER app:app
|
USER app:app
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"hfb_sys/backend/internal/config"
|
||||||
|
filemodule "hfb_sys/backend/internal/modules/file"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7"
|
||||||
|
)
|
||||||
|
|
||||||
|
type migrationConfig struct {
|
||||||
|
source config.StorageConfig
|
||||||
|
target config.StorageConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
type summary struct {
|
||||||
|
objectsCopied int64
|
||||||
|
bytesCopied int64
|
||||||
|
objectsSkipped int64
|
||||||
|
objectsFailed int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
verifyOnly := flag.Bool("verify", false, "仅校验目标对象,不上传文件")
|
||||||
|
dryRun := flag.Bool("dry-run", false, "只输出待迁移对象,不上传文件")
|
||||||
|
prefix := flag.String("prefix", "", "仅迁移指定前缀")
|
||||||
|
flag.Parse()
|
||||||
|
if *verifyOnly && *dryRun {
|
||||||
|
log.Fatal("--verify 与 --dry-run 不能同时使用")
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := loadMigrationConfig()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
if sameStorage(cfg.source, cfg.target) {
|
||||||
|
log.Fatal("源存储和目标存储相同,已拒绝执行")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
source, err := filemodule.NewStorageClient(cfg.source)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("创建 MinIO 源客户端失败: %v", err)
|
||||||
|
}
|
||||||
|
target, err := filemodule.NewStorageClient(cfg.target)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("创建 OSS 目标客户端失败: %v", err)
|
||||||
|
}
|
||||||
|
if err := requireBucket(ctx, source, cfg.source.Bucket, "源"); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := requireBucket(ctx, target, cfg.target.Bucket, "目标"); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mode := "同步"
|
||||||
|
if *verifyOnly {
|
||||||
|
mode = "校验"
|
||||||
|
} else if *dryRun {
|
||||||
|
mode = "预演"
|
||||||
|
}
|
||||||
|
log.Printf("开始%s:%s/%s -> %s/%s,前缀=%q", mode, cfg.source.Endpoint, cfg.source.Bucket, cfg.target.Endpoint, cfg.target.Bucket, *prefix)
|
||||||
|
|
||||||
|
result, err := migrate(ctx, source, cfg.source.Bucket, target, cfg.target.Bucket, *prefix, *verifyOnly, *dryRun)
|
||||||
|
log.Printf("结束:已复制 %d 个对象(%d 字节),已跳过 %d 个对象,失败 %d 个对象", result.objectsCopied, result.bytesCopied, result.objectsSkipped, result.objectsFailed)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadMigrationConfig() (migrationConfig, error) {
|
||||||
|
source := config.StorageConfig{
|
||||||
|
Endpoint: os.Getenv("STORAGE_ENDPOINT"),
|
||||||
|
Bucket: os.Getenv("STORAGE_BUCKET"),
|
||||||
|
AccessKeyID: os.Getenv("STORAGE_ACCESS_KEY_ID"),
|
||||||
|
SecretAccessKey: os.Getenv("STORAGE_SECRET_ACCESS_KEY"),
|
||||||
|
Region: os.Getenv("STORAGE_REGION"),
|
||||||
|
BucketLookup: valueOrDefault("STORAGE_BUCKET_LOOKUP", "auto"),
|
||||||
|
}
|
||||||
|
target := config.StorageConfig{
|
||||||
|
Endpoint: os.Getenv("STORAGE_MIRROR_ENDPOINT"),
|
||||||
|
Bucket: os.Getenv("STORAGE_MIRROR_BUCKET"),
|
||||||
|
AccessKeyID: os.Getenv("STORAGE_MIRROR_ACCESS_KEY_ID"),
|
||||||
|
SecretAccessKey: os.Getenv("STORAGE_MIRROR_SECRET_ACCESS_KEY"),
|
||||||
|
Region: os.Getenv("STORAGE_MIRROR_REGION"),
|
||||||
|
BucketLookup: valueOrDefault("STORAGE_MIRROR_BUCKET_LOOKUP", "auto"),
|
||||||
|
}
|
||||||
|
if err := validateStorageConfig("源 MinIO", source); err != nil {
|
||||||
|
return migrationConfig{}, err
|
||||||
|
}
|
||||||
|
if err := validateStorageConfig("目标 OSS", target); err != nil {
|
||||||
|
return migrationConfig{}, err
|
||||||
|
}
|
||||||
|
return migrationConfig{source: source, target: target}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func valueOrDefault(key, fallback string) string {
|
||||||
|
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateStorageConfig(name string, cfg config.StorageConfig) error {
|
||||||
|
if strings.TrimSpace(cfg.Endpoint) == "" || strings.TrimSpace(cfg.Bucket) == "" || strings.TrimSpace(cfg.AccessKeyID) == "" || strings.TrimSpace(cfg.SecretAccessKey) == "" {
|
||||||
|
return fmt.Errorf("%s 配置不完整,请检查 endpoint、bucket 和访问密钥", name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameStorage(source, target config.StorageConfig) bool {
|
||||||
|
return strings.TrimRight(strings.ToLower(source.Endpoint), "/") == strings.TrimRight(strings.ToLower(target.Endpoint), "/") && source.Bucket == target.Bucket
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireBucket(ctx context.Context, client *minio.Client, bucket, label string) error {
|
||||||
|
exists, err := client.BucketExists(ctx, bucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("检查%s Bucket 失败: %w", label, err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return fmt.Errorf("%s Bucket 不存在: %s", label, bucket)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func migrate(ctx context.Context, source *minio.Client, sourceBucket string, target *minio.Client, targetBucket, prefix string, verifyOnly, dryRun bool) (summary, error) {
|
||||||
|
var result summary
|
||||||
|
for item := range source.ListObjects(ctx, sourceBucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
|
||||||
|
if item.Err != nil {
|
||||||
|
return result, fmt.Errorf("列出源对象失败: %w", item.Err)
|
||||||
|
}
|
||||||
|
sourceInfo, err := source.StatObject(ctx, sourceBucket, item.Key, minio.StatObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
result.objectsFailed++
|
||||||
|
log.Printf("跳过对象 %s:读取源对象元数据失败: %v", item.Key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
matches, err := targetMatches(ctx, target, targetBucket, item.Key, sourceInfo)
|
||||||
|
if err != nil {
|
||||||
|
result.objectsFailed++
|
||||||
|
log.Printf("跳过对象 %s:检查目标对象失败: %v", item.Key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if matches {
|
||||||
|
result.objectsSkipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if verifyOnly {
|
||||||
|
result.objectsFailed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if dryRun {
|
||||||
|
log.Printf("待复制:%s(%d 字节)", item.Key, sourceInfo.Size)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := copyObject(ctx, source, sourceBucket, target, targetBucket, item.Key, sourceInfo); err != nil {
|
||||||
|
result.objectsFailed++
|
||||||
|
log.Printf("复制对象 %s 失败: %v", item.Key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result.objectsCopied++
|
||||||
|
result.bytesCopied += sourceInfo.Size
|
||||||
|
}
|
||||||
|
if result.objectsFailed > 0 {
|
||||||
|
if verifyOnly {
|
||||||
|
return result, fmt.Errorf("校验失败:%d 个对象缺失、内容不匹配或无法检查", result.objectsFailed)
|
||||||
|
}
|
||||||
|
return result, fmt.Errorf("同步未完成:%d 个对象处理失败", result.objectsFailed)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func targetMatches(ctx context.Context, target *minio.Client, bucket, key string, source minio.ObjectInfo) (bool, error) {
|
||||||
|
targetInfo, err := target.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
if isNotFound(err) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("读取目标对象 %s 元数据失败: %w", key, err)
|
||||||
|
}
|
||||||
|
if targetInfo.Size != source.Size {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return sameETag(source.ETag, targetInfo.ETag), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyObject(ctx context.Context, source *minio.Client, sourceBucket string, target *minio.Client, targetBucket, key string, info minio.ObjectInfo) error {
|
||||||
|
object, err := source.GetObject(ctx, sourceBucket, key, minio.GetObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("读取源对象 %s 失败: %w", key, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = object.Close() }()
|
||||||
|
_, err = target.PutObject(ctx, targetBucket, key, object, info.Size, minio.PutObjectOptions{
|
||||||
|
ContentType: info.ContentType,
|
||||||
|
UserMetadata: info.UserMetadata,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("复制对象 %s 到目标 OSS 失败: %w", key, err)
|
||||||
|
}
|
||||||
|
verified, err := targetMatches(ctx, target, targetBucket, key, info)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !verified {
|
||||||
|
return fmt.Errorf("复制对象 %s 后校验失败", key)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameETag(left, right string) bool {
|
||||||
|
return strings.Trim(strings.ToLower(left), "\"") == strings.Trim(strings.ToLower(right), "\"")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isNotFound(err error) bool {
|
||||||
|
response := minio.ToErrorResponse(err)
|
||||||
|
switch response.Code {
|
||||||
|
case "NoSuchKey", "NoSuchObject", "NoSuchBucket", "NotFound":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,6 +24,7 @@ type Config struct {
|
|||||||
BootstrapAdminPassword string
|
BootstrapAdminPassword string
|
||||||
BootstrapAdminNickname string
|
BootstrapAdminNickname string
|
||||||
Storage StorageConfig
|
Storage StorageConfig
|
||||||
|
StorageMirror StorageConfig
|
||||||
SMS SMSConfig
|
SMS SMSConfig
|
||||||
Realname RealnameConfig
|
Realname RealnameConfig
|
||||||
Log LogConfig
|
Log LogConfig
|
||||||
@@ -35,6 +36,8 @@ type StorageConfig struct {
|
|||||||
Bucket string
|
Bucket string
|
||||||
AccessKeyID string
|
AccessKeyID string
|
||||||
SecretAccessKey string
|
SecretAccessKey string
|
||||||
|
Region string
|
||||||
|
BucketLookup string
|
||||||
}
|
}
|
||||||
|
|
||||||
type SMSConfig struct {
|
type SMSConfig struct {
|
||||||
@@ -91,6 +94,17 @@ func Load() Config {
|
|||||||
Bucket: getEnv("STORAGE_BUCKET", "hfb-sys"),
|
Bucket: getEnv("STORAGE_BUCKET", "hfb-sys"),
|
||||||
AccessKeyID: getEnv("STORAGE_ACCESS_KEY_ID", "minioadmin"),
|
AccessKeyID: getEnv("STORAGE_ACCESS_KEY_ID", "minioadmin"),
|
||||||
SecretAccessKey: getEnv("STORAGE_SECRET_ACCESS_KEY", "minioadmin"),
|
SecretAccessKey: getEnv("STORAGE_SECRET_ACCESS_KEY", "minioadmin"),
|
||||||
|
Region: getEnv("STORAGE_REGION", ""),
|
||||||
|
BucketLookup: getEnv("STORAGE_BUCKET_LOOKUP", "auto"),
|
||||||
|
},
|
||||||
|
// StorageMirror 仅在对象存储迁移期间使用。镜像端配置完整时,新上传文件会同步写入两个存储端。
|
||||||
|
StorageMirror: StorageConfig{
|
||||||
|
Endpoint: getEnv("STORAGE_MIRROR_ENDPOINT", ""),
|
||||||
|
Bucket: getEnv("STORAGE_MIRROR_BUCKET", ""),
|
||||||
|
AccessKeyID: getEnv("STORAGE_MIRROR_ACCESS_KEY_ID", ""),
|
||||||
|
SecretAccessKey: getEnv("STORAGE_MIRROR_SECRET_ACCESS_KEY", ""),
|
||||||
|
Region: getEnv("STORAGE_MIRROR_REGION", ""),
|
||||||
|
BucketLookup: getEnv("STORAGE_MIRROR_BUCKET_LOOKUP", "auto"),
|
||||||
},
|
},
|
||||||
SMS: SMSConfig{
|
SMS: SMSConfig{
|
||||||
Provider: getEnv("SMS_PROVIDER", "mock"),
|
Provider: getEnv("SMS_PROVIDER", "mock"),
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package file
|
package file
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
@@ -24,6 +25,8 @@ type Storage struct {
|
|||||||
bucket string
|
bucket string
|
||||||
bucketMu sync.Mutex
|
bucketMu sync.Mutex
|
||||||
bucketReady bool
|
bucketReady bool
|
||||||
|
mirror *Storage
|
||||||
|
mirrorErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
type Object struct {
|
type Object struct {
|
||||||
@@ -33,14 +36,7 @@ type Object struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewStorage(cfg config.StorageConfig) (*Storage, error) {
|
func NewStorage(cfg config.StorageConfig) (*Storage, error) {
|
||||||
endpoint, secure, err := normalizeEndpoint(cfg.Endpoint)
|
client, err := NewStorageClient(cfg)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
client, err := minio.New(endpoint, &minio.Options{
|
|
||||||
Creds: credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
|
|
||||||
Secure: secure,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -51,6 +47,43 @@ func NewStorage(cfg config.StorageConfig) (*Storage, error) {
|
|||||||
return storage, nil
|
return storage, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewStorageClient 根据项目存储配置创建 S3 兼容客户端,供迁移工具复用。
|
||||||
|
func NewStorageClient(cfg config.StorageConfig) (*minio.Client, error) {
|
||||||
|
endpoint, secure, err := normalizeEndpoint(cfg.Endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
lookup, err := parseBucketLookup(cfg.BucketLookup)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return minio.New(endpoint, &minio.Options{
|
||||||
|
Creds: credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
|
||||||
|
Secure: secure,
|
||||||
|
Region: cfg.Region,
|
||||||
|
BucketLookup: lookup,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseBucketLookup(raw string) (minio.BucketLookupType, error) {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||||
|
case "", "auto":
|
||||||
|
return minio.BucketLookupAuto, nil
|
||||||
|
case "dns":
|
||||||
|
return minio.BucketLookupDNS, nil
|
||||||
|
case "path":
|
||||||
|
return minio.BucketLookupPath, nil
|
||||||
|
default:
|
||||||
|
return minio.BucketLookupAuto, fmt.Errorf("unsupported storage bucket lookup: %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetMirror 启用迁移期间的同步双写。镜像端不可用时上传会失败,避免出现未同步的业务文件。
|
||||||
|
func (s *Storage) SetMirror(mirror *Storage, err error) {
|
||||||
|
s.mirror = mirror
|
||||||
|
s.mirrorErr = err
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Storage) Put(ctx context.Context, scene string, header *multipart.FileHeader, reader io.Reader, contentType string) (string, error) {
|
func (s *Storage) Put(ctx context.Context, scene string, header *multipart.FileHeader, reader io.Reader, contentType string) (string, error) {
|
||||||
key, err := newObjectKey(scene, header.Filename)
|
key, err := newObjectKey(scene, header.Filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -65,6 +98,28 @@ func (s *Storage) Put(ctx context.Context, scene string, header *multipart.FileH
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Storage) PutObject(ctx context.Context, key string, reader io.Reader, size int64, contentType string, metadata map[string]string) error {
|
func (s *Storage) PutObject(ctx context.Context, key string, reader io.Reader, size int64, contentType string, metadata map[string]string) error {
|
||||||
|
if s.mirrorErr != nil {
|
||||||
|
return fmt.Errorf("storage mirror unavailable: %w", s.mirrorErr)
|
||||||
|
}
|
||||||
|
if s.mirror == nil {
|
||||||
|
return s.putObject(ctx, key, reader, size, contentType, metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 当前文件上传大小被限制在 10MB;镜像写入前缓存在内存中,确保两个端点得到完全相同的字节流。
|
||||||
|
data, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.putObject(ctx, key, bytes.NewReader(data), int64(len(data)), contentType, metadata); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.mirror.putObject(ctx, key, bytes.NewReader(data), int64(len(data)), contentType, metadata); err != nil {
|
||||||
|
return fmt.Errorf("storage mirror upload failed: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Storage) putObject(ctx context.Context, key string, reader io.Reader, size int64, contentType string, metadata map[string]string) error {
|
||||||
if err := s.ensureBucketReady(ctx); err != nil {
|
if err := s.ensureBucketReady(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package file
|
|||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSanitizeObjectMetadataEscapesNonASCII(t *testing.T) {
|
func TestSanitizeObjectMetadataEscapesNonASCII(t *testing.T) {
|
||||||
@@ -15,6 +17,29 @@ func TestSanitizeObjectMetadataEscapesNonASCII(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseBucketLookup(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
want minio.BucketLookupType
|
||||||
|
}{
|
||||||
|
{name: "auto", raw: "auto", want: minio.BucketLookupAuto},
|
||||||
|
{name: "dns", raw: "dns", want: minio.BucketLookupDNS},
|
||||||
|
{name: "path", raw: "path", want: minio.BucketLookupPath},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := parseBucketLookup(tt.raw)
|
||||||
|
if err != nil || got != tt.want {
|
||||||
|
t.Fatalf("parseBucketLookup(%q) = %v, %v", tt.raw, got, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if _, err := parseBucketLookup("invalid"); err == nil {
|
||||||
|
t.Fatal("invalid bucket lookup should fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAnnouncementSceneUsesPublicFileURL(t *testing.T) {
|
func TestAnnouncementSceneUsesPublicFileURL(t *testing.T) {
|
||||||
key := "announcement/2026/06/15/example.webp"
|
key := "announcement/2026/06/15/example.webp"
|
||||||
got := fileURLForScene("announcement", key)
|
got := fileURLForScene("announcement", key)
|
||||||
|
|||||||
@@ -333,6 +333,26 @@ func New(cfg config.Config, deps Dependencies, logger *zap.Logger) *gin.Engine {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warn("文件存储桶尚未就绪,接口请求时将重试", zap.Error(err))
|
logger.Warn("文件存储桶尚未就绪,接口请求时将重试", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
if cfg.StorageMirror.Endpoint != "" {
|
||||||
|
mirrorStorage, mirrorErr := filemodule.NewStorage(cfg.StorageMirror)
|
||||||
|
if fileStorage != nil {
|
||||||
|
// 客户端已创建时,即使启动检查失败也由上传请求重新探测 Bucket。
|
||||||
|
if mirrorStorage != nil {
|
||||||
|
fileStorage.SetMirror(mirrorStorage, nil)
|
||||||
|
} else {
|
||||||
|
fileStorage.SetMirror(nil, mirrorErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mirrorErr != nil {
|
||||||
|
if mirrorStorage != nil {
|
||||||
|
logger.Warn("文件存储镜像启动检查失败,上传时将重试", zap.Error(mirrorErr))
|
||||||
|
} else {
|
||||||
|
logger.Error("文件存储镜像客户端创建失败,已阻止新文件上传", zap.Error(mirrorErr))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
logger.Info("文件存储镜像已启用")
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
fileService := filemodule.NewService(fileStorage)
|
fileService := filemodule.NewService(fileStorage)
|
||||||
fileHandler := filemodule.NewHandler(fileService, fileStorage)
|
fileHandler := filemodule.NewHandler(fileService, fileStorage)
|
||||||
|
|||||||
@@ -49,6 +49,30 @@ id -g yml
|
|||||||
|
|
||||||
后端容器会使用该 UID/GID 运行。部署脚本由 root 执行时会自动把已有日志修正为该属主;非 root 执行且发现旧的 root 日志时,会给出需要执行的 `chown` 命令后停止。
|
后端容器会使用该 UID/GID 运行。部署脚本由 root 执行时会自动把已有日志修正为该属主;非 root 执行且发现旧的 root 日志时,会给出需要执行的 `chown` 命令后停止。
|
||||||
|
|
||||||
|
### 迁移对象存储到阿里云 OSS
|
||||||
|
|
||||||
|
创建私有 Bucket 后,先保持 `STORAGE_*` 指向当前 MinIO,在 `backend/.env` 配置 OSS 镜像端:
|
||||||
|
|
||||||
|
```text
|
||||||
|
STORAGE_MIRROR_ENDPOINT=https://oss-cn-hangzhou-internal.aliyuncs.com
|
||||||
|
STORAGE_MIRROR_BUCKET=hfb-sys-assets
|
||||||
|
STORAGE_MIRROR_ACCESS_KEY_ID=change-oss-access-key-id
|
||||||
|
STORAGE_MIRROR_SECRET_ACCESS_KEY=change-oss-access-key-secret
|
||||||
|
STORAGE_MIRROR_REGION=cn-hangzhou
|
||||||
|
STORAGE_MIRROR_BUCKET_LOOKUP=dns
|
||||||
|
```
|
||||||
|
|
||||||
|
RAM 凭证只授予该 Bucket 的列举、读取、写入对象权限,不要使用阿里云主账号 AccessKey。镜像端启用后,新上传文件会同步写入 MinIO 和 OSS;任一镜像写入失败时上传会失败,避免数据库引用未同步文件。
|
||||||
|
|
||||||
|
部署支持镜像配置的镜像后,执行以下命令同步历史对象;命令可重复执行,不会删除 MinIO 数据:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/migrate-minio-to-oss.sh
|
||||||
|
./scripts/migrate-minio-to-oss.sh --verify
|
||||||
|
```
|
||||||
|
|
||||||
|
校验通过后,将 `STORAGE_*` 改为同一组 OSS 配置,再清空所有 `STORAGE_MIRROR_*` 配置并重新部署。杭州 ECS 必须使用 `oss-cn-hangzhou-internal.aliyuncs.com`,避免跨公网访问。MinIO 保留至少 30 天后再考虑下线。
|
||||||
|
|
||||||
### 同机部署选号网(hfb_show,可选)
|
### 同机部署选号网(hfb_show,可选)
|
||||||
|
|
||||||
在 `backend/.env` 增加(域名不要带 `https://`):
|
在 `backend/.env` 增加(域名不要带 `https://`):
|
||||||
|
|||||||
Executable
+33
@@ -0,0 +1,33 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
COMPOSE_FILE="${ROOT_DIR}/deploy/docker-compose.prod.yml"
|
||||||
|
BACKEND_ENV="${ROOT_DIR}/backend/.env"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
用法: ./scripts/migrate-minio-to-oss.sh [--verify|--dry-run] [--prefix 前缀]
|
||||||
|
|
||||||
|
迁移配置写入 backend/.env:
|
||||||
|
STORAGE_* 当前 MinIO 源端配置
|
||||||
|
STORAGE_MIRROR_* 杭州 OSS 目标端配置
|
||||||
|
|
||||||
|
命令会复制同名 object key,不删除 MinIO 数据;可重复执行。
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ ! -f "${BACKEND_ENV}" ]]; then
|
||||||
|
echo "未找到 backend/.env" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
exec docker compose --env-file "${BACKEND_ENV}" -f "${COMPOSE_FILE}" \
|
||||||
|
run --rm --no-deps backend /app/hfb-storage-migrate "$@"
|
||||||
Reference in New Issue
Block a user