准备迁移1

2026-08-23 12:54:38 +08:00
parent 360c5ea20e
commit 6d7f9b5f3f
16 changed files with 999 additions and 339 deletions
@@ -0,0 +1,48 @@
+import assert from 'node:assert/strict'
+import { createServer } from 'node:http'
+import test from 'node:test'
+
+import { ObjectStorage } from './object-storage.js'
+
+test('S3 签名使用配置的对象存储 region', async () => {
+  const authorizationHeaders: string[] = []
+  const server = createServer((request, response) => {
+    const authorization = request.headers.authorization
+    if (authorization) authorizationHeaders.push(authorization)
+    response.statusCode = 200
+    response.end()
+  })
+  await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+
+  try {
+    const address = server.address()
+    assert.ok(address && typeof address === 'object')
+    const storage = new ObjectStorage({
+      endpoint: `http://127.0.0.1:${address.port}`,
+      bucket: 'order-site',
+      accessKeyId: 'test-access-key',
+      secretAccessKey: 'test-secret-key',
+      region: 'oss-cn-hangzhou',
+    })
+
+    await storage.putObject({
+      key: 'test/example.jpg',
+      content: Buffer.from('test'),
+      contentType: 'image/jpeg',
+    })
+
+    assert.ok(authorizationHeaders.length >= 2)
+    assert.match(
+      authorizationHeaders[0] || '',
+      /Credential=test-access-key\/\d{8}\/oss-cn-hangzhou\//,
+    )
+    assert.match(
+      authorizationHeaders.at(-1) || '',
+      /Credential=test-access-key\/\d{8}\/oss-cn-hangzhou\//,
+    )
+  } finally {
+    await new Promise<void>((resolve, reject) =>
+      server.close((error) => (error ? reject(error) : resolve())),
+    )
+  }
+})
@@ -1,11 +1,19 @@
 import { Readable } from 'node:stream'

-import * as Minio from 'minio'
+import {
+  CreateBucketCommand,
+  GetObjectCommand,
+  HeadBucketCommand,
+  HeadObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3'

 import { runtimeConfig } from '../../config/runtime.js'
 import { createHttpError } from '../../utils/http.js'

-type ObjectStorageConfig = {
+export type ObjectStorageConfig = {
  endpoint: string
  bucket: string
  accessKeyId: string
@@ -13,28 +21,50 @@ type ObjectStorageConfig = {
  region: string
 }

-type StoredObject = {
+export type StorageObjectSummary = {
+  key: string
+  size: number
+  etag: string
+  lastModified?: Date
+}
+
+export type StoredObjectStat = {
+  key: string
+  size: number
+  etag: string
+  contentType: string
+  lastModified?: Date
+}
+
+export type StoredObject = {
  reader: Readable
  contentType: string
  size: number
 }

 export class ObjectStorage {
-  private readonly client: Minio.Client
+  private readonly client: S3Client
  private readonly bucket: string
-  private readonly region: string
+  private readonly pathStyle: boolean
  private bucketReady = false
+  private bucketReadyPromise: Promise<void> | null = null

  constructor(config: ObjectStorageConfig) {
    const endpoint = normalizeStorageEndpoint(config.endpoint)
    this.bucket = String(config.bucket || '').trim()
-    this.region = String(config.region || 'us-east-1').trim() || 'us-east-1'
-    this.client = new Minio.Client({
-      endPoint: endpoint.endPoint,
-      port: endpoint.port,
-      useSSL: endpoint.useSSL,
-      accessKey: String(config.accessKeyId || '').trim(),
-      secretKey: String(config.secretAccessKey || '').trim(),
+    // 阿里云 OSS 强制虚拟主机风格（bucket.endpoint）；MinIO 等自建端点走 path-style。
+    this.pathStyle = !endpoint.endPoint.toLowerCase().endsWith('.aliyuncs.com')
+    this.client = new S3Client({
+      region: String(config.region || 'us-east-1').trim() || 'us-east-1',
+      endpoint: buildEndpointUrl(endpoint),
+      forcePathStyle: this.pathStyle,
+      credentials: {
+        accessKeyId: String(config.accessKeyId || '').trim(),
+        secretAccessKey: String(config.secretAccessKey || '').trim(),
+      },
+      // 默认的 CRC32 请求校验头不被 OSS 的 S3 兼容层支持，关闭以保持兼容。
+      requestChecksumCalculation: 'WHEN_REQUIRED',
+      responseChecksumValidation: 'WHEN_REQUIRED',
    })
  }

@@ -45,61 +75,197 @@ export class ObjectStorage {
    metadata?: Record<string, string>
  }): Promise<void> {
    await this.ensureBucketReady()
-    await this.client.putObject(this.bucket, input.key, input.content, input.content.length, {
-      'Content-Type': input.contentType,
-      ...sanitizeObjectMetadata(input.metadata),
-    })
+    await this.client.send(
+      new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: input.key,
+        Body: input.content,
+        ContentType: input.contentType,
+        ...(input.metadata ? { Metadata: sanitizeObjectMetadata(input.metadata) } : {}),
+      }),
+    )
  }

  async getObject(key: string): Promise<StoredObject> {
    await this.ensureBucketReady()
    const objectKey = normalizeObjectKey(key)
-    const stat = await this.client.statObject(this.bucket, objectKey)
-    const reader = await this.client.getObject(this.bucket, objectKey)
-    const metadata = stat.metaData || {}
-    const contentType = String(
-      metadata['content-type'] ||
-        metadata['Content-Type'] ||
-        metadata.contentType ||
-        'application/octet-stream',
+    const output = await this.client.send(
+      new GetObjectCommand({ Bucket: this.bucket, Key: objectKey }),
    )
+    const reader = output.Body as Readable
+    if (!reader) {
+      throw createHttpError('文件不存在或暂不可访问', {
+        statusCode: 404,
+        errorCode: 'file_not_found',
+      })
+    }

    return {
      reader,
-      contentType,
-      size: Number(stat.size || 0),
+      contentType: String(output.ContentType || 'application/octet-stream'),
+      size: Number(output.ContentLength || 0),
    }
  }

+  async statObject(key: string): Promise<StoredObjectStat> {
+    await this.ensureBucketReady()
+    const objectKey = normalizeObjectKey(key)
+    const output = await this.client.send(
+      new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey }),
+    )
+    const stat: StoredObjectStat = {
+      key: objectKey,
+      size: Number(output.ContentLength || 0),
+      etag: String(output.ETag || ''),
+      contentType: String(output.ContentType || 'application/octet-stream'),
+    }
+    if (output.LastModified) {
+      stat.lastModified = output.LastModified
+    }
+    return stat
+  }
+
+  async listObjects(): Promise<StorageObjectSummary[]> {
+    await this.ensureBucketReady()
+    const objects: StorageObjectSummary[] = []
+    let continuationToken: string | undefined
+    do {
+      const output = await this.client.send(
+        new ListObjectsV2Command({ Bucket: this.bucket, ContinuationToken: continuationToken }),
+      )
+      for (const item of output.Contents || []) {
+        if (!item.Key) {
+          continue
+        }
+        const summary: StorageObjectSummary = {
+          key: item.Key,
+          size: Number(item.Size || 0),
+          etag: String(item.ETag || ''),
+        }
+        if (item.LastModified) {
+          summary.lastModified = item.LastModified
+        }
+        objects.push(summary)
+      }
+      continuationToken = output.IsTruncated ? output.NextContinuationToken : undefined
+    } while (continuationToken)
+    return objects
+  }
+
  private async ensureBucketReady(): Promise<void> {
    if (this.bucketReady) {
      return
    }

-    const exists = await this.client.bucketExists(this.bucket)
-    if (!exists) {
-      await this.client.makeBucket(this.bucket, this.region)
+    if (!this.bucketReadyPromise) {
+      this.bucketReadyPromise = (async () => {
+        try {
+          await this.client.send(new HeadBucketCommand({ Bucket: this.bucket }))
+        } catch (error) {
+          if (!isNotFoundError(error)) {
+            throw error
+          }
+          if (!this.pathStyle) {
+            throw new Error(`OSS bucket ${this.bucket} 不存在，请先在阿里云控制台创建`)
+          }
+          await this.client.send(new CreateBucketCommand({ Bucket: this.bucket }))
+        }
+        this.bucketReady = true
+      })()
+    }
+    try {
+      await this.bucketReadyPromise
+    } catch (error) {
+      this.bucketReadyPromise = null
+      throw error
    }
-    this.bucketReady = true
  }
 }

-let storageInstance: ObjectStorage | null = null
+export class StorageRouter {
+  private readonly mode: 'minio' | 'dual' | 'oss'
+  private readonly legacy: ObjectStorage
+  private readonly oss: ObjectStorage | null

-export function getObjectStorage(): ObjectStorage {
+  constructor(mode: 'minio' | 'dual' | 'oss', legacy: ObjectStorage, oss: ObjectStorage | null) {
+    this.mode = mode
+    this.legacy = legacy
+    this.oss = oss
+    if ((mode === 'dual' || mode === 'oss') && !oss) {
+      throw new Error(`存储模式 ${mode} 缺少 OSS 配置`)
+    }
+  }
+
+  async putObject(input: Parameters<ObjectStorage['putObject']>[0]): Promise<void> {
+    if (this.mode === 'minio') {
+      await this.legacy.putObject(input)
+      return
+    }
+    if (this.mode === 'oss') {
+      await this.oss!.putObject(input)
+      return
+    }
+    // 双写要求两个后端都成功，避免数据库记录指向未完成迁移的对象。
+    await Promise.all([this.legacy.putObject(input), this.oss!.putObject(input)])
+  }
+
+  async getObject(key: string): Promise<StoredObject> {
+    if (this.mode === 'minio') {
+      return this.legacy.getObject(key)
+    }
+    if (this.mode === 'oss') {
+      return this.oss!.getObject(key)
+    }
+    try {
+      return await this.legacy.getObject(key)
+    } catch {
+      return this.oss!.getObject(key)
+    }
+  }
+}
+
+let storageInstance: StorageRouter | null = null
+
+export function getObjectStorage(): StorageRouter {
  if (!storageInstance) {
-    storageInstance = new ObjectStorage({
-      endpoint: runtimeConfig.storage.endpoint,
-      bucket: runtimeConfig.storage.bucket,
-      accessKeyId: runtimeConfig.storage.accessKeyId,
-      secretAccessKey: runtimeConfig.storage.secretAccessKey,
-      region: runtimeConfig.storage.region,
-    })
+    const mode = normalizeStorageMode(runtimeConfig.storage.mode)
+    const legacy = new ObjectStorage(toObjectStorageConfig(runtimeConfig.storage))
+    const oss =
+      mode === 'minio' ? null : new ObjectStorage(toObjectStorageConfig(runtimeConfig.storage.oss))
+    storageInstance = new StorageRouter(mode, legacy, oss)
  }

  return storageInstance
 }

+export function createConfiguredObjectStorage(config: ObjectStorageConfig): ObjectStorage {
+  return new ObjectStorage(config)
+}
+
+function normalizeStorageMode(value: unknown): 'minio' | 'dual' | 'oss' {
+  const mode = String(value || 'minio')
+    .trim()
+    .toLowerCase()
+  if (mode === 'dual' || mode === 'oss') return mode
+  return 'minio'
+}
+
+function toObjectStorageConfig(config: {
+  endpoint: string
+  bucket: string
+  accessKeyId: string
+  secretAccessKey: string
+  region: string
+}): ObjectStorageConfig {
+  return {
+    endpoint: config.endpoint,
+    bucket: config.bucket,
+    accessKeyId: config.accessKeyId,
+    secretAccessKey: config.secretAccessKey,
+    region: config.region,
+  }
+}
+
 export function normalizeStorageEndpoint(rawEndpoint: unknown) {
  const endpoint = String(rawEndpoint || '').trim()
  const parsed = new URL(/^https?:\/\//i.test(endpoint) ? endpoint : `http://${endpoint}`)
@@ -112,7 +278,26 @@ export function normalizeStorageEndpoint(rawEndpoint: unknown) {
  }
 }

-export function normalizeObjectKey(rawKey: unknown): string {
+function buildEndpointUrl(endpoint: { endPoint: string; port: number; useSSL: boolean }): string {
+  const scheme = endpoint.useSSL ? 'https' : 'http'
+  const defaultPort = endpoint.useSSL ? 443 : 80
+  const suffix = endpoint.port && endpoint.port !== defaultPort ? `:${endpoint.port}` : ''
+  return `${scheme}://${endpoint.endPoint}${suffix}`
+}
+
+function isNotFoundError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') {
+    return false
+  }
+  const candidate = error as { name?: string; $metadata?: { httpStatusCode?: number } }
+  return (
+    candidate.name === 'NotFound' ||
+    candidate.name === 'NoSuchBucket' ||
+    candidate.$metadata?.httpStatusCode === 404
+  )
+}
+
+export function normalizeObjectKey(rawKey: unknown) {
  const key = String(rawKey || '')
    .trim()
    .replace(/^\/+/, '')