yml/order_site
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

准备迁移1

Browse Source
This commit is contained in:
yml2213
2026-08-23 12:54:38 +08:00
parent 360c5ea20e
commit 6d7f9b5f3f
16 changed files with 999 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/src/config/defaults.ts
+8
View File
@@ -32,12 +32,20 @@ export function createDefaultRuntimeConfig(projectRoot: string): RuntimeConfig {
},
storage: {
mode: 'minio',
endpoint: 'http://127.0.0.1:9000',
bucket: 'order-site',
accessKeyId: 'minioadmin',
secretAccessKey: 'minioadmin',
region: 'us-east-1',
maxUploadSizeMb: 10,
oss: {
endpoint: '',
bucket: '',
accessKeyId: '',
secretAccessKey: '',
region: 'oss-cn-hangzhou',
},
},
orders: {
apps/backend/src/config/env-overrides.ts
+6
View File
@@ -40,12 +40,18 @@ export const ENV_OVERRIDES: readonly EnvOverride[] = [
integerEnv('DATABASE_CONNECTION_TIMEOUT_MS', ['database', 'connectionTimeoutMs']),
integerEnv('DATABASE_STATEMENT_TIMEOUT_MS', ['database', 'statementTimeoutMs']),
integerEnv('DATABASE_SLOW_QUERY_THRESHOLD_MS', ['database', 'slowQueryThresholdMs']),
stringEnv('STORAGE_MODE', ['storage', 'mode']),
stringEnv('STORAGE_ENDPOINT', ['storage', 'endpoint']),
stringEnv('STORAGE_BUCKET', ['storage', 'bucket']),
stringEnv('STORAGE_ACCESS_KEY_ID', ['storage', 'accessKeyId']),
stringEnv('STORAGE_SECRET_ACCESS_KEY', ['storage', 'secretAccessKey']),
stringEnv('STORAGE_REGION', ['storage', 'region']),
integerEnv('STORAGE_MAX_UPLOAD_SIZE_MB', ['storage', 'maxUploadSizeMb']),
stringEnv('STORAGE_OSS_ENDPOINT', ['storage', 'oss', 'endpoint']),
stringEnv('STORAGE_OSS_BUCKET', ['storage', 'oss', 'bucket']),
stringEnv('STORAGE_OSS_ACCESS_KEY_ID', ['storage', 'oss', 'accessKeyId']),
stringEnv('STORAGE_OSS_SECRET_ACCESS_KEY', ['storage', 'oss', 'secretAccessKey']),
stringEnv('STORAGE_OSS_REGION', ['storage', 'oss', 'region']),
integerEnv('CLAIM_TOKEN_TTL_HOURS', ['orders', 'tokenTtlHours']),
stringEnv('ADMIN_SESSION_SECRET', ['admin', 'sessionSecret']),
integerEnv('ADMIN_SESSION_TTL_HOURS', ['admin', 'sessionTtlHours']),
apps/backend/src/config/runtime-validation.ts
+43 -7
View File
@@ -79,6 +79,13 @@ export function validateRuntimeConfig(
min: 1,
max: 100,
})
const storageMode = normalizeStorageMode(config.storage?.mode)
if (!storageMode) {
issues.push({
path: 'storage.mode',
message: '仅支持 minio、dual 或 oss',
})
}
requireInteger(issues, 'orders.tokenTtlHours', config.orders?.tokenTtlHours, { min: 0 })
requireInteger(issues, 'admin.sessionTtlHours', config.admin?.sessionTtlHours, { min: 1 })
requireInteger(
@@ -116,17 +123,17 @@ export function validateRuntimeConfig(
validateOptionalHttpUrl(issues, 'orders.claimBaseUrl', config.orders?.claimBaseUrl)
validateOptionalStorageEndpoint(issues, 'storage.endpoint', config.storage?.endpoint)
validateOptionalStorageEndpoint(issues, 'storage.oss.endpoint', config.storage?.oss?.endpoint)
validateRequiredString(issues, 'data.root', config.data?.root)
if (productionLike) {
validateRequiredString(issues, 'database.url', config.database?.url)
validateRequiredString(issues, 'storage.endpoint', config.storage?.endpoint)
validateOptionalStorageEndpoint(issues, 'storage.endpoint', config.storage?.endpoint)
validateRequiredString(issues, 'storage.bucket', config.storage?.bucket)
validateRequiredString(issues, 'storage.accessKeyId', config.storage?.accessKeyId)
validateSecret(issues, 'storage.secretAccessKey', config.storage?.secretAccessKey, {
minLength: 8,
})
if (storageMode === 'minio' || storageMode === 'dual') {
validateStorageCredentials(issues, 'storage', config.storage)
}
if (storageMode === 'oss' || storageMode === 'dual') {
validateStorageCredentials(issues, 'storage.oss', config.storage?.oss)
}
validateRequiredHttpUrl(issues, 'orders.claimBaseUrl', config.orders?.claimBaseUrl)
validateSecret(issues, 'admin.sessionSecret', config.admin?.sessionSecret, { minLength: 32 })
validateAdminDefaultUsers(issues, config.admin?.defaultUsers)
@@ -135,6 +142,35 @@ export function validateRuntimeConfig(
return issues
}
function normalizeStorageMode(value: unknown): 'minio' | 'dual' | 'oss' | '' {
const mode = String(value || 'minio')
.trim()
.toLowerCase()
return mode === 'minio' || mode === 'dual' || mode === 'oss' ? mode : ''
}
function validateStorageCredentials(
issues: RuntimeConfigValidationIssue[],
configPath: string,
storage:
| {
endpoint?: unknown
bucket?: unknown
accessKeyId?: unknown
secretAccessKey?: unknown
}
| null
| undefined,
): void {
validateRequiredString(issues, `${configPath}.endpoint`, storage?.endpoint)
validateOptionalStorageEndpoint(issues, `${configPath}.endpoint`, storage?.endpoint)
validateRequiredString(issues, `${configPath}.bucket`, storage?.bucket)
validateRequiredString(issues, `${configPath}.accessKeyId`, storage?.accessKeyId)
validateSecret(issues, `${configPath}.secretAccessKey`, storage?.secretAccessKey, {
minLength: 8,
})
}
export function isProductionLike(env: RuntimeEnvironment = process.env): boolean {
return env.NODE_ENV === 'production'
}
apps/backend/src/services/file-storage/object-storage.test.ts
+48
View File
@@ -0,0 +1,48 @@
import assert from 'node:assert/strict'
import { createServer } from 'node:http'
import test from 'node:test'
import { ObjectStorage } from './object-storage.js'
test('S3 签名使用配置的对象存储 region', async () => {
const authorizationHeaders: string[] = []
const server = createServer((request, response) => {
const authorization = request.headers.authorization
if (authorization) authorizationHeaders.push(authorization)
response.statusCode = 200
response.end()
})
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
try {
const address = server.address()
assert.ok(address && typeof address === 'object')
const storage = new ObjectStorage({
endpoint: `http://127.0.0.1:${address.port}`,
bucket: 'order-site',
accessKeyId: 'test-access-key',
secretAccessKey: 'test-secret-key',
region: 'oss-cn-hangzhou',
})
await storage.putObject({
key: 'test/example.jpg',
content: Buffer.from('test'),
contentType: 'image/jpeg',
})
assert.ok(authorizationHeaders.length >= 2)
assert.match(
authorizationHeaders[0] || '',
/Credential=test-access-key\/\d{8}\/oss-cn-hangzhou\//,
)
assert.match(
authorizationHeaders.at(-1) || '',
/Credential=test-access-key\/\d{8}\/oss-cn-hangzhou\//,
)
} finally {
await new Promise<void>((resolve, reject) =>
server.close((error) => (error ? reject(error) : resolve())),
)
}
})
apps/backend/src/services/file-storage/object-storage.ts
+225 -40
View File
@@ -1,11 +1,19 @@
import { Readable } from 'node:stream'
import * as Minio from 'minio'
import {
CreateBucketCommand,
GetObjectCommand,
HeadBucketCommand,
HeadObjectCommand,
ListObjectsV2Command,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3'
import { runtimeConfig } from '../../config/runtime.js'
import { createHttpError } from '../../utils/http.js'
type ObjectStorageConfig = {
export type ObjectStorageConfig = {
endpoint: string
bucket: string
accessKeyId: string
@@ -13,28 +21,50 @@ type ObjectStorageConfig = {
region: string
}
type StoredObject = {
export type StorageObjectSummary = {
key: string
size: number
etag: string
lastModified?: Date
}
export type StoredObjectStat = {
key: string
size: number
etag: string
contentType: string
lastModified?: Date
}
export type StoredObject = {
reader: Readable
contentType: string
size: number
}
export class ObjectStorage {
private readonly client: Minio.Client
private readonly client: S3Client
private readonly bucket: string
private readonly region: string
private readonly pathStyle: boolean
private bucketReady = false
private bucketReadyPromise: Promise<void> | null = null
constructor(config: ObjectStorageConfig) {
const endpoint = normalizeStorageEndpoint(config.endpoint)
this.bucket = String(config.bucket || '').trim()
this.region = String(config.region || 'us-east-1').trim() || 'us-east-1'
this.client = new Minio.Client({
endPoint: endpoint.endPoint,
port: endpoint.port,
useSSL: endpoint.useSSL,
accessKey: String(config.accessKeyId || '').trim(),
secretKey: String(config.secretAccessKey || '').trim(),
// 阿里云 OSS 强制虚拟主机风格（bucket.endpoint）；MinIO 等自建端点走 path-style。
this.pathStyle = !endpoint.endPoint.toLowerCase().endsWith('.aliyuncs.com')
this.client = new S3Client({
region: String(config.region || 'us-east-1').trim() || 'us-east-1',
endpoint: buildEndpointUrl(endpoint),
forcePathStyle: this.pathStyle,
credentials: {
accessKeyId: String(config.accessKeyId || '').trim(),
secretAccessKey: String(config.secretAccessKey || '').trim(),
},
// 默认的 CRC32 请求校验头不被 OSS 的 S3 兼容层支持，关闭以保持兼容。
requestChecksumCalculation: 'WHEN_REQUIRED',
responseChecksumValidation: 'WHEN_REQUIRED',
})
}
@@ -45,61 +75,197 @@ export class ObjectStorage {
metadata?: Record<string, string>
}): Promise<void> {
await this.ensureBucketReady()
await this.client.putObject(this.bucket, input.key, input.content, input.content.length, {
'Content-Type': input.contentType,
...sanitizeObjectMetadata(input.metadata),
})
await this.client.send(
new PutObjectCommand({
Bucket: this.bucket,
Key: input.key,
Body: input.content,
ContentType: input.contentType,
...(input.metadata ? { Metadata: sanitizeObjectMetadata(input.metadata) } : {}),
}),
)
}
async getObject(key: string): Promise<StoredObject> {
await this.ensureBucketReady()
const objectKey = normalizeObjectKey(key)
const stat = await this.client.statObject(this.bucket, objectKey)
const reader = await this.client.getObject(this.bucket, objectKey)
const metadata = stat.metaData || {}
const contentType = String(
metadata['content-type'] ||
metadata['Content-Type'] ||
metadata.contentType ||
'application/octet-stream',
const output = await this.client.send(
new GetObjectCommand({ Bucket: this.bucket, Key: objectKey }),
)
const reader = output.Body as Readable
if (!reader) {
throw createHttpError('文件不存在或暂不可访问', {
statusCode: 404,
errorCode: 'file_not_found',
})
}
return {
reader,
contentType,
size: Number(stat.size || 0),
contentType: String(output.ContentType || 'application/octet-stream'),
size: Number(output.ContentLength || 0),
}
}
async statObject(key: string): Promise<StoredObjectStat> {
await this.ensureBucketReady()
const objectKey = normalizeObjectKey(key)
const output = await this.client.send(
new HeadObjectCommand({ Bucket: this.bucket, Key: objectKey }),
)
const stat: StoredObjectStat = {
key: objectKey,
size: Number(output.ContentLength || 0),
etag: String(output.ETag || ''),
contentType: String(output.ContentType || 'application/octet-stream'),
}
if (output.LastModified) {
stat.lastModified = output.LastModified
}
return stat
}
async listObjects(): Promise<StorageObjectSummary[]> {
await this.ensureBucketReady()
const objects: StorageObjectSummary[] = []
let continuationToken: string | undefined
do {
const output = await this.client.send(
new ListObjectsV2Command({ Bucket: this.bucket, ContinuationToken: continuationToken }),
)
for (const item of output.Contents || []) {
if (!item.Key) {
continue
}
const summary: StorageObjectSummary = {
key: item.Key,
size: Number(item.Size || 0),
etag: String(item.ETag || ''),
}
if (item.LastModified) {
summary.lastModified = item.LastModified
}
objects.push(summary)
}
continuationToken = output.IsTruncated ? output.NextContinuationToken : undefined
} while (continuationToken)
return objects
}
private async ensureBucketReady(): Promise<void> {
if (this.bucketReady) {
return
}
const exists = await this.client.bucketExists(this.bucket)
if (!exists) {
await this.client.makeBucket(this.bucket, this.region)
if (!this.bucketReadyPromise) {
this.bucketReadyPromise = (async () => {
try {
await this.client.send(new HeadBucketCommand({ Bucket: this.bucket }))
} catch (error) {
if (!isNotFoundError(error)) {
throw error
}
if (!this.pathStyle) {
throw new Error(`OSS bucket ${this.bucket} 不存在，请先在阿里云控制台创建`)
}
await this.client.send(new CreateBucketCommand({ Bucket: this.bucket }))
}
this.bucketReady = true
})()
}
try {
await this.bucketReadyPromise
} catch (error) {
this.bucketReadyPromise = null
throw error
}
this.bucketReady = true
}
}
let storageInstance: ObjectStorage | null = null
export class StorageRouter {
private readonly mode: 'minio' | 'dual' | 'oss'
private readonly legacy: ObjectStorage
private readonly oss: ObjectStorage | null
export function getObjectStorage(): ObjectStorage {
constructor(mode: 'minio' | 'dual' | 'oss', legacy: ObjectStorage, oss: ObjectStorage | null) {
this.mode = mode
this.legacy = legacy
this.oss = oss
if ((mode === 'dual' || mode === 'oss') && !oss) {
throw new Error(`存储模式 ${mode} 缺少 OSS 配置`)
}
}
async putObject(input: Parameters<ObjectStorage['putObject']>[0]): Promise<void> {
if (this.mode === 'minio') {
await this.legacy.putObject(input)
return
}
if (this.mode === 'oss') {
await this.oss!.putObject(input)
return
}
// 双写要求两个后端都成功，避免数据库记录指向未完成迁移的对象。
await Promise.all([this.legacy.putObject(input), this.oss!.putObject(input)])
}
async getObject(key: string): Promise<StoredObject> {
if (this.mode === 'minio') {
return this.legacy.getObject(key)
}
if (this.mode === 'oss') {
return this.oss!.getObject(key)
}
try {
return await this.legacy.getObject(key)
} catch {
return this.oss!.getObject(key)
}
}
}
let storageInstance: StorageRouter | null = null
export function getObjectStorage(): StorageRouter {
if (!storageInstance) {
storageInstance = new ObjectStorage({
endpoint: runtimeConfig.storage.endpoint,
bucket: runtimeConfig.storage.bucket,
accessKeyId: runtimeConfig.storage.accessKeyId,
secretAccessKey: runtimeConfig.storage.secretAccessKey,
region: runtimeConfig.storage.region,
})
const mode = normalizeStorageMode(runtimeConfig.storage.mode)
const legacy = new ObjectStorage(toObjectStorageConfig(runtimeConfig.storage))
const oss =
mode === 'minio' ? null : new ObjectStorage(toObjectStorageConfig(runtimeConfig.storage.oss))
storageInstance = new StorageRouter(mode, legacy, oss)
}
return storageInstance
}
export function createConfiguredObjectStorage(config: ObjectStorageConfig): ObjectStorage {
return new ObjectStorage(config)
}
function normalizeStorageMode(value: unknown): 'minio' | 'dual' | 'oss' {
const mode = String(value || 'minio')
.trim()
.toLowerCase()
if (mode === 'dual' || mode === 'oss') return mode
return 'minio'
}
function toObjectStorageConfig(config: {
endpoint: string
bucket: string
accessKeyId: string
secretAccessKey: string
region: string
}): ObjectStorageConfig {
return {
endpoint: config.endpoint,
bucket: config.bucket,
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
region: config.region,
}
}
export function normalizeStorageEndpoint(rawEndpoint: unknown) {
const endpoint = String(rawEndpoint || '').trim()
const parsed = new URL(/^https?:\/\//i.test(endpoint) ? endpoint : `http://${endpoint}`)
@@ -112,7 +278,26 @@ export function normalizeStorageEndpoint(rawEndpoint: unknown) {
}
}
export function normalizeObjectKey(rawKey: unknown): string {
function buildEndpointUrl(endpoint: { endPoint: string; port: number; useSSL: boolean }): string {
const scheme = endpoint.useSSL ? 'https' : 'http'
const defaultPort = endpoint.useSSL ? 443 : 80
const suffix = endpoint.port && endpoint.port !== defaultPort ? `:${endpoint.port}` : ''
return `${scheme}://${endpoint.endPoint}${suffix}`
}
function isNotFoundError(error: unknown): boolean {
if (!error || typeof error !== 'object') {
return false
}
const candidate = error as { name?: string; $metadata?: { httpStatusCode?: number } }
return (
candidate.name === 'NotFound' ||
candidate.name === 'NoSuchBucket' ||
candidate.$metadata?.httpStatusCode === 404
)
}
export function normalizeObjectKey(rawKey: unknown) {
const key = String(rawKey || '')
.trim()
.replace(/^\/+/, '')
apps/backend/src/storage-migrate.ts
+176
View File
@@ -0,0 +1,176 @@
import crypto from 'node:crypto'
import process from 'node:process'
import {
createConfiguredObjectStorage,
type ObjectStorage,
} from './services/file-storage/object-storage.js'
const command = process.argv[2] || ''
const concurrency = readConcurrency()
if (!['copy', 'verify'].includes(command)) {
console.error('用法：node dist/storage-migrate.js copy|verify')
process.exit(2)
}
const source = createConfiguredObjectStorage(readStorageConfig('STORAGE'))
const destination = createConfiguredObjectStorage(readStorageConfig('STORAGE_OSS'))
if (command === 'copy') {
await copyObjects(source, destination)
} else {
await verifyObjects(source, destination)
}
function readStorageConfig(prefix: 'STORAGE' | 'STORAGE_OSS') {
return {
endpoint: required(`${prefix}_ENDPOINT`),
bucket: required(`${prefix}_BUCKET`),
accessKeyId: required(`${prefix}_ACCESS_KEY_ID`),
secretAccessKey: required(`${prefix}_SECRET_ACCESS_KEY`),
region: process.env[`${prefix}_REGION`] || 'us-east-1',
}
}
async function copyObjects(sourceStorage: ObjectStorage, destinationStorage: ObjectStorage) {
const objects = await sourceStorage.listObjects()
console.info(`源存储对象数：${objects.length}`)
let copied = 0
let skipped = 0
let completed = 0
await runConcurrent(objects, concurrency, async (object) => {
const existing = await readStat(destinationStorage, object.key)
const sourceEtag = normalizeEtag(object.etag)
const destinationEtag = normalizeEtag(existing?.etag)
if (
existing &&
Number(existing.size || 0) === object.size &&
sourceEtag &&
destinationEtag &&
destinationEtag === sourceEtag
) {
skipped += 1
} else {
const stored = await sourceStorage.getObject(object.key)
const content = await readBuffer(stored.reader)
await destinationStorage.putObject({
key: object.key,
content,
contentType: stored.contentType,
})
copied += 1
}
completed += 1
logProgress('复制', completed, objects.length)
})
console.info(`复制完成：新增/覆盖 ${copied}，已存在且大小、ETag 一致 ${skipped}`)
}
async function verifyObjects(sourceStorage: ObjectStorage, destinationStorage: ObjectStorage) {
const sourceObjects = await sourceStorage.listObjects()
const destinationObjects = await destinationStorage.listObjects()
const destinationByKey = new Map(destinationObjects.map((item) => [item.key, item]))
const missing = sourceObjects.filter((item) => !destinationByKey.has(item.key))
const mismatches: string[] = []
let completed = 0
await runConcurrent(sourceObjects, concurrency, async (object) => {
const destination = destinationByKey.get(object.key)
if (!destination) {
completed += 1
logProgress('校验', completed, sourceObjects.length)
return
}
if (Number(destination.size || 0) !== object.size) {
mismatches.push(object.key)
completed += 1
logProgress('校验', completed, sourceObjects.length)
return
}
const sourceStored = await sourceStorage.getObject(object.key)
const destinationStored = await destinationStorage.getObject(object.key)
const [sourceHash, destinationHash] = await Promise.all([
hashStream(sourceStored.reader),
hashStream(destinationStored.reader),
])
if (
sourceHash !== destinationHash ||
sourceStored.contentType !== destinationStored.contentType ||
sourceStored.size !== destinationStored.size
) {
mismatches.push(object.key)
}
completed += 1
logProgress('校验', completed, sourceObjects.length)
})
if (missing.length || mismatches.length) {
console.error(`校验失败：缺失 ${missing.length}，内容不一致 ${mismatches.length}`)
if (missing.length) console.error(`缺失对象：${missing.slice(0, 20).join(', ')}`)
if (mismatches.length) console.error(`不一致对象：${mismatches.slice(0, 20).join(', ')}`)
process.exit(1)
}
console.info(`校验通过：${sourceObjects.length} 个源对象全部存在且内容一致`)
}
async function runConcurrent<T>(
items: T[],
limit: number,
worker: (item: T) => Promise<void>,
): Promise<void> {
let nextIndex = 0
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
while (true) {
const index = nextIndex++
const item = items[index]
if (item === undefined) return
await worker(item)
}
})
await Promise.all(workers)
}
function normalizeEtag(value: unknown): string {
return String(value || '')
.trim()
.replace(/^"|"$/g, '')
.toLowerCase()
}
function logProgress(label: string, completed: number, total: number): void {
if (completed === total || completed % 100 === 0) {
console.info(`${label}进度：${completed}/${total}（并发 ${concurrency}）`)
}
}
async function readStat(storage: ObjectStorage, key: string) {
try {
return await storage.statObject(key)
} catch {
return null
}
}
async function readBuffer(reader: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = []
for await (const chunk of reader) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
return Buffer.concat(chunks)
}
async function hashStream(reader: NodeJS.ReadableStream): Promise<string> {
const hash = crypto.createHash('sha256')
for await (const chunk of reader) hash.update(chunk)
return hash.digest('hex')
}
function required(name: string): string {
const value = String(process.env[name] || '').trim()
if (!value) {
console.error(`缺少环境变量：${name}`)
process.exit(2)
}
return value
}
function readConcurrency(): number {
const value = Number(process.env.STORAGE_MIGRATION_CONCURRENCY || 8)
return Number.isInteger(value) && value > 0 ? Math.min(value, 32) : 8
}
apps/backend/src/types/runtime-config.ts
+8
View File
@@ -43,12 +43,20 @@ export type RuntimeConfig = {
slowQueryThresholdMs?: number
}
storage: {
mode: 'minio' | 'dual' | 'oss'
endpoint: string
bucket: string
accessKeyId: string
secretAccessKey: string
region: string
maxUploadSizeMb: number
oss: {
endpoint: string
bucket: string
accessKeyId: string
secretAccessKey: string
region: string
}
}
orders: {
claimBaseUrl: string