218 lines
8.1 KiB
TypeScript
218 lines
8.1 KiB
TypeScript
#!/usr/bin/env npx tsx
|
||
/**
|
||
* 本地存储 → MinIO 迁移脚本
|
||
* 使用 @aws-sdk/client-s3（项目已有依赖）
|
||
*
|
||
* 用法: npx tsx scripts/migrate-local-to-minio.ts
|
||
*/
|
||
|
||
import { S3Client, PutObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'
|
||
import * as fs from 'fs/promises'
|
||
import * as path from 'path'
|
||
import { createReadStream } from 'fs'
|
||
|
||
// ==================== 配置 ====================
|
||
const LOCAL_DIR = process.env.LOCAL_UPLOAD_DIR || './data/uploads'
|
||
const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT || 'http://127.0.0.1:19000'
|
||
const MINIO_BUCKET = process.env.MINIO_BUCKET || 'waoowaoo'
|
||
const MINIO_REGION = process.env.MINIO_REGION || 'us-east-1'
|
||
const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY || 'minioadmin'
|
||
const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY || 'minioadmin'
|
||
const CONCURRENCY = parseInt(process.env.MIGRATE_CONCURRENCY || '10')
|
||
const DRY_RUN = process.env.MIGRATE_DRY_RUN === 'true'
|
||
|
||
// ==================== S3 客户端 ====================
|
||
const s3 = new S3Client({
|
||
endpoint: MINIO_ENDPOINT,
|
||
region: MINIO_REGION,
|
||
forcePathStyle: true,
|
||
credentials: {
|
||
accessKeyId: MINIO_ACCESS_KEY,
|
||
secretAccessKey: MINIO_SECRET_KEY,
|
||
},
|
||
})
|
||
|
||
// ==================== 工具函数 ====================
|
||
function guessContentType(filename: string): string {
|
||
const ext = path.extname(filename).toLowerCase()
|
||
const types: Record<string, string> = {
|
||
'.jpg': 'image/jpeg',
|
||
'.jpeg': 'image/jpeg',
|
||
'.png': 'image/png',
|
||
'.gif': 'image/gif',
|
||
'.webp': 'image/webp',
|
||
'.mp4': 'video/mp4',
|
||
'.webm': 'video/webm',
|
||
'.mp3': 'audio/mpeg',
|
||
'.wav': 'audio/wav',
|
||
'.ogg': 'audio/ogg',
|
||
'.json': 'application/json',
|
||
'.txt': 'text/plain',
|
||
}
|
||
return types[ext] || 'application/octet-stream'
|
||
}
|
||
|
||
function formatBytes(bytes: number): string {
|
||
if (bytes === 0) return '0 B'
|
||
const k = 1024
|
||
const sizes = ['B', 'KB', 'MB', 'GB']
|
||
const i = Math.floor(Math.log(bytes) / Math.log(k))
|
||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]
|
||
}
|
||
|
||
// ==================== 扫描本地文件 ====================
|
||
async function scanLocalFiles(dir: string, basePath = ''): Promise<Array<{ localPath: string; key: string; size: number }>> {
|
||
const files: Array<{ localPath: string; key: string; size: number }> = []
|
||
|
||
try {
|
||
const entries = await fs.readdir(dir, { withFileTypes: true })
|
||
for (const entry of entries) {
|
||
const fullPath = path.join(dir, entry.name)
|
||
const relativePath = path.join(basePath, entry.name)
|
||
|
||
if (entry.isDirectory()) {
|
||
files.push(...await scanLocalFiles(fullPath, relativePath))
|
||
} else {
|
||
// 跳过隐藏文件
|
||
if (entry.name.startsWith('.')) continue
|
||
const stats = await fs.stat(fullPath)
|
||
files.push({
|
||
localPath: fullPath,
|
||
key: relativePath.replace(/\\/g, '/'),
|
||
size: stats.size,
|
||
})
|
||
}
|
||
}
|
||
} catch (err: unknown) {
|
||
console.error(` ⚠️ 无法读取目录: ${dir}`, (err as Error).message)
|
||
}
|
||
|
||
return files
|
||
}
|
||
|
||
// ==================== 检查文件是否已存在 ====================
|
||
async function objectExists(key: string): Promise<boolean> {
|
||
try {
|
||
await s3.send(new HeadObjectCommand({ Bucket: MINIO_BUCKET, Key: key }))
|
||
return true
|
||
} catch {
|
||
return false
|
||
}
|
||
}
|
||
|
||
// ==================== 上传文件 ====================
|
||
async function uploadFile(file: { localPath: string; key: string; size: number }): Promise<'success' | 'skipped' | 'error'> {
|
||
// 检查是否已存在
|
||
if (await objectExists(file.key)) {
|
||
return 'skipped'
|
||
}
|
||
|
||
if (DRY_RUN) {
|
||
console.log(` [DRY RUN] 将上传: ${file.key} (${formatBytes(file.size)})`)
|
||
return 'skipped'
|
||
}
|
||
|
||
try {
|
||
const body = await fs.readFile(file.localPath)
|
||
await s3.send(new PutObjectCommand({
|
||
Bucket: MINIO_BUCKET,
|
||
Key: file.key,
|
||
Body: body,
|
||
ContentType: guessContentType(file.key),
|
||
}))
|
||
return 'success'
|
||
} catch (err: unknown) {
|
||
console.error(` ✗ 上传失败: ${file.key}`, (err as Error).message)
|
||
return 'error'
|
||
}
|
||
}
|
||
|
||
// ==================== 并行控制 ====================
|
||
async function runBatched<T>(items: T[], concurrency: number, fn: (item: T) => Promise<void>) {
|
||
for (let i = 0; i < items.length; i += concurrency) {
|
||
const batch = items.slice(i, i + concurrency)
|
||
await Promise.all(batch.map(fn))
|
||
}
|
||
}
|
||
|
||
// ==================== 主流程 ====================
|
||
async function main() {
|
||
console.log()
|
||
console.log('╔══════════════════════════════════════════════════════╗')
|
||
console.log('║ Local Storage → MinIO Migration Tool ║')
|
||
console.log('╚══════════════════════════════════════════════════════╝')
|
||
console.log()
|
||
console.log(` 📂 源目录: ${path.resolve(LOCAL_DIR)}`)
|
||
console.log(` 🪣 目标桶: ${MINIO_ENDPOINT}/${MINIO_BUCKET}`)
|
||
console.log(` ⚡ 并发数: ${CONCURRENCY}`)
|
||
console.log(` 🔍 干运行: ${DRY_RUN}`)
|
||
console.log()
|
||
|
||
// 1. 扫描文件
|
||
console.log('📦 扫描本地文件...')
|
||
const files = await scanLocalFiles(LOCAL_DIR)
|
||
|
||
if (files.length === 0) {
|
||
console.log(' 没有需要迁移的文件')
|
||
return
|
||
}
|
||
|
||
const totalSize = files.reduce((sum, f) => sum + f.size, 0)
|
||
console.log(` 找到 ${files.length} 个文件, 总大小: ${formatBytes(totalSize)}`)
|
||
console.log()
|
||
|
||
// 2. 开始上传
|
||
console.log('🚀 开始迁移...')
|
||
const startTime = Date.now()
|
||
let success = 0
|
||
let skipped = 0
|
||
let failed = 0
|
||
let processed = 0
|
||
|
||
await runBatched(files, CONCURRENCY, async (file) => {
|
||
const result = await uploadFile(file)
|
||
processed++
|
||
|
||
if (result === 'success') {
|
||
success++
|
||
if (success % 50 === 0 || success <= 5) {
|
||
console.log(` ✓ [${processed}/${files.length}] ${file.key} (${formatBytes(file.size)})`)
|
||
}
|
||
} else if (result === 'skipped') {
|
||
skipped++
|
||
} else {
|
||
failed++
|
||
}
|
||
|
||
if (processed % 100 === 0) {
|
||
const pct = ((processed / files.length) * 100).toFixed(1)
|
||
console.log(` 📊 进度: ${pct}% (${processed}/${files.length}) | ✓${success} ⏭${skipped} ✗${failed}`)
|
||
}
|
||
})
|
||
|
||
// 3. 结果
|
||
const duration = ((Date.now() - startTime) / 1000).toFixed(1)
|
||
console.log()
|
||
console.log('╔══════════════════════════════════════════════════════╗')
|
||
console.log('║ 迁移完成 ║')
|
||
console.log('╠══════════════════════════════════════════════════════╣')
|
||
console.log(`║ 总文件: ${String(files.length).padEnd(40)} ║`)
|
||
console.log(`║ 成功: ${String(success).padEnd(40)} ║`)
|
||
console.log(`║ 跳过: ${String(skipped).padEnd(40)} ║`)
|
||
console.log(`║ 失败: ${String(failed).padEnd(40)} ║`)
|
||
console.log(`║ 耗时: ${String(duration + 's').padEnd(40)} ║`)
|
||
console.log(`║ 大小: ${formatBytes(totalSize).padEnd(40)} ║`)
|
||
console.log('╚══════════════════════════════════════════════════════╝')
|
||
|
||
if (failed > 0) {
|
||
console.log()
|
||
console.log('⚠️ 有文件上传失败，请重新运行脚本（已上传的会自动跳过）')
|
||
process.exit(1)
|
||
}
|
||
}
|
||
|
||
main().catch(err => {
|
||
console.error('迁移失败:', err)
|
||
process.exit(1)
|
||
})
|