gongxiangp2/scripts/migrate-audio-source-to-local.ts

271 lines
8.1 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* 音乐源文件迁回本地脚本
*
* 目标:将旧数据中存放在 MinIO(S3) 上的音乐源文件(zipFileName 字段),下载到本地 uploads 目录,
* 并把 zipFileName / zipFilePath 更新为本地文件,同时(如存在)同步修正 videoSourceUrl。
*
* 说明:
* - 只处理 materialType === '音乐' 的素材
* - 仅迁移 zipFileName 指向 S3_PUBLIC_DOMAIN 域名的记录(老数据)
* - 预览图 / 封面图仍然保留在 MinIO,不做改动
*/
import { PrismaClient } from '@prisma/client'
import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3'
import { Readable } from 'stream'
import { mkdir, writeFile, stat as fsStat } from 'fs/promises'
import { existsSync } from 'fs'
import path from 'path'
const prisma = new PrismaClient()
// S3 / MinIO 客户端配置
const s3Client = new S3Client({
endpoint: process.env.S3_ENDPOINT || '',
region: process.env.S3_REGION || 'us-east-1',
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY_ID!,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY!,
},
forcePathStyle: true,
requestHandler: {
requestTimeout: 300000, // 5分钟超时
connectionTimeout: 30000, // 30秒连接超时
},
maxAttempts: 5,
retryMode: 'adaptive',
})
const S3_BUCKET = process.env.S3_BUCKET || ''
const S3_PUBLIC_DOMAIN = process.env.S3_PUBLIC_DOMAIN || ''
const UPLOAD_DIR = process.env.UPLOAD_DIR || './uploads'
function isHttpUrl(url: string | null | undefined): url is string {
return !!url && /^https?:\/\//.test(url)
}
function isS3PublicUrl(url: string): boolean {
if (!url || !isHttpUrl(url) || !S3_PUBLIC_DOMAIN) return false
try {
const urlObj = new URL(url)
const domainObj = new URL(S3_PUBLIC_DOMAIN)
return urlObj.hostname === domainObj.hostname
} catch {
return false
}
}
/**
* 从 S3 公网 URL 提取对象 Key
* 兼容 path-style:https://s3.1080p.hk/jsxiangche/由壹点AI/audio-xxx.mp3
*/
function extractS3KeyFromPublicUrl(url: string): string {
const urlObj = new URL(url)
// pathname 可能包含 URL 编码的中文,需要先 decode
let key = decodeURIComponent(urlObj.pathname.slice(1)) // 去掉开头的 '/'
// 如果前缀中带有 bucket 名称,则去掉 bucket 部分,仅保留真实对象 Key
if (S3_BUCKET && key.startsWith(`${S3_BUCKET}/`)) {
key = key.replace(`${S3_BUCKET}/`, '')
}
return key
}
/**
* 从 S3 下载整个对象为 Buffer
*/
async function downloadS3Object(key: string): Promise<Buffer> {
const cmd = new GetObjectCommand({
Bucket: S3_BUCKET,
Key: key,
})
const res = await s3Client.send(cmd)
const body = res.Body as Readable | undefined
if (!body) {
throw new Error('S3 对象 Body 为空')
}
const chunks: Buffer[] = []
for await (const chunk of body) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
}
return Buffer.concat(chunks)
}
/**
* 生成本地文件名(与上传接口中生成逻辑保持一致:audio-source 前缀)
*/
async function generateLocalFilename(materialId: string, ext: string): Promise<string> {
const safeExt = ext || '.mp3'
let index = 0
while (true) {
const suffix = index === 0 ? '' : `-${index}`
const name = `audio-source-${materialId}${suffix}-${Date.now()}${safeExt}`
const fullPath = path.join(UPLOAD_DIR, name)
if (!existsSync(fullPath)) {
return name
}
index++
}
}
async function migrateOneMaterial(material: {
id: string
zipFileName: string | null
zipFilePath: string | null
videoSourceUrl: string | null
}) {
const { id, zipFileName, videoSourceUrl } = material
if (!zipFileName || !isHttpUrl(zipFileName)) {
console.log(`[${id}] 跳过:zipFileName 为空或不是 HTTP URL`)
return { skipped: true }
}
if (!isS3PublicUrl(zipFileName)) {
console.log(`[${id}] 跳过:zipFileName 不是当前 S3_PUBLIC_DOMAIN 域名`)
return { skipped: true }
}
console.log(`\n[${id}] 开始迁移 音乐 源文件`)
console.log(` 旧 zipFileName: ${zipFileName}`)
const key = extractS3KeyFromPublicUrl(zipFileName)
console.log(` S3 Key: ${key}`)
const ext = path.extname(key).toLowerCase() || '.mp3'
// 确保 uploads 目录存在
if (!existsSync(UPLOAD_DIR)) {
await mkdir(UPLOAD_DIR, { recursive: true })
}
// 生成不冲突的本地文件名
const localFileName = await generateLocalFilename(id, ext)
const localFullPath = path.join(UPLOAD_DIR, localFileName)
// 从 S3 下载
console.log(' 📥 正在从 S3 下载音乐源文件...')
const buffer = await downloadS3Object(key)
console.log(` 📦 下载完成,大小:${buffer.length} 字节`)
// 写入本地
console.log(` 💾 写入本地文件: ${localFullPath}`)
await writeFile(localFullPath, buffer)
// 校验
const st = await fsStat(localFullPath)
if (st.size !== buffer.length) {
throw new Error(`本地文件大小不匹配,期望 ${buffer.length} 实际 ${st.size}`)
}
// 更新数据库:zipFileName / zipFilePath 指向本地文件
// 与新上传逻辑保持一致:zipFileName 存文件名,zipFilePath 存完整路径
const newZipFileName = localFileName
const newZipFilePath = localFullPath
// 如 videoSourceUrl 之前错误指向了 S3 URL,则同步更新为本地路径(/uploads/xxx)
let newVideoSourceUrl: string | null | undefined = undefined
if (videoSourceUrl && isS3PublicUrl(videoSourceUrl)) {
newVideoSourceUrl = `/uploads/${localFileName}`
}
await prisma.material.update({
where: { id },
data: {
zipFileName: newZipFileName,
zipFilePath: newZipFilePath,
...(newVideoSourceUrl ? { videoSourceUrl: newVideoSourceUrl } : {}),
} as any,
})
console.log(` ✅ 迁移完成,新 zipFileName: ${newZipFileName}`)
if (newVideoSourceUrl) {
console.log(` ✅ 同步更新 videoSourceUrl: ${newVideoSourceUrl}`)
}
console.log(' ⚠️ 注意:当前脚本不会删除 MinIO 上的旧源文件,以便出现问题时可以回滚。')
return { skipped: false }
}
async function main() {
console.log('🚀 开始迁移旧音乐源文件到本地 uploads...')
if (!S3_BUCKET || !S3_PUBLIC_DOMAIN) {
console.error('❌ 缺少必要环境变量:S3_BUCKET 或 S3_PUBLIC_DOMAIN')
process.exit(1)
}
console.log('📋 配置信息:')
console.log(` S3_BUCKET: ${S3_BUCKET}`)
console.log(` S3_PUBLIC_DOMAIN: ${S3_PUBLIC_DOMAIN}`)
console.log(` UPLOAD_DIR: ${UPLOAD_DIR}`)
console.log('')
// 查询需要迁移的素材:音乐类型,zipFileName 指向 S3_PUBLIC_DOMAIN
const materials = await prisma.material.findMany({
where: {
deletedAt: null,
materialType: '音乐' as any,
zipFileName: {
contains: S3_PUBLIC_DOMAIN.replace(/^https?:\/\//, ''),
} as any,
},
select: {
id: true,
zipFileName: true,
zipFilePath: true,
videoSourceUrl: true,
},
})
if (materials.length === 0) {
console.log('✅ 没有需要迁移的音乐源文件')
return
}
console.log(`🔍 共找到 ${materials.length} 条音乐素材需要迁回本地`)
let success = 0
let skipped = 0
let failed = 0
for (let i = 0; i < materials.length; i++) {
const m = materials[i]
console.log(`\n==== [${i + 1}/${materials.length}] 处理素材 ${m.id} (音乐) ====`)
try {
const result = await migrateOneMaterial(m)
if (result.skipped) {
skipped++
} else {
success++
}
} catch (err: any) {
failed++
console.error(`❌ 迁移素材 ${m.id} 失败:`, err?.message || err)
}
}
console.log('\n📊 迁移结果统计:')
console.log(` 成功迁移: ${success}`)
console.log(` 跳过: ${skipped}`)
console.log(` 失败: ${failed}`)
if (failed === 0) {
console.log('\n✅ 音乐源文件迁回本地完成(旧 S3 源保留,可按需后续手工清理)')
} else {
console.log('\n⚠️ 部分素材迁移失败,请根据上方日志逐条检查')
}
}
main()
.catch((err) => {
console.error('❌ 脚本执行出错:', err)
process.exit(1)
})
.finally(async () => {
await prisma.$disconnect()
})