gongxiangp2/scripts/migrate-video-source-to-local.ts

255 lines
7.7 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* 视频/音频源文件迁回本地脚本
*
* 目标:将旧数据中存放在 MinIO(S3) 上的 videoSourceUrl 源视频,下载到本地 uploads 目录,
* 并把 videoSourceUrl 更新为本地相对路径(例如:/uploads/video-source-xxx.mp4)。
*
* 说明:
* - 只处理「视频类型」素材:主图视频 / 安装视频 / 种草视频
* - 目前不会删除 MinIO 上的旧源文件,只是把数据库指向改为本地,方便出问题时回滚
* - 预览视频 / 预览片段 / 封面图仍然保留在 MinIO,不做改动
*/
import { PrismaClient } from '@prisma/client'
import { S3Client, GetObjectCommand } from '@aws-sdk/client-s3'
import { Readable } from 'stream'
import { mkdir, writeFile, stat as fsStat } from 'fs/promises'
import { existsSync } from 'fs'
import path from 'path'
const prisma = new PrismaClient()
// S3 / MinIO 客户端配置(复用线上配置)
const s3Client = new S3Client({
endpoint: process.env.S3_ENDPOINT || '',
region: process.env.S3_REGION || 'us-east-1',
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY_ID!,
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY!,
},
forcePathStyle: true,
requestHandler: {
requestTimeout: 300000, // 5分钟超时
connectionTimeout: 30000, // 30秒连接超时
},
maxAttempts: 5,
retryMode: 'adaptive',
})
const S3_BUCKET = process.env.S3_BUCKET || ''
const S3_PUBLIC_DOMAIN = process.env.S3_PUBLIC_DOMAIN || ''
const UPLOAD_DIR = process.env.UPLOAD_DIR || './uploads'
// 仅迁移以下视频类型
const VIDEO_TYPES = ['主图视频', '安装视频', '种草视频']
function isHttpUrl(url: string | null | undefined): url is string {
return !!url && /^https?:\/\//.test(url)
}
function isS3PublicUrl(url: string): boolean {
if (!url || !isHttpUrl(url) || !S3_PUBLIC_DOMAIN) return false
try {
const urlObj = new URL(url)
const domainObj = new URL(S3_PUBLIC_DOMAIN)
return urlObj.hostname === domainObj.hostname
} catch {
return false
}
}
/**
* 从 S3 公网 URL 提取对象 Key
*/
function extractS3KeyFromPublicUrl(url: string): string {
const urlObj = new URL(url)
// pathname 形如:/jsxiangche/video-xxx.mp4 或 /gongxiangpan/由壹点AI/video-xxx.mp4
let key = urlObj.pathname.slice(1) // 去掉开头的 '/'
// 如果路径前面带有 bucket 名称,去掉 bucket 前缀,保留真正的对象 Key
if (S3_BUCKET && key.startsWith(`${S3_BUCKET}/`)) {
key = key.replace(`${S3_BUCKET}/`, '')
}
return key
}
/**
* 从 S3 下载整个对象为 Buffer
*/
async function downloadS3Object(key: string): Promise<Buffer> {
const cmd = new GetObjectCommand({
Bucket: S3_BUCKET,
Key: key,
})
const res = await s3Client.send(cmd)
const body = res.Body as Readable | undefined
if (!body) {
throw new Error('S3 对象 Body 为空')
}
const chunks: Buffer[] = []
for await (const chunk of body) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
}
return Buffer.concat(chunks)
}
/**
* 生成本地文件名(不会与现有 uploads 中的文件冲突)
*/
async function generateLocalFilename(materialId: string, ext: string): Promise<string> {
const safeExt = ext || '.mp4'
// 以 materialId + 时间戳 生成唯一文件名
let index = 0
while (true) {
const suffix = index === 0 ? '' : `-${index}`
const name = `video-source-${materialId}${suffix}-${Date.now()}${safeExt}`
const fullPath = path.join(UPLOAD_DIR, name)
if (!existsSync(fullPath)) {
return name
}
index++
}
}
async function migrateOneMaterial(material: { id: string; videoSourceUrl: string | null; materialType: string }) {
const { id, videoSourceUrl, materialType } = material
if (!videoSourceUrl || !isHttpUrl(videoSourceUrl)) {
console.log(`[${id}] 跳过:videoSourceUrl 为空或不是 HTTP URL`)
return { skipped: true }
}
if (!isS3PublicUrl(videoSourceUrl)) {
console.log(`[${id}] 跳过:videoSourceUrl 不是当前 S3_PUBLIC_DOMAIN 域名`)
return { skipped: true }
}
console.log(`\n[${id}] 开始迁移 ${materialType} 源视频`)
console.log(` 旧 URL: ${videoSourceUrl}`)
const key = extractS3KeyFromPublicUrl(videoSourceUrl)
console.log(` S3 Key: ${key}`)
const ext = path.extname(key).toLowerCase() || '.mp4'
// 确保 uploads 目录存在
if (!existsSync(UPLOAD_DIR)) {
await mkdir(UPLOAD_DIR, { recursive: true })
}
// 生成不冲突的本地文件名
const localFileName = await generateLocalFilename(id, ext)
const localFullPath = path.join(UPLOAD_DIR, localFileName)
// 从 S3 下载
console.log(' 📥 正在从 S3 下载源视频...')
const buffer = await downloadS3Object(key)
console.log(` 📦 下载完成,大小:${buffer.length} 字节`)
// 写入本地文件
console.log(` 💾 写入本地文件: ${localFullPath}`)
await writeFile(localFullPath, buffer)
// 校验本地文件大小
const stat = await fsStat(localFullPath)
if (stat.size !== buffer.length) {
throw new Error(`本地文件大小不匹配,期望 ${buffer.length} 实际 ${stat.size}`)
}
// 更新数据库:仅更新 videoSourceUrl,指向本地相对路径 /uploads/xxx
const newVideoSourceUrl = `/uploads/${localFileName}`
await prisma.material.update({
where: { id },
data: {
videoSourceUrl: newVideoSourceUrl,
} as any,
})
console.log(` ✅ 迁移完成,新 videoSourceUrl: ${newVideoSourceUrl}`)
console.log(' ⚠️ 注意:当前脚本不会删除 MinIO 上的旧源视频,以便出现问题时可以回滚。')
return { skipped: false }
}
async function main() {
console.log('🚀 开始迁移旧视频源文件到本地 uploads...')
if (!S3_BUCKET || !S3_PUBLIC_DOMAIN) {
console.error('❌ 缺少必要环境变量:S3_BUCKET 或 S3_PUBLIC_DOMAIN')
process.exit(1)
}
console.log('📋 配置信息:')
console.log(` S3_BUCKET: ${S3_BUCKET}`)
console.log(` S3_PUBLIC_DOMAIN: ${S3_PUBLIC_DOMAIN}`)
console.log(` UPLOAD_DIR: ${UPLOAD_DIR}`)
console.log('')
// 查询需要迁移的素材(仅视频类型,且 videoSourceUrl 在 S3_PUBLIC_DOMAIN 下)
const materials = await prisma.material.findMany({
where: {
deletedAt: null,
materialType: { in: VIDEO_TYPES } as any,
videoSourceUrl: {
not: null,
contains: S3_PUBLIC_DOMAIN.replace(/^https?:\/\//, ''),
} as any,
},
select: {
id: true,
materialType: true,
videoSourceUrl: true,
},
})
if (materials.length === 0) {
console.log('✅ 没有需要迁移的视频源文件')
return
}
console.log(`🔍 共找到 ${materials.length} 条视频素材需要迁回本地`)
let success = 0
let skipped = 0
let failed = 0
for (let i = 0; i < materials.length; i++) {
const m = materials[i]
console.log(`\n==== [${i + 1}/${materials.length}] 处理素材 ${m.id} (${m.materialType}) ====`)
try {
const result = await migrateOneMaterial(m)
if (result.skipped) {
skipped++
} else {
success++
}
} catch (err: any) {
failed++
console.error(`❌ 迁移素材 ${m.id} 失败:`, err?.message || err)
}
}
console.log('\n📊 迁移结果统计:')
console.log(` 成功迁移: ${success}`)
console.log(` 跳过: ${skipped}`)
console.log(` 失败: ${failed}`)
if (failed === 0) {
console.log('\n✅ 视频源文件迁回本地完成(旧 S3 源保留,可按需后续手工清理)')
} else {
console.log('\n⚠️ 部分素材迁移失败,请根据上方日志逐条检查')
}
}
main()
.catch((err) => {
console.error('❌ 脚本执行出错:', err)
process.exit(1)
})
.finally(async () => {
await prisma.$disconnect()
})