375 lines
11 KiB
TypeScript
375 lines
11 KiB
TypeScript
/**
|
||
* 清理影子系统中的测试素材及其文件
|
||
* 找出影子系统中有但正式系统中没有的素材，删除这些素材及其所有相关文件
|
||
*/
|
||
|
||
import { PrismaClient } from '@prisma/client'
|
||
import { S3Client, DeleteObjectCommand } from '@aws-sdk/client-s3'
|
||
import { existsSync, unlinkSync } from 'fs'
|
||
import path from 'path'
|
||
|
||
const PROD_DB_URL = process.env.PROD_DATABASE_URL
|
||
const SHADOW_DB_URL = process.env.SHADOW_DATABASE_URL
|
||
|
||
if (!PROD_DB_URL || !SHADOW_DB_URL) {
|
||
throw new Error('必须设置 PROD_DATABASE_URL 和 SHADOW_DATABASE_URL')
|
||
}
|
||
|
||
const prodPrisma = new PrismaClient({
|
||
datasources: {
|
||
db: {
|
||
url: PROD_DB_URL,
|
||
},
|
||
},
|
||
})
|
||
|
||
const shadowPrisma = new PrismaClient({
|
||
datasources: {
|
||
db: {
|
||
url: SHADOW_DB_URL,
|
||
},
|
||
},
|
||
})
|
||
|
||
// MinIO客户端
|
||
const s3Client = new S3Client({
|
||
endpoint: process.env.S3_ENDPOINT || 'http://192.168.58.20:9000',
|
||
region: process.env.S3_REGION || 'us-east-1',
|
||
credentials: {
|
||
accessKeyId: process.env.S3_ACCESS_KEY_ID!,
|
||
secretAccessKey: process.env.S3_SECRET_ACCESS_KEY!,
|
||
},
|
||
forcePathStyle: true,
|
||
})
|
||
|
||
const S3_BUCKET = process.env.S3_BUCKET || 'gongxiangpan'
|
||
const UPLOAD_DIR = process.env.UPLOAD_DIR || path.join(process.cwd(), 'uploads-shadow')
|
||
|
||
/**
|
||
* 从URL提取MinIO Key
|
||
*/
|
||
function extractS3Key(url: string): string | null {
|
||
try {
|
||
const urlObj = new URL(url)
|
||
let key = urlObj.pathname.replace(/^\//, '')
|
||
|
||
// 如果路径包含bucket名称，移除它
|
||
if (key.startsWith(`${S3_BUCKET}/`)) {
|
||
key = key.replace(`${S3_BUCKET}/`, '')
|
||
}
|
||
|
||
// 解码URL编码的路径
|
||
try {
|
||
key = decodeURIComponent(key)
|
||
} catch {
|
||
// 如果解码失败，使用原始路径
|
||
}
|
||
|
||
return key
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 删除S3文件
|
||
*/
|
||
async function deleteS3File(url: string): Promise<boolean> {
|
||
try {
|
||
const key = extractS3Key(url)
|
||
if (!key) {
|
||
console.warn(`[Cleanup] 无法提取S3 Key: ${url}`)
|
||
return false
|
||
}
|
||
|
||
const command = new DeleteObjectCommand({
|
||
Bucket: S3_BUCKET,
|
||
Key: key,
|
||
})
|
||
|
||
await s3Client.send(command)
|
||
return true
|
||
} catch (error: any) {
|
||
if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
|
||
// 文件不存在，视为成功
|
||
return true
|
||
}
|
||
console.error(`[Cleanup] 删除S3文件失败: ${url}`, error.message)
|
||
return false
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 删除本地文件
|
||
*/
|
||
function deleteLocalFile(filePath: string): boolean {
|
||
try {
|
||
const absolutePath = path.isAbsolute(filePath)
|
||
? filePath
|
||
: path.join(process.cwd(), filePath)
|
||
|
||
if (existsSync(absolutePath)) {
|
||
unlinkSync(absolutePath)
|
||
return true
|
||
}
|
||
return true // 文件不存在，视为成功
|
||
} catch (error: any) {
|
||
console.error(`[Cleanup] 删除本地文件失败: ${filePath}`, error.message)
|
||
return false
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 收集素材的所有文件URL（使用统一的收集函数）
|
||
*/
|
||
async function collectMaterialFiles(materialId: string, material: any): Promise<string[]> {
|
||
const files: string[] = []
|
||
|
||
// 预览图
|
||
if (material.previewUrl && typeof material.previewUrl === 'string') {
|
||
files.push(material.previewUrl)
|
||
}
|
||
|
||
// 视频相关文件
|
||
if (material.videoSourceUrl && typeof material.videoSourceUrl === 'string') {
|
||
files.push(material.videoSourceUrl)
|
||
}
|
||
if (material.videoPreview540Url && typeof material.videoPreview540Url === 'string') {
|
||
files.push(material.videoPreview540Url)
|
||
}
|
||
if (material.videoPreviewClipUrl && typeof material.videoPreviewClipUrl === 'string') {
|
||
files.push(material.videoPreviewClipUrl)
|
||
}
|
||
if (material.videoCoverUrl && typeof material.videoCoverUrl === 'string') {
|
||
files.push(material.videoCoverUrl)
|
||
}
|
||
|
||
// 压缩包文件
|
||
if (material.zipFileName && typeof material.zipFileName === 'string') {
|
||
// 如果是URL，直接添加
|
||
if (material.zipFileName.startsWith('http://') || material.zipFileName.startsWith('https://')) {
|
||
files.push(material.zipFileName)
|
||
} else {
|
||
// 本地文件路径（相对于uploads目录）
|
||
const localPath = path.join(UPLOAD_DIR, material.zipFileName)
|
||
files.push(localPath)
|
||
}
|
||
}
|
||
if (material.zipFilePath && typeof material.zipFilePath === 'string') {
|
||
if (!material.zipFilePath.startsWith('http://') && !material.zipFilePath.startsWith('https://')) {
|
||
// 本地文件路径
|
||
files.push(material.zipFilePath)
|
||
}
|
||
}
|
||
|
||
// 展示图
|
||
if (Array.isArray(material.displayImages)) {
|
||
material.displayImages.forEach((img: any) => {
|
||
if (typeof img === 'string') {
|
||
files.push(img)
|
||
}
|
||
})
|
||
}
|
||
|
||
// 任务-详情：查询资产文件
|
||
if (material.materialType === '任务-详情') {
|
||
const assets = await shadowPrisma.materialAsset.findMany({
|
||
where: { materialId },
|
||
select: { fileUrl: true },
|
||
})
|
||
assets.forEach((asset: any) => {
|
||
if (asset.fileUrl) {
|
||
files.push(asset.fileUrl)
|
||
}
|
||
})
|
||
}
|
||
|
||
// 标准模型：查询SKU文件
|
||
if (material.materialType === '标准模型') {
|
||
const skus = await shadowPrisma.materialSku.findMany({
|
||
where: { materialId },
|
||
select: { filePath: true, previewUrl: true },
|
||
})
|
||
skus.forEach((sku: any) => {
|
||
if (sku.filePath) {
|
||
files.push(sku.filePath)
|
||
}
|
||
if (sku.previewUrl) {
|
||
files.push(sku.previewUrl)
|
||
}
|
||
})
|
||
}
|
||
|
||
return files.filter(Boolean)
|
||
}
|
||
|
||
async function main() {
|
||
console.log('开始清理影子系统中的测试素材...')
|
||
console.log('='.repeat(60))
|
||
console.log()
|
||
|
||
try {
|
||
// 1. 获取正式系统中的所有素材ID
|
||
console.log('📋 步骤 1: 获取正式系统中的素材ID列表...')
|
||
const prodMaterials = await prodPrisma.material.findMany({
|
||
where: {
|
||
deletedAt: null,
|
||
},
|
||
select: {
|
||
id: true,
|
||
},
|
||
})
|
||
const prodMaterialIds = new Set(prodMaterials.map(m => m.id))
|
||
console.log(` 正式系统素材数量: ${prodMaterialIds.size}`)
|
||
console.log()
|
||
|
||
// 2. 获取影子系统中的所有素材
|
||
console.log('📋 步骤 2: 获取影子系统中的素材...')
|
||
const shadowMaterials = await shadowPrisma.material.findMany({
|
||
where: {
|
||
deletedAt: null,
|
||
},
|
||
select: {
|
||
id: true,
|
||
productModel: true,
|
||
materialType: true,
|
||
previewUrl: true,
|
||
videoSourceUrl: true,
|
||
videoPreview540Url: true,
|
||
videoPreviewClipUrl: true,
|
||
videoCoverUrl: true,
|
||
zipFileName: true,
|
||
zipFilePath: true,
|
||
displayImages: true,
|
||
},
|
||
})
|
||
console.log(` 影子系统素材数量: ${shadowMaterials.length}`)
|
||
console.log()
|
||
|
||
// 3. 找出测试素材（影子系统中有但正式系统中没有的）
|
||
const testMaterials = shadowMaterials.filter(m => !prodMaterialIds.has(m.id))
|
||
console.log(`🔍 步骤 3: 识别测试素材...`)
|
||
console.log(` 找到 ${testMaterials.length} 个测试素材`)
|
||
console.log()
|
||
|
||
if (testMaterials.length === 0) {
|
||
console.log('✅ 没有发现测试素材，无需清理')
|
||
return
|
||
}
|
||
|
||
// 4. 显示测试素材列表
|
||
console.log('测试素材列表:')
|
||
testMaterials.forEach((m, index) => {
|
||
console.log(` ${index + 1}. ${m.productModel} (${m.materialType}) - ID: ${m.id}`)
|
||
})
|
||
console.log()
|
||
|
||
// 5. 收集所有需要删除的文件
|
||
console.log('📦 步骤 4: 收集测试素材的文件...')
|
||
const allFiles: string[] = []
|
||
for (const material of testMaterials) {
|
||
const files = await collectMaterialFiles(material.id, material)
|
||
allFiles.push(...files)
|
||
}
|
||
const uniqueFiles = Array.from(new Set(allFiles))
|
||
console.log(` 找到 ${uniqueFiles.length} 个文件需要删除`)
|
||
console.log()
|
||
|
||
// 6. 删除文件
|
||
console.log('🗑️ 步骤 5: 删除测试素材的文件...')
|
||
let deletedCount = 0
|
||
let failedCount = 0
|
||
|
||
for (const fileUrl of uniqueFiles) {
|
||
if (!fileUrl || typeof fileUrl !== 'string') continue
|
||
|
||
let success = false
|
||
if (fileUrl.startsWith('http://') || fileUrl.startsWith('https://')) {
|
||
// S3文件
|
||
success = await deleteS3File(fileUrl)
|
||
} else {
|
||
// 本地文件（可能是相对路径或绝对路径）
|
||
// 如果是相对路径，需要转换为绝对路径
|
||
let filePath = fileUrl
|
||
if (!path.isAbsolute(filePath)) {
|
||
// 如果路径以 uploads 开头，使用 UPLOAD_DIR
|
||
if (filePath.startsWith('uploads/') || filePath.startsWith('uploads\\')) {
|
||
filePath = path.join(process.cwd(), filePath.replace(/^uploads[\/\\]/, ''))
|
||
} else {
|
||
// 其他相对路径，尝试在 UPLOAD_DIR 中查找
|
||
filePath = path.join(process.cwd(), UPLOAD_DIR, filePath)
|
||
}
|
||
}
|
||
success = deleteLocalFile(filePath)
|
||
}
|
||
|
||
if (success) {
|
||
deletedCount++
|
||
} else {
|
||
failedCount++
|
||
}
|
||
}
|
||
|
||
console.log(` 删除结果: ${deletedCount} 成功, ${failedCount} 失败`)
|
||
console.log()
|
||
|
||
// 7. 删除关联记录（MaterialAsset, MaterialSku等）
|
||
console.log('🗑️ 步骤 6: 删除测试素材的关联记录...')
|
||
const testMaterialIds = testMaterials.map(m => m.id)
|
||
|
||
// 删除 MaterialAsset
|
||
const deletedAssets = await shadowPrisma.materialAsset.deleteMany({
|
||
where: {
|
||
materialId: { in: testMaterialIds },
|
||
},
|
||
})
|
||
console.log(` 删除 MaterialAsset: ${deletedAssets.count} 条`)
|
||
|
||
// 删除 MaterialSku
|
||
const deletedSkus = await shadowPrisma.materialSku.deleteMany({
|
||
where: {
|
||
materialId: { in: testMaterialIds },
|
||
},
|
||
})
|
||
console.log(` 删除 MaterialSku: ${deletedSkus.count} 条`)
|
||
|
||
// 删除 MaterialLog
|
||
const deletedLogs = await shadowPrisma.materialLog.deleteMany({
|
||
where: {
|
||
materialId: { in: testMaterialIds },
|
||
},
|
||
})
|
||
console.log(` 删除 MaterialLog: ${deletedLogs.count} 条`)
|
||
console.log()
|
||
|
||
// 8. 删除素材记录
|
||
console.log('🗑️ 步骤 7: 删除测试素材记录...')
|
||
const deletedMaterials = await shadowPrisma.material.deleteMany({
|
||
where: {
|
||
id: { in: testMaterialIds },
|
||
},
|
||
})
|
||
console.log(` 删除 Material: ${deletedMaterials.count} 条`)
|
||
console.log()
|
||
|
||
console.log('='.repeat(60))
|
||
console.log('✅ 清理完成！')
|
||
console.log('='.repeat(60))
|
||
console.log()
|
||
console.log(`删除统计:`)
|
||
console.log(` - 测试素材: ${testMaterials.length} 个`)
|
||
console.log(` - 文件: ${deletedCount} 个成功, ${failedCount} 个失败`)
|
||
console.log(` - 关联记录: ${deletedAssets.count} 个Asset, ${deletedSkus.count} 个SKU, ${deletedLogs.count} 个Log`)
|
||
console.log()
|
||
|
||
} catch (error) {
|
||
console.error('清理失败:', error)
|
||
process.exit(1)
|
||
} finally {
|
||
await prodPrisma.$disconnect()
|
||
await shadowPrisma.$disconnect()
|
||
}
|
||
}
|
||
|
||
main()
|