374 lines
12 KiB
TypeScript
374 lines
12 KiB
TypeScript
/**
|
||
* 分析旧存储桶中的视频/音频素材
|
||
* 统计需要迁移的素材数量、大小，并计算迁移时间
|
||
*
|
||
* 使用方法：
|
||
* npx tsx scripts/analyze-old-storage-migration.ts [--network-speed <Mbps>]
|
||
*
|
||
* 示例：
|
||
* npx tsx scripts/analyze-old-storage-migration.ts --network-speed 100
|
||
*/
|
||
|
||
import { PrismaClient } from '@prisma/client';
|
||
import { S3Client, HeadObjectCommand } from '@aws-sdk/client-s3';
|
||
|
||
const DATABASE_URL = process.env.DATABASE_URL || 'postgresql://gongxiangpan:gongxiangpan123@192.168.58.20:5432/lobechat';
|
||
const prisma = new PrismaClient({
|
||
datasources: {
|
||
db: {
|
||
url: DATABASE_URL,
|
||
},
|
||
},
|
||
});
|
||
|
||
// 旧MinIO配置
|
||
const OLD_S3_ENDPOINT = 'https://s3.1080p.hk';
|
||
const OLD_S3_BUCKET = 'jsxiangche';
|
||
const OLD_S3_ACCESS_KEY_ID = process.env.OLD_S3_ACCESS_KEY_ID || process.env.S3_ACCESS_KEY_ID_OLD || '';
|
||
const OLD_S3_SECRET_ACCESS_KEY = process.env.OLD_S3_SECRET_ACCESS_KEY || process.env.S3_SECRET_ACCESS_KEY_OLD || '';
|
||
|
||
// 创建旧S3客户端（如果配置了凭据）
|
||
let oldS3Client: S3Client | null = null;
|
||
if (OLD_S3_ACCESS_KEY_ID && OLD_S3_SECRET_ACCESS_KEY) {
|
||
oldS3Client = new S3Client({
|
||
endpoint: OLD_S3_ENDPOINT,
|
||
region: 'us-east-1',
|
||
credentials: {
|
||
accessKeyId: OLD_S3_ACCESS_KEY_ID,
|
||
secretAccessKey: OLD_S3_SECRET_ACCESS_KEY,
|
||
},
|
||
forcePathStyle: true,
|
||
});
|
||
}
|
||
|
||
// 视频/音频素材类型
|
||
const VIDEO_TYPES = ['主图视频', '安装视频', '种草视频'];
|
||
const AUDIO_TYPES = ['音频素材', '音乐'];
|
||
|
||
// 从命令行参数获取网速（Mbps，默认100Mbps）
|
||
const args = process.argv.slice(2);
|
||
const networkSpeedArgIndex = args.indexOf('--network-speed');
|
||
const networkSpeedMbps = networkSpeedArgIndex >= 0 && args[networkSpeedArgIndex + 1]
|
||
? parseFloat(args[networkSpeedArgIndex + 1])
|
||
: 100; // 默认100Mbps
|
||
|
||
/**
|
||
* 从URL提取bucket和key
|
||
*/
|
||
function parseUrl(url: string): { bucket: string; key: string; domain: string } | null {
|
||
try {
|
||
const urlObj = new URL(url);
|
||
const pathParts = urlObj.pathname.split('/').filter(p => p);
|
||
if (pathParts.length === 0) return null;
|
||
const bucket = decodeURIComponent(pathParts[0]);
|
||
const key = pathParts.slice(1).map(p => decodeURIComponent(p)).join('/');
|
||
return { bucket, key, domain: urlObj.hostname };
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检查URL是否指向旧存储桶
|
||
*/
|
||
function isOldStorageUrl(url: string | null): boolean {
|
||
if (!url) return false;
|
||
const parsed = parseUrl(url);
|
||
if (!parsed) return false;
|
||
return parsed.bucket === 'jsxiangche' || parsed.domain.includes('1080p.hk');
|
||
}
|
||
|
||
/**
|
||
* 通过HTTP HEAD请求获取文件大小
|
||
*/
|
||
async function getFileSizeFromUrl(url: string): Promise<number | null> {
|
||
try {
|
||
const response = await fetch(url, {
|
||
method: 'HEAD',
|
||
signal: AbortSignal.timeout(10000), // 10秒超时
|
||
});
|
||
if (response.ok) {
|
||
const contentLength = response.headers.get('content-length');
|
||
return contentLength ? parseInt(contentLength, 10) : null;
|
||
}
|
||
return null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 通过S3 API获取文件大小
|
||
*/
|
||
async function getFileSizeFromS3(bucket: string, key: string): Promise<number | null> {
|
||
if (!oldS3Client) return null;
|
||
try {
|
||
const command = new HeadObjectCommand({
|
||
Bucket: bucket,
|
||
Key: key,
|
||
});
|
||
const response = await oldS3Client.send(command);
|
||
return response.ContentLength || null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 获取素材的所有相关文件大小
|
||
*/
|
||
async function getMaterialFileSizes(material: any): Promise<{
|
||
previewUrl: number | null;
|
||
videoCoverUrl: number | null;
|
||
videoPreview540Url: number | null;
|
||
videoPreviewClipUrl: number | null;
|
||
total: number;
|
||
}> {
|
||
const sizes = {
|
||
previewUrl: null as number | null,
|
||
videoCoverUrl: null as number | null,
|
||
videoPreview540Url: null as number | null,
|
||
videoPreviewClipUrl: null as number | null,
|
||
total: 0,
|
||
};
|
||
|
||
// 并行获取所有URL的文件大小
|
||
const promises: Promise<void>[] = [];
|
||
|
||
if (material.previewUrl) {
|
||
promises.push(
|
||
getFileSizeFromUrl(material.previewUrl).then(size => {
|
||
sizes.previewUrl = size;
|
||
if (size) sizes.total += size;
|
||
})
|
||
);
|
||
}
|
||
|
||
if (material.videoCoverUrl) {
|
||
promises.push(
|
||
getFileSizeFromUrl(material.videoCoverUrl).then(size => {
|
||
sizes.videoCoverUrl = size;
|
||
if (size) sizes.total += size;
|
||
})
|
||
);
|
||
}
|
||
|
||
if (material.videoPreview540Url) {
|
||
promises.push(
|
||
getFileSizeFromUrl(material.videoPreview540Url).then(size => {
|
||
sizes.videoPreview540Url = size;
|
||
if (size) sizes.total += size;
|
||
})
|
||
);
|
||
}
|
||
|
||
if (material.videoPreviewClipUrl) {
|
||
promises.push(
|
||
getFileSizeFromUrl(material.videoPreviewClipUrl).then(size => {
|
||
sizes.videoPreviewClipUrl = size;
|
||
if (size) sizes.total += size;
|
||
})
|
||
);
|
||
}
|
||
|
||
await Promise.all(promises);
|
||
|
||
return sizes;
|
||
}
|
||
|
||
/**
|
||
* 格式化文件大小
|
||
*/
|
||
function formatFileSize(bytes: number): string {
|
||
if (bytes < 1024) return `${bytes} B`;
|
||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`;
|
||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
|
||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
||
}
|
||
|
||
/**
|
||
* 计算迁移时间（秒）
|
||
*/
|
||
function calculateMigrationTime(bytes: number, speedMbps: number): number {
|
||
const bits = bytes * 8;
|
||
const speedBps = (speedMbps * 1024 * 1024) / 8; // 转换为字节/秒
|
||
// 考虑网络利用率（假设70%的实际利用率）
|
||
const effectiveSpeed = speedBps * 0.7;
|
||
return Math.ceil(bits / (speedMbps * 1024 * 1024));
|
||
}
|
||
|
||
/**
|
||
* 格式化时间
|
||
*/
|
||
function formatTime(seconds: number): string {
|
||
if (seconds < 60) return `${seconds}秒`;
|
||
if (seconds < 3600) return `${Math.floor(seconds / 60)}分钟${seconds % 60}秒`;
|
||
const hours = Math.floor(seconds / 3600);
|
||
const minutes = Math.floor((seconds % 3600) / 60);
|
||
const secs = seconds % 60;
|
||
if (hours < 24) return `${hours}小时${minutes}分钟${secs}秒`;
|
||
const days = Math.floor(hours / 24);
|
||
const remainingHours = hours % 24;
|
||
return `${days}天${remainingHours}小时${minutes}分钟`;
|
||
}
|
||
|
||
async function main() {
|
||
console.log('='.repeat(60));
|
||
console.log('旧存储桶迁移分析');
|
||
console.log('='.repeat(60));
|
||
console.log();
|
||
console.log('配置信息:');
|
||
console.log(` 旧存储桶: ${OLD_S3_ENDPOINT}/${OLD_S3_BUCKET}`);
|
||
console.log(` 网速: ${networkSpeedMbps} Mbps`);
|
||
console.log(` S3客户端: ${oldS3Client ? '已配置' : '未配置（将使用HTTP请求）'}`);
|
||
console.log();
|
||
|
||
// 1. 查询所有视频/音频素材
|
||
console.log('查询视频/音频素材...');
|
||
const allMaterials = await prisma.material.findMany({
|
||
where: {
|
||
materialType: {
|
||
in: [...VIDEO_TYPES, ...AUDIO_TYPES],
|
||
},
|
||
deletedAt: null,
|
||
},
|
||
select: {
|
||
id: true,
|
||
productModel: true,
|
||
materialType: true,
|
||
createdAt: true,
|
||
previewUrl: true,
|
||
videoCoverUrl: true,
|
||
videoPreview540Url: true,
|
||
videoPreviewClipUrl: true,
|
||
} as any,
|
||
});
|
||
|
||
console.log(` 找到 ${allMaterials.length} 个视频/音频素材`);
|
||
console.log();
|
||
|
||
// 2. 筛选使用旧存储桶的素材
|
||
console.log('筛选使用旧存储桶的素材...');
|
||
const oldStorageMaterials = allMaterials.filter(material => {
|
||
return isOldStorageUrl((material as any).previewUrl) ||
|
||
isOldStorageUrl((material as any).videoCoverUrl) ||
|
||
isOldStorageUrl((material as any).videoPreview540Url) ||
|
||
isOldStorageUrl((material as any).videoPreviewClipUrl);
|
||
});
|
||
|
||
console.log(` 使用旧存储桶的素材: ${oldStorageMaterials.length} 个`);
|
||
console.log();
|
||
|
||
// 3. 按类型统计
|
||
const videoMaterials = oldStorageMaterials.filter(m => VIDEO_TYPES.includes((m as any).materialType));
|
||
const audioMaterials = oldStorageMaterials.filter(m => AUDIO_TYPES.includes((m as any).materialType));
|
||
|
||
console.log('类型统计:');
|
||
console.log(` 视频素材: ${videoMaterials.length} 个`);
|
||
console.log(` 音频素材: ${audioMaterials.length} 个`);
|
||
console.log();
|
||
|
||
// 4. 获取文件大小（抽样或全部）
|
||
console.log('获取文件大小信息...');
|
||
console.log(` （正在处理 ${oldStorageMaterials.length} 个素材，可能需要一些时间）`);
|
||
console.log();
|
||
|
||
let totalSize = 0;
|
||
let successCount = 0;
|
||
let failCount = 0;
|
||
const materialSizes: Array<{
|
||
material: any;
|
||
size: number;
|
||
sizes: {
|
||
previewUrl: number | null;
|
||
videoCoverUrl: number | null;
|
||
videoPreview540Url: number | null;
|
||
videoPreviewClipUrl: number | null;
|
||
total: number;
|
||
};
|
||
}> = [];
|
||
|
||
// 批量处理，每批10个
|
||
const batchSize = 10;
|
||
for (let i = 0; i < oldStorageMaterials.length; i += batchSize) {
|
||
const batch = oldStorageMaterials.slice(i, i + batchSize);
|
||
const batchPromises = batch.map(async (material) => {
|
||
try {
|
||
const sizes = await getMaterialFileSizes(material);
|
||
if (sizes.total > 0) {
|
||
totalSize += sizes.total;
|
||
successCount++;
|
||
materialSizes.push({ material, size: sizes.total, sizes });
|
||
} else {
|
||
failCount++;
|
||
}
|
||
} catch (error) {
|
||
failCount++;
|
||
}
|
||
});
|
||
|
||
await Promise.all(batchPromises);
|
||
|
||
// 显示进度
|
||
const processed = Math.min(i + batchSize, oldStorageMaterials.length);
|
||
process.stdout.write(`\r 进度: ${processed}/${oldStorageMaterials.length} (${((processed / oldStorageMaterials.length) * 100).toFixed(1)}%)`);
|
||
}
|
||
process.stdout.write('\n\n');
|
||
|
||
console.log('文件大小统计:');
|
||
console.log(` 成功获取: ${successCount} 个`);
|
||
console.log(` 获取失败: ${failCount} 个`);
|
||
console.log(` 总大小: ${formatFileSize(totalSize)}`);
|
||
console.log(` 平均大小: ${formatFileSize(totalSize / Math.max(successCount, 1))}`);
|
||
console.log();
|
||
|
||
// 5. 计算迁移时间
|
||
console.log('迁移时间估算:');
|
||
const migrationTimeSeconds = calculateMigrationTime(totalSize, networkSpeedMbps);
|
||
console.log(` 总大小: ${formatFileSize(totalSize)}`);
|
||
console.log(` 网速: ${networkSpeedMbps} Mbps`);
|
||
console.log(` 估算时间: ${formatTime(migrationTimeSeconds)}`);
|
||
console.log(` 平均每个素材: ${formatTime(Math.ceil(migrationTimeSeconds / Math.max(successCount, 1)))}`);
|
||
console.log();
|
||
|
||
// 6. 详细统计（按类型）
|
||
console.log('按类型详细统计:');
|
||
const videoSizes = materialSizes.filter(m => VIDEO_TYPES.includes((m.material as any).materialType));
|
||
const audioSizes = materialSizes.filter(m => AUDIO_TYPES.includes((m.material as any).materialType));
|
||
|
||
const videoTotalSize = videoSizes.reduce((sum, m) => sum + m.size, 0);
|
||
const audioTotalSize = audioSizes.reduce((sum, m) => sum + m.size, 0);
|
||
|
||
console.log(` 视频素材:`);
|
||
console.log(` 数量: ${videoSizes.length} 个`);
|
||
console.log(` 总大小: ${formatFileSize(videoTotalSize)}`);
|
||
console.log(` 平均大小: ${formatFileSize(videoTotalSize / Math.max(videoSizes.length, 1))}`);
|
||
console.log(` 估算时间: ${formatTime(calculateMigrationTime(videoTotalSize, networkSpeedMbps))}`);
|
||
console.log();
|
||
|
||
console.log(` 音频素材:`);
|
||
console.log(` 数量: ${audioSizes.length} 个`);
|
||
console.log(` 总大小: ${formatFileSize(audioTotalSize)}`);
|
||
console.log(` 平均大小: ${formatFileSize(audioTotalSize / Math.max(audioSizes.length, 1))}`);
|
||
console.log(` 估算时间: ${formatTime(calculateMigrationTime(audioTotalSize, networkSpeedMbps))}`);
|
||
console.log();
|
||
|
||
// 7. 总结
|
||
console.log('='.repeat(60));
|
||
console.log('总结');
|
||
console.log('='.repeat(60));
|
||
console.log(`需要迁移的素材总数: ${oldStorageMaterials.length} 个`);
|
||
console.log(` - 视频素材: ${videoMaterials.length} 个`);
|
||
console.log(` - 音频素材: ${audioMaterials.length} 个`);
|
||
console.log(`总文件大小: ${formatFileSize(totalSize)}`);
|
||
console.log(`在 ${networkSpeedMbps} Mbps 网速下，估算迁移时间: ${formatTime(migrationTimeSeconds)}`);
|
||
console.log();
|
||
|
||
await prisma.$disconnect();
|
||
}
|
||
|
||
main().catch(error => {
|
||
console.error('执行失败:', error);
|
||
process.exit(1);
|
||
});
|