263 lines
7.8 KiB
JavaScript
263 lines
7.8 KiB
JavaScript
/**
|
||
* 分析CSV文件，统计老素材
|
||
* 使用Node.js原生模块，不需要额外依赖
|
||
*/
|
||
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const csvFile = process.argv[2] || 'material-inventory-2026-01-13T09-04-07.csv';
|
||
const outputDir = process.argv[3] || 'D:\\qwertyuiop\\old-materials-backup';
|
||
|
||
const cutoffDate = new Date('2025-10-27T23:59:59.999Z');
|
||
|
||
function isOldMaterial(row) {
|
||
// 1. 检查创建时间
|
||
try {
|
||
const createdAt = new Date(row['创建时间']);
|
||
if (createdAt <= cutoffDate) {
|
||
return true;
|
||
}
|
||
} catch (e) {
|
||
// 日期解析失败
|
||
}
|
||
|
||
// 2. 检查预览图URL
|
||
const previewUrl = row['预览图URL'] || '';
|
||
if (previewUrl.includes('s3.1080p.hk') &&
|
||
previewUrl.includes('/jsxiangche/') &&
|
||
!previewUrl.includes('由壹点AI/')) {
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
function isOldMaterialUrl(url) {
|
||
if (!url || typeof url !== 'string') return false;
|
||
return url.includes('s3.1080p.hk') &&
|
||
url.includes('/jsxiangche/') &&
|
||
!url.includes('由壹点AI/');
|
||
}
|
||
|
||
function parseCSV(content) {
|
||
const lines = content.split('\n').filter(line => line.trim());
|
||
if (lines.length === 0) return { headers: [], rows: [] };
|
||
|
||
// 解析表头
|
||
const headers = lines[0].split(',').map(h => h.replace(/^"|"$/g, ''));
|
||
|
||
// 解析数据行
|
||
const rows = [];
|
||
for (let i = 1; i < lines.length; i++) {
|
||
const line = lines[i];
|
||
if (!line.trim()) continue;
|
||
|
||
// 简单的CSV解析（处理引号内的逗号）
|
||
const values = [];
|
||
let current = '';
|
||
let inQuotes = false;
|
||
|
||
for (let j = 0; j < line.length; j++) {
|
||
const char = line[j];
|
||
if (char === '"') {
|
||
inQuotes = !inQuotes;
|
||
} else if (char === ',' && !inQuotes) {
|
||
values.push(current.replace(/^"|"$/g, ''));
|
||
current = '';
|
||
} else {
|
||
current += char;
|
||
}
|
||
}
|
||
values.push(current.replace(/^"|"$/g, ''));
|
||
|
||
const row = {};
|
||
headers.forEach((header, index) => {
|
||
row[header] = values[index] || '';
|
||
});
|
||
rows.push(row);
|
||
}
|
||
|
||
return { headers, rows };
|
||
}
|
||
|
||
function main() {
|
||
console.log('='.repeat(60));
|
||
console.log('分析CSV文件，统计老素材');
|
||
console.log('='.repeat(60));
|
||
console.log(`CSV文件: ${csvFile}`);
|
||
console.log(`输出目录: ${outputDir}`);
|
||
console.log();
|
||
|
||
if (!fs.existsSync(csvFile)) {
|
||
console.error(`❌ CSV文件不存在: ${csvFile}`);
|
||
process.exit(1);
|
||
}
|
||
|
||
// 读取CSV文件
|
||
console.log('📋 读取CSV文件...');
|
||
const content = fs.readFileSync(csvFile, 'utf-8');
|
||
const { headers, rows } = parseCSV(content);
|
||
|
||
console.log(` 读取到 ${rows.length} 条记录`);
|
||
console.log();
|
||
|
||
// 筛选老素材
|
||
console.log('🔍 筛选老素材...');
|
||
const oldMaterials = rows.filter(isOldMaterial);
|
||
console.log(` 找到 ${oldMaterials.length} 个老素材`);
|
||
console.log();
|
||
|
||
// 统计源文件
|
||
let localPathCount = 0;
|
||
let httpUrlCount = 0;
|
||
let s3UrlCount = 0;
|
||
let emptyCount = 0;
|
||
|
||
let localPathSize = 0;
|
||
let httpUrlSize = 0;
|
||
let s3UrlSize = 0;
|
||
|
||
const downloadTasks = [];
|
||
|
||
for (const material of oldMaterials) {
|
||
const materialId = material['素材ID'];
|
||
const zipFilePath = material['压缩包路径'] || '';
|
||
const zipFileName = material['压缩包文件名'] || '';
|
||
const zipFileSize = parseInt(material['压缩包大小(字节)'] || '0', 10);
|
||
const previewUrl = material['预览图URL'] || '';
|
||
|
||
// 收集预览图
|
||
if (previewUrl && isOldMaterialUrl(previewUrl)) {
|
||
downloadTasks.push({
|
||
materialId,
|
||
url: previewUrl,
|
||
field: 'previewUrl',
|
||
type: 'S3',
|
||
});
|
||
}
|
||
|
||
// 收集源文件
|
||
if (zipFilePath) {
|
||
if (zipFilePath.startsWith('http://') || zipFilePath.startsWith('https://')) {
|
||
httpUrlCount++;
|
||
httpUrlSize += zipFileSize;
|
||
downloadTasks.push({
|
||
materialId,
|
||
url: zipFilePath,
|
||
field: 'zipFilePath',
|
||
type: 'HTTP',
|
||
size: zipFileSize,
|
||
});
|
||
} else if (zipFilePath.startsWith('uploads/') || zipFilePath.startsWith('/app/uploads/')) {
|
||
localPathCount++;
|
||
localPathSize += zipFileSize;
|
||
downloadTasks.push({
|
||
materialId,
|
||
url: zipFilePath,
|
||
field: 'zipFilePath',
|
||
type: 'LOCAL',
|
||
size: zipFileSize,
|
||
});
|
||
}
|
||
}
|
||
|
||
if (zipFileName && isOldMaterialUrl(zipFileName)) {
|
||
s3UrlCount++;
|
||
downloadTasks.push({
|
||
materialId,
|
||
url: zipFileName,
|
||
field: 'zipFileName',
|
||
type: 'S3',
|
||
});
|
||
}
|
||
|
||
if (!zipFilePath && !zipFileName) {
|
||
emptyCount++;
|
||
}
|
||
}
|
||
|
||
// 输出统计
|
||
console.log('源文件存储位置统计：');
|
||
console.log('='.repeat(60));
|
||
console.log(`本地路径（uploads/...）: ${localPathCount} 个`);
|
||
console.log(` - 总大小: ${(localPathSize / 1024 / 1024 / 1024).toFixed(2)} GB`);
|
||
if (localPathCount > 0) {
|
||
console.log(` - 平均大小: ${(localPathSize / localPathCount / 1024 / 1024).toFixed(2)} MB`);
|
||
}
|
||
console.log();
|
||
console.log(`HTTP URL（http://103.118.245.6/...）: ${httpUrlCount} 个`);
|
||
console.log(` - 总大小: ${(httpUrlSize / 1024 / 1024 / 1024).toFixed(2)} GB`);
|
||
if (httpUrlCount > 0) {
|
||
console.log(` - 平均大小: ${(httpUrlSize / httpUrlCount / 1024 / 1024).toFixed(2)} MB`);
|
||
}
|
||
console.log();
|
||
console.log(`S3 URL（s3.1080p.hk/...）: ${s3UrlCount} 个`);
|
||
console.log();
|
||
console.log(`无源文件: ${emptyCount} 个`);
|
||
console.log();
|
||
|
||
const totalSize = localPathSize + httpUrlSize;
|
||
console.log('总计：');
|
||
console.log(` - 总素材数: ${oldMaterials.length} 个`);
|
||
console.log(` - 有源文件的素材: ${localPathCount + httpUrlCount + s3UrlCount} 个`);
|
||
console.log(` - 源文件总大小: ${(totalSize / 1024 / 1024 / 1024).toFixed(2)} GB`);
|
||
console.log(` - 下载任务数: ${downloadTasks.length} 个`);
|
||
console.log();
|
||
|
||
// 创建输出目录
|
||
if (!fs.existsSync(outputDir)) {
|
||
fs.mkdirSync(outputDir, { recursive: true });
|
||
}
|
||
|
||
// 保存统计信息
|
||
const stats = {
|
||
totalMaterials: rows.length,
|
||
oldMaterials: oldMaterials.length,
|
||
localPathCount,
|
||
httpUrlCount,
|
||
s3UrlCount,
|
||
emptyCount,
|
||
localPathSize,
|
||
httpUrlSize,
|
||
s3UrlSize,
|
||
totalSize,
|
||
totalSizeGB: (totalSize / 1024 / 1024 / 1024).toFixed(2),
|
||
downloadTasksCount: downloadTasks.length,
|
||
timestamp: new Date().toISOString(),
|
||
};
|
||
|
||
const statsFile = path.join(outputDir, 'old-materials-stats.json');
|
||
fs.writeFileSync(statsFile, JSON.stringify(stats, null, 2), 'utf-8');
|
||
console.log(`✅ 统计信息已保存: ${statsFile}`);
|
||
|
||
// 保存老素材列表（前100个作为示例）
|
||
const sampleMaterials = oldMaterials.slice(0, 100).map(m => ({
|
||
id: m['素材ID'],
|
||
type: m['素材类型'],
|
||
createdAt: m['创建时间'],
|
||
previewUrl: m['预览图URL'],
|
||
zipFilePath: m['压缩包路径'],
|
||
zipFileSize: m['压缩包大小(字节)'],
|
||
}));
|
||
|
||
const sampleFile = path.join(outputDir, 'old-materials-sample.json');
|
||
fs.writeFileSync(sampleFile, JSON.stringify(sampleMaterials, null, 2), 'utf-8');
|
||
console.log(`✅ 示例素材列表已保存: ${sampleFile} (前100个)`);
|
||
|
||
console.log();
|
||
console.log('='.repeat(60));
|
||
console.log('分析完成');
|
||
console.log('='.repeat(60));
|
||
console.log();
|
||
console.log('📊 关键数据：');
|
||
console.log(` - 老素材总数: ${oldMaterials.length} 个`);
|
||
console.log(` - 本地路径源文件: ${localPathCount} 个, ${(localPathSize / 1024 / 1024 / 1024).toFixed(2)} GB`);
|
||
console.log(` - HTTP URL源文件: ${httpUrlCount} 个, ${(httpUrlSize / 1024 / 1024 / 1024).toFixed(2)} GB`);
|
||
console.log(` - 总下载大小: ${stats.totalSizeGB} GB`);
|
||
console.log();
|
||
console.log('✅ 确认：这些是老素材的文件（创建时间 ≤ 2025-10-27 或使用旧MinIO URL）');
|
||
}
|
||
|
||
main();
|