gongxiangp2/scripts/migrate-product-package-old-materials.ts
chenpei cd6e40d9d2 优化:网络传输性能和稳定性 - TCP缓冲区+并发控制优化
主要改动:
1. TCP缓冲区优化(16MB)- 解决公网上传速度慢问题
2. 并发数降低到1-3 - 避免4并发导致ERR_CONNECTION_RESET
3. 动态并发控制 - 根据网络环境自动调整
4. Nginx超时增加到120s - 支持大分片上传
5. 上传进度修复 - 修复剩余时间计算
6. 新增任务-页面上传器和详情组件
7. 统一部署文档为deploy-remote-build.ps1

问题修复:
- 公网用户上传从200-900KB/s提升到预期3-5MB/s
- 内网用户上传更稳定,无连接重置错误
- UploadProgress组件剩余时间计算修正

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-27 09:33:09 +08:00

630 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* 迁移"任务-详情"老素材脚本
*
* 功能:
* 1. 从S3下载预览图
* 2. 将预览图打包成分类zip(main.zip、sku.zip、detail.zip、all.zip)
* 3. 生成新的预览图资产记录(main-preview、sku-preview、detail-preview)
* 4. 更新数据库
*
* 安全措施:
* - 保留老数据不删除
* - 分批迁移(每次处理指定数量)
* - 完整的错误处理和回滚机制
* - 详细的日志记录
*/
import { PrismaClient } from '@prisma/client';
import { mkdir, writeFile, unlink, readFile } from 'fs/promises';
import path from 'path';
import archiver from 'archiver';
import { createWriteStream } from 'fs';
const prisma = new PrismaClient();
// 配置
const CONFIG = {
batchSize: 10, // 每批处理的素材数量
dryRun: false, // 是否为演练模式(不实际修改数据库)
skipDownload: false, // 是否跳过下载(用于测试打包逻辑)
targetProductModel: undefined, // 只迁移指定型号(用于测试)
};
// 日志记录
interface MigrationLog {
materialId: string;
productModel: string;
taskNo: string | null;
status: 'pending' | 'downloading' | 'packing' | 'saving' | 'success' | 'failed';
error?: string;
startTime: Date;
endTime?: Date;
assetsDownloaded?: number;
zipFilesCreated?: string[];
}
const migrationLogs: MigrationLog[] = [];
/**
* 从S3下载文件
*/
async function downloadFromS3(url: string, savePath: string): Promise<void> {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`下载失败: ${url}, 状态码: ${response.status}`);
}
const buffer = await response.arrayBuffer();
await writeFile(savePath, Buffer.from(buffer));
}
/**
* 创建zip文件
*/
async function createZip(
files: Array<{ path: string; name: string }>,
outputPath: string
): Promise<number> {
return new Promise((resolve, reject) => {
const output = createWriteStream(outputPath);
const archive = archiver('zip', { zlib: { level: 9 } });
let fileSize = 0;
output.on('close', () => {
fileSize = archive.pointer();
resolve(fileSize);
});
archive.on('error', (err) => reject(err));
output.on('error', (err) => reject(err));
archive.pipe(output);
for (const file of files) {
archive.file(file.path, { name: file.name });
}
archive.finalize();
});
}
/**
* 清理文件系统不安全的字符
*/
function sanitizeFileName(name: string): string {
return name.replace(/[\/\\:*?"<>|]/g, '_');
}
/**
* 迁移单个素材
*/
async function migrateMaterial(materialId: string): Promise<MigrationLog> {
const log: MigrationLog = {
materialId,
productModel: '',
taskNo: null,
status: 'pending',
startTime: new Date(),
};
try {
// 1. 查询素材信息
console.log(`\n[${materialId}] 开始迁移...`);
const material = await prisma.material.findUnique({
where: { id: materialId },
include: {
assets: {
where: {
assetType: { in: ['main', 'sku', 'detail', 'psd'] },
},
orderBy: { sortOrder: 'asc' },
},
user: {
select: {
realName: true,
username: true,
},
},
},
});
if (!material) {
throw new Error('素材不存在');
}
log.productModel = material.productModel;
log.taskNo = material.taskNo;
console.log(`[${materialId}] 素材信息: ${material.productModel}, 任务号: ${material.taskNo || '无'}`);
// 2. 按类型分组资产(只选择 S3 上的预览图,排除本地 zip 文件)
const mainAssets = material.assets.filter((a) => a.assetType === 'main' && a.fileUrl.startsWith('https://'));
const skuAssets = material.assets.filter((a) => a.assetType === 'sku' && a.fileUrl.startsWith('https://'));
const detailAssets = material.assets.filter((a) => a.assetType === 'detail' && a.fileUrl.startsWith('https://'));
const psdAssets = material.assets.filter((a) => a.assetType === 'psd');
console.log(`[${materialId}] 资产统计: 主图=${mainAssets.length}, SKU=${skuAssets.length}, 详情=${detailAssets.length}, PSD=${psdAssets.length}`);
if (mainAssets.length === 0 && skuAssets.length === 0 && detailAssets.length === 0) {
throw new Error('没有可迁移的资产');
}
// 3. 创建临时目录
const tempDir = path.join(process.cwd(), 'uploads', 'temp', `migrate-${materialId}`);
await mkdir(tempDir, { recursive: true });
const tempFiles: string[] = [];
try {
// 4. 下载预览图
log.status = 'downloading';
console.log(`[${materialId}] 开始下载预览图...`);
const downloadedFiles: {
main: Array<{ path: string; name: string; size: number }>;
sku: Array<{ path: string; name: string; size: number }>;
detail: Array<{ path: string; name: string; size: number }>;
} = { main: [], sku: [], detail: [] };
let downloadCount = 0;
// 下载主图
for (let i = 0; i < mainAssets.length; i++) {
const asset = mainAssets[i];
const originalFileName = asset.fileName || `main-${i}.jpg`;
const ext = path.extname(originalFileName);
const baseName = path.basename(originalFileName, ext);
// 确保 zip 包内的文件名唯一(添加序号前缀)
const uniqueFileName = mainAssets.length > 1
? `${String(i + 1).padStart(2, '0')}_${baseName}${ext}`
: originalFileName;
const filePath = path.join(tempDir, `main-${i}-${originalFileName}`);
if (!CONFIG.skipDownload) {
await downloadFromS3(asset.fileUrl, filePath);
}
downloadedFiles.main.push({
path: filePath,
name: uniqueFileName, // 使用唯一文件名
size: Number(asset.fileSize || 0),
});
tempFiles.push(filePath);
downloadCount++;
}
// 下载SKU图
for (let i = 0; i < skuAssets.length; i++) {
const asset = skuAssets[i];
const originalFileName = asset.fileName || `sku-${i}.jpg`;
const ext = path.extname(originalFileName);
const baseName = path.basename(originalFileName, ext);
// 确保 zip 包内的文件名唯一(添加序号前缀)
const uniqueFileName = skuAssets.length > 1
? `${String(i + 1).padStart(2, '0')}_${baseName}${ext}`
: originalFileName;
const filePath = path.join(tempDir, `sku-${i}-${originalFileName}`);
if (!CONFIG.skipDownload) {
await downloadFromS3(asset.fileUrl, filePath);
}
downloadedFiles.sku.push({
path: filePath,
name: uniqueFileName, // 使用唯一文件名
size: Number(asset.fileSize || 0),
});
tempFiles.push(filePath);
downloadCount++;
}
// 下载详情图
for (let i = 0; i < detailAssets.length; i++) {
const asset = detailAssets[i];
const originalFileName = asset.fileName || `detail-${i}.jpg`;
const ext = path.extname(originalFileName);
const baseName = path.basename(originalFileName, ext);
// 确保 zip 包内的文件名唯一(添加序号前缀)
const uniqueFileName = detailAssets.length > 1
? `${String(i + 1).padStart(2, '0')}_${baseName}${ext}`
: originalFileName;
const filePath = path.join(tempDir, `detail-${i}-${originalFileName}`);
if (!CONFIG.skipDownload) {
await downloadFromS3(asset.fileUrl, filePath);
}
downloadedFiles.detail.push({
path: filePath,
name: uniqueFileName, // 使用唯一文件名
size: Number(asset.fileSize || 0),
});
tempFiles.push(filePath);
downloadCount++;
}
log.assetsDownloaded = downloadCount;
console.log(`[${materialId}] 下载完成: ${downloadCount} 个文件`);
// 5. 打包分类zip
log.status = 'packing';
console.log(`[${materialId}] 开始打包zip文件...`);
const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
await mkdir(uploadsDir, { recursive: true });
const timestamp = Date.now();
const sanitizedProductModel = sanitizeFileName(material.productModel);
const authorName = material.user?.realName || material.user?.username || '未知作者';
const sanitizedAuthorName = sanitizeFileName(authorName);
const categoryName = material.categoryName || '未知店铺';
const sanitizedCategoryName = sanitizeFileName(categoryName);
const zipInfos: {
main?: { filePath: string; fileName: string; fileSize: number };
sku?: { filePath: string; fileName: string; fileSize: number };
detail?: { filePath: string; fileName: string; fileSize: number };
all: { filePath: string; fileName: string; fileSize: number };
} = {} as any;
const zipFilesForAll: Array<{ path: string; name: string }> = [];
const createdZipPaths: string[] = [];
// 打包主图
if (downloadedFiles.main.length > 0) {
const zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_主图.zip`;
const zipFilePath = path.join(uploadsDir, zipFileName);
const relativeZipPath = `uploads/materials/${zipFileName}`; // 强制使用正斜杠
const fileSize = await createZip(downloadedFiles.main, zipFilePath);
zipInfos.main = {
filePath: relativeZipPath,
fileName: zipFileName,
fileSize,
};
zipFilesForAll.push({ path: zipFilePath, name: zipFileName });
createdZipPaths.push(zipFilePath);
console.log(`[${materialId}] 主图打包完成: ${zipFileName} (${fileSize} bytes)`);
}
// 打包SKU图
if (downloadedFiles.sku.length > 0) {
const zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_SKU图.zip`;
const zipFilePath = path.join(uploadsDir, zipFileName);
const relativeZipPath = `uploads/materials/${zipFileName}`; // 强制使用正斜杠
const fileSize = await createZip(downloadedFiles.sku, zipFilePath);
zipInfos.sku = {
filePath: relativeZipPath,
fileName: zipFileName,
fileSize,
};
zipFilesForAll.push({ path: zipFilePath, name: zipFileName });
createdZipPaths.push(zipFilePath);
console.log(`[${materialId}] SKU图打包完成: ${zipFileName} (${fileSize} bytes)`);
}
// 打包详情图
if (downloadedFiles.detail.length > 0) {
const zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_详情图.zip`;
const zipFilePath = path.join(uploadsDir, zipFileName);
const relativeZipPath = `uploads/materials/${zipFileName}`; // 强制使用正斜杠
const fileSize = await createZip(downloadedFiles.detail, zipFilePath);
zipInfos.detail = {
filePath: relativeZipPath,
fileName: zipFileName,
fileSize,
};
zipFilesForAll.push({ path: zipFilePath, name: zipFileName });
createdZipPaths.push(zipFilePath);
console.log(`[${materialId}] 详情图打包完成: ${zipFileName} (${fileSize} bytes)`);
}
// 打包总zip
const allZipFileName = `${sanitizedProductModel}_all_${materialId}_${timestamp}.zip`;
const allZipFilePath = path.join(uploadsDir, allZipFileName);
const relativeAllZipPath = `uploads/materials/${allZipFileName}`; // 强制使用正斜杠
const allFileSize = await createZip(zipFilesForAll, allZipFilePath);
zipInfos.all = {
filePath: relativeAllZipPath,
fileName: allZipFileName,
fileSize: allFileSize,
};
createdZipPaths.push(allZipFilePath);
console.log(`[${materialId}] 总zip打包完成: ${allZipFileName} (${allFileSize} bytes)`);
log.zipFilesCreated = createdZipPaths;
// 6. 保存到数据库
if (!CONFIG.dryRun) {
log.status = 'saving';
console.log(`[${materialId}] 开始保存到数据库...`);
await prisma.$transaction(async (tx) => {
// 6.0 删除老的资产记录
// 删除老的预览图记录(*-preview)
await tx.materialAsset.deleteMany({
where: {
materialId,
assetType: { in: ['main-preview', 'sku-preview', 'detail-preview'] },
},
});
// 删除老的 S3 预览图记录(assetType='main'/'sku'/'detail')
await tx.materialAsset.deleteMany({
where: {
materialId,
assetType: { in: ['main', 'sku', 'detail'] },
fileUrl: { startsWith: 'https://' },
},
});
// 删除旧的 zip 文件资产记录(如果存在)
await tx.materialAsset.deleteMany({
where: {
materialId,
assetType: { in: ['main', 'sku', 'detail', 'all'] },
fileUrl: { startsWith: 'uploads/' },
},
});
// 6.1 创建新的 *-preview 记录(用于前端展示)
const newAssetRecords: any[] = [];
// 主图预览
mainAssets.forEach((asset, index) => {
newAssetRecords.push({
materialId,
assetType: 'main-preview',
fileUrl: asset.fileUrl,
fileName: asset.fileName,
fileSize: asset.fileSize,
sortOrder: index,
});
});
// SKU图预览
skuAssets.forEach((asset, index) => {
newAssetRecords.push({
materialId,
assetType: 'sku-preview',
fileUrl: asset.fileUrl,
fileName: asset.fileName,
fileSize: asset.fileSize,
sortOrder: index,
});
});
// 详情图预览
detailAssets.forEach((asset, index) => {
newAssetRecords.push({
materialId,
assetType: 'detail-preview',
fileUrl: asset.fileUrl,
fileName: asset.fileName,
fileSize: asset.fileSize,
sortOrder: index,
});
});
// 6.2 新增 zip 文件资产记录
if (zipInfos.main) {
newAssetRecords.push({
materialId,
assetType: 'main',
fileUrl: zipInfos.main.filePath,
fileName: zipInfos.main.fileName,
fileSize: BigInt(zipInfos.main.fileSize),
sortOrder: 0,
});
}
if (zipInfos.sku) {
newAssetRecords.push({
materialId,
assetType: 'sku',
fileUrl: zipInfos.sku.filePath,
fileName: zipInfos.sku.fileName,
fileSize: BigInt(zipInfos.sku.fileSize),
sortOrder: 0,
});
}
if (zipInfos.detail) {
newAssetRecords.push({
materialId,
assetType: 'detail',
fileUrl: zipInfos.detail.filePath,
fileName: zipInfos.detail.fileName,
fileSize: BigInt(zipInfos.detail.fileSize),
sortOrder: 0,
});
}
newAssetRecords.push({
materialId,
assetType: 'all',
fileUrl: zipInfos.all.filePath,
fileName: zipInfos.all.fileName,
fileSize: BigInt(zipInfos.all.fileSize),
sortOrder: 0,
});
// 批量创建新资产记录
await tx.materialAsset.createMany({
data: newAssetRecords,
});
// 6.3 更新素材记录
await tx.material.update({
where: { id: materialId },
data: {
zipFilePath: zipInfos.all.filePath,
zipFileName: zipInfos.all.fileName,
zipFileSize: BigInt(zipInfos.all.fileSize),
},
});
console.log(`[${materialId}] 数据库更新完成: 新增 ${newAssetRecords.length} 条资产记录`);
});
} else {
console.log(`[${materialId}] [DRY RUN] 跳过数据库保存`);
}
// 7. 清理临时文件
for (const file of tempFiles) {
try {
await unlink(file);
} catch (err) {
console.warn(`[${materialId}] 清理临时文件失败: ${file}`);
}
}
log.status = 'success';
log.endTime = new Date();
console.log(`[${materialId}] ✅ 迁移成功`);
} catch (error) {
// 清理临时文件
for (const file of tempFiles) {
try {
await unlink(file);
} catch (err) {
// 忽略清理错误
}
}
throw error;
}
} catch (error) {
log.status = 'failed';
log.error = error instanceof Error ? error.message : String(error);
log.endTime = new Date();
console.error(`[${materialId}] ❌ 迁移失败: ${log.error}`);
}
return log;
}
/**
* 主函数
*/
async function main() {
console.log('='.repeat(60));
console.log('任务-详情老素材迁移脚本');
console.log('='.repeat(60));
console.log(`配置: 批量大小=${CONFIG.batchSize}, 演练模式=${CONFIG.dryRun}`);
console.log('');
try {
// 1. 查询需要迁移的素材
// 查询条件:有 S3 预览图但没有本地 zip 文件的素材
const allMaterials = await prisma.material.findMany({
where: {
materialType: '任务-详情',
deletedAt: null,
...(CONFIG.targetProductModel ? { productModel: CONFIG.targetProductModel } : {}),
},
select: {
id: true,
productModel: true,
taskNo: true,
assets: {
select: {
assetType: true,
fileUrl: true,
},
},
},
orderBy: { createdAt: 'asc' },
});
// 过滤出有 S3 预览图但没有本地 zip 文件的素材
const materials = allMaterials
.filter(m => {
const hasS3Previews = m.assets.some(a =>
['main', 'sku', 'detail'].includes(a.assetType) && a.fileUrl.startsWith('https://')
);
const hasLocalZips = m.assets.some(a =>
['main', 'sku', 'detail', 'all'].includes(a.assetType) && a.fileUrl.startsWith('uploads/')
);
return hasS3Previews && !hasLocalZips;
})
.slice(0, CONFIG.batchSize)
.map(m => ({ id: m.id, productModel: m.productModel, taskNo: m.taskNo }));
console.log(`找到 ${materials.length} 个需要迁移的素材`);
console.log('');
if (materials.length === 0) {
console.log('没有需要迁移的素材');
return;
}
// 2. 逐个迁移
for (let i = 0; i < materials.length; i++) {
const material = materials[i];
console.log(`\n进度: ${i + 1}/${materials.length}`);
const log = await migrateMaterial(material.id);
migrationLogs.push(log);
// 短暂延迟,避免过载
await new Promise((resolve) => setTimeout(resolve, 1000));
}
// 3. 输出统计
console.log('\n' + '='.repeat(60));
console.log('迁移完成统计');
console.log('='.repeat(60));
const successCount = migrationLogs.filter((l) => l.status === 'success').length;
const failedCount = migrationLogs.filter((l) => l.status === 'failed').length;
console.log(`总计: ${migrationLogs.length} 个素材`);
console.log(`成功: ${successCount} 个`);
console.log(`失败: ${failedCount} 个`);
if (failedCount > 0) {
console.log('\n失败的素材:');
migrationLogs
.filter((l) => l.status === 'failed')
.forEach((l) => {
console.log(` - ${l.productModel} (${l.materialId}): ${l.error}`);
});
}
// 4. 保存日志到文件
const logFileName = `migration-log-${Date.now()}.json`;
const logFilePath = path.join(process.cwd(), 'logs', logFileName);
await mkdir(path.dirname(logFilePath), { recursive: true });
await writeFile(logFilePath, JSON.stringify(migrationLogs, null, 2));
console.log(`\n日志已保存到: ${logFilePath}`);
} catch (error) {
console.error('迁移过程中发生错误:', error);
process.exit(1);
} finally {
await prisma.$disconnect();
}
}
// 执行
main();