gongxiangp2/lib/task-material-processor.ts
chenpei cd6e40d9d2 优化:网络传输性能和稳定性 - TCP缓冲区+并发控制优化
主要改动:
1. TCP缓冲区优化(16MB)- 解决公网上传速度慢问题
2. 并发数降低到1-3 - 避免4并发导致ERR_CONNECTION_RESET
3. 动态并发控制 - 根据网络环境自动调整
4. Nginx超时增加到120s - 支持大分片上传
5. 上传进度修复 - 修复剩余时间计算
6. 新增任务-页面上传器和详情组件
7. 统一部署文档为deploy-remote-build.ps1

问题修复:
- 公网用户上传从200-900KB/s提升到预期3-5MB/s
- 内网用户上传更稳定,无连接重置错误
- UploadProgress组件剩余时间计算修正

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-27 09:33:09 +08:00

723 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* 任务类型素材的通用处理逻辑
* 用于:任务-3D原图、任务-AI原图、任务-实拍原图、任务-视频截图、任务-详情
*/
import { prisma } from '@/lib/prisma';
import { uploadToS3, deleteFromS3 } from '@/lib/s3';
import { compressImage } from '@/lib/image-processor';
import { mkdir, unlink, readFile, rm } from 'fs/promises';
import { createWriteStream } from 'fs';
import path from 'path';
import archiver from 'archiver';
import { deleteUploadSession } from '@/lib/upload-sessions-redis';
// 资产类型常量
export const ASSET_TYPE = {
PREVIEW: 'preview',
MAIN_PREVIEW: 'main-preview',
SKU_PREVIEW: 'sku-preview',
DETAIL_PREVIEW: 'detail-preview',
ZIP: 'zip',
MAIN: 'main',
SKU: 'sku',
DETAIL: 'detail',
ALL: 'all',
PSD: 'psd',
} as const;
/**
* 从chunk临时目录打包文件成zip
*/
export async function createZipFromFiles(
files: Array<{ path: string; name: string; size: number }>,
materialId: string,
productModel: string,
materialType: string,
zipType: string = 'all', // 'main', 'sku', 'detail', 'all'
authorName?: string, // 作者名(上传人姓名)
categoryName?: string // 店铺名
): Promise<{ filePath: string; fileName: string; fileSize: number }> {
if (!files || files.length === 0) {
throw new Error('没有文件需要打包');
}
// 验证所有文件路径存在
const { access } = await import('fs/promises');
for (const file of files) {
try {
await access(file.path);
} catch (error) {
throw new Error(`文件不存在: ${file.path}`);
}
}
// 创建目标目录
const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
await mkdir(uploadsDir, { recursive: true });
// 生成zip文件名和路径(保留中文,只清理文件系统不安全字符)
const timestamp = Date.now();
// 只清理文件系统不安全的字符:/ \ : * ? " < > |
const sanitizedProductModel = productModel.replace(/[\/\\:*?"<>|]/g, '_');
// 如果提供了作者名和店铺名,使用新的命名格式:SPU型号_作者名_店铺名_类型.zip
let zipFileName: string;
if (authorName && categoryName && zipType !== 'all') {
const sanitizedAuthorName = authorName.replace(/[\/\\:*?"<>|]/g, '_');
const sanitizedCategoryName = categoryName.replace(/[\/\\:*?"<>|]/g, '_');
zipFileName = `${sanitizedProductModel}_${sanitizedAuthorName}_${sanitizedCategoryName}_${zipType}.zip`;
} else {
// 保持原有命名格式(用于all.zip或其他情况)
zipFileName = `${sanitizedProductModel}_${zipType}_${materialId}_${timestamp}.zip`;
}
const zipFilePath = path.join(uploadsDir, zipFileName);
const relativeZipPath = path.join('uploads', 'materials', zipFileName);
console.log(`[Zip] Creating ${zipType} zip file for ${materialType}:`, zipFilePath);
console.log(`[Zip] Files count:`, files.length);
return new Promise((resolve, reject) => {
let output: any = null;
let archive: any = null;
const timeout = setTimeout(() => {
cleanup();
reject(new Error('打包超时(30秒)'));
}, 30000); // 30秒超时
// 清理函数
const cleanup = () => {
try {
clearTimeout(timeout);
if (output) {
output.removeAllListeners();
output.destroy();
output = null; // ✅ 置空引用
}
if (archive) {
archive.removeAllListeners();
archive.destroy();
archive = null; // ✅ 置空引用
}
} catch (err) {
console.error('[Zip Cleanup] Error:', err);
}
};
try {
// 创建输出流
output = createWriteStream(zipFilePath);
archive = archiver('zip', {
zlib: { level: 9 } // 最高压缩级别
});
// 监听完成事件
output.on('close', () => {
clearTimeout(timeout);
const fileSize = archive.pointer();
console.log(`[Zip] ${zipType} zip file created successfully: ${fileSize} bytes`);
cleanup();
resolve({
filePath: relativeZipPath,
fileName: zipFileName,
fileSize
});
});
// 监听错误
archive.on('error', (err: Error) => {
clearTimeout(timeout);
console.error('[Zip] Archive error:', err);
cleanup();
reject(new Error(`打包失败: ${err.message}`));
});
output.on('error', (err: Error) => {
clearTimeout(timeout);
console.error('[Zip] Output stream error:', err);
cleanup();
reject(new Error(`写入文件失败: ${err.message}`));
});
// 监听警告
archive.on('warning', (err: any) => {
if (err.code === 'ENOENT') {
console.warn('[Zip] Warning:', err);
} else {
clearTimeout(timeout);
cleanup();
reject(err);
}
});
// 连接archive到输出流
archive.pipe(output);
// 添加每个文件到zip
for (const file of files) {
console.log(`[Zip] Adding file: ${file.name} (${file.size} bytes)`);
archive.file(file.path, { name: file.name });
}
// 完成压缩
archive.finalize();
} catch (error) {
clearTimeout(timeout);
console.error('[Zip] Error:', error);
reject(error);
}
});
}
/**
* 打包已有的zip文件成一个总zip(用于任务-详情的all.zip)
* 注意:不再打包PSD文件,PSD文件单独存储和下载
*/
export async function createZipFromZips(
zipFiles: Array<{ path: string; name: string }>,
materialId: string,
productModel: string,
materialType: string
): Promise<{ filePath: string; fileName: string; fileSize: number }> {
if (zipFiles.length === 0) {
throw new Error('没有文件需要打包');
}
// 验证所有文件路径存在
const { access } = await import('fs/promises');
for (const file of zipFiles) {
try {
const fullPath = path.isAbsolute(file.path) ? file.path : path.join(process.cwd(), file.path);
await access(fullPath);
} catch (error) {
throw new Error(`文件不存在: ${file.path}`);
}
}
// 创建目标目录
const uploadsDir = path.join(process.cwd(), 'uploads', 'materials');
await mkdir(uploadsDir, { recursive: true });
// 生成zip文件名和路径
const timestamp = Date.now();
const sanitizedProductModel = productModel.replace(/[\/\\:*?"<>|]/g, '_');
const zipFileName = `${sanitizedProductModel}_all_${materialId}_${timestamp}.zip`;
const zipFilePath = path.join(uploadsDir, zipFileName);
const relativeZipPath = path.join('uploads', 'materials', zipFileName);
console.log(`[Zip] Creating all.zip for ${materialType}:`, zipFilePath);
console.log(`[Zip] Zip files count:`, zipFiles.length);
return new Promise((resolve, reject) => {
let output: any = null;
let archive: any = null;
const timeout = setTimeout(() => {
cleanup();
reject(new Error('打包超时(60秒)'));
}, 60000); // 60秒超时(打包多个zip需要更长时间)
// 清理函数
const cleanup = () => {
try {
clearTimeout(timeout);
if (output) {
output.removeAllListeners();
output.destroy();
output = null; // ✅ 置空引用
}
if (archive) {
archive.removeAllListeners();
archive.destroy();
archive = null; // ✅ 置空引用
}
} catch (err) {
console.error('[Zip Cleanup] Error:', err);
}
};
try {
output = createWriteStream(zipFilePath);
archive = archiver('zip', {
zlib: { level: 9 } // 最高压缩级别
});
output.on('close', () => {
clearTimeout(timeout);
const fileSize = archive.pointer();
console.log(`[Zip] all.zip created successfully: ${fileSize} bytes`);
cleanup();
resolve({
filePath: relativeZipPath,
fileName: zipFileName,
fileSize
});
});
archive.on('error', (err: Error) => {
clearTimeout(timeout);
console.error('[Zip] Archive error:', err);
cleanup();
reject(new Error(`打包失败: ${err.message}`));
});
output.on('error', (err: Error) => {
clearTimeout(timeout);
console.error('[Zip] Output stream error:', err);
cleanup();
reject(new Error(`写入文件失败: ${err.message}`));
});
archive.on('warning', (err: any) => {
if (err.code === 'ENOENT') {
console.warn('[Zip] Warning:', err);
} else {
clearTimeout(timeout);
cleanup();
reject(err);
}
});
archive.pipe(output);
// 只添加zip文件,不添加PSD文件
for (const file of zipFiles) {
const fullPath = path.isAbsolute(file.path) ? file.path : path.join(process.cwd(), file.path);
console.log(`[Zip] Adding zip file: ${file.name}`);
archive.file(fullPath, { name: file.name });
}
archive.finalize();
} catch (error) {
clearTimeout(timeout);
console.error('[Zip] Error:', error);
reject(error);
}
});
}
/**
* 为图片生成预览图并上传到S3
* 注意:必须为所有图片生成预览图,如果任何一张失败则整个流程失败
*/
export async function generatePreviewImages(
uploadIds: Array<{ uploadId: string; fileName: string }>,
materialType: string,
compressOptions?: {
maxWidth?: number;
maxHeight?: number;
maxSizeKB?: number;
}
): Promise<string[]> {
if (!uploadIds || uploadIds.length === 0) {
return [];
}
const { getUploadedFile } = await import('@/lib/upload-sessions');
const uploadedPreviewUrls: string[] = [];
const errors: string[] = [];
// 默认压缩参数(任务-3D原图使用)
const defaultOptions = {
maxWidth: 1440,
maxHeight: 2400,
maxSizeKB: 500,
};
// 合并压缩参数
const options = { ...defaultOptions, ...compressOptions };
console.log(`[Preview] Generating ${uploadIds.length} preview images for ${materialType}...`);
console.log(`[Preview] Compress options:`, options);
for (let i = 0; i < uploadIds.length; i++) {
const item = uploadIds[i];
try {
const filePath = await getUploadedFile(item.uploadId);
if (!filePath) {
const error = `文件不存在: ${item.fileName}`;
console.error(`[Preview] ${error}`);
errors.push(error);
continue;
}
const buffer = await readFile(filePath);
// 验证是否为图片
if (!buffer || buffer.length === 0) {
const error = `文件为空: ${item.fileName}`;
console.error(`[Preview] ${error}`);
errors.push(error);
continue;
}
// 压缩并上传预览图
const compressedBuffer = await compressImage(buffer, {
maxWidth: options.maxWidth,
maxHeight: options.maxHeight,
maxSizeKB: options.maxSizeKB,
format: 'webp',
});
const timestamp = Date.now();
const randomSuffix = Math.random().toString(36).substring(2, 9);
const fileName = `preview-${timestamp}-${i}-${randomSuffix}.webp`;
const s3Path = `product-package/${fileName}`;
const previewUrl = await uploadToS3(compressedBuffer, s3Path, 'image/webp');
uploadedPreviewUrls.push(previewUrl);
console.log(`[Preview] Generated preview ${i + 1}/${uploadIds.length}:`, previewUrl);
} catch (err) {
const error = `生成预览图失败 ${item.fileName}: ${err instanceof Error ? err.message : String(err)}`;
console.error(`[Preview] ${error}`);
errors.push(error);
}
}
// 检查是否所有预览图都生成成功
if (errors.length > 0) {
console.error(`[Preview] Failed to generate ${errors.length} preview images:`);
errors.forEach(err => console.error(` - ${err}`));
throw new Error(`预览图生成失败:${errors.length}/${uploadIds.length} 张失败。详情:${errors.join('; ')}`);
}
// 验证生成的预览图数量是否与原图数量一致
if (uploadedPreviewUrls.length !== uploadIds.length) {
throw new Error(`预览图数量不匹配:期望 ${uploadIds.length} 张,实际生成 ${uploadedPreviewUrls.length} 张`);
}
console.log(`[Preview] Successfully generated all ${uploadedPreviewUrls.length} preview images`);
return uploadedPreviewUrls;
}
/**
* 清理临时文件和目录
*/
export async function cleanupTempFiles(
tempFiles: string[],
uploadIds: string[]
): Promise<void> {
// 清理临时文件
if (tempFiles.length > 0) {
console.log(`[Cleanup] Cleaning up ${tempFiles.length} temporary files...`);
for (const filePath of tempFiles) {
try {
await unlink(filePath);
console.log(`[Cleanup] Deleted temp file: ${filePath}`);
} catch (err) {
console.warn(`[Cleanup] Failed to delete temp file: ${filePath}`, err);
}
}
}
// 清理chunk临时目录
if (uploadIds.length > 0) {
console.log(`[Cleanup] Cleaning up ${uploadIds.length} chunk temp directories...`);
for (const uploadId of uploadIds) {
const tempDir = path.join(process.env.UPLOAD_DIR || './uploads', 'temp', uploadId);
try {
await rm(tempDir, { recursive: true, force: true });
console.log(`[Cleanup] Deleted chunk temp dir: ${tempDir}`);
} catch (err) {
console.warn(`[Cleanup] Failed to delete chunk temp dir: ${tempDir}`);
}
}
}
// 清理Redis上传会话(关键修复:防止孤立会话堆积)
if (uploadIds.length > 0) {
console.log(`[Cleanup] Cleaning up ${uploadIds.length} Redis upload sessions...`);
for (const uploadId of uploadIds) {
try {
await deleteUploadSession(uploadId);
console.log(`[Cleanup] Deleted Redis session: ${uploadId}`);
} catch (err) {
console.warn(`[Cleanup] Failed to delete Redis session: ${uploadId}`, err);
}
}
}
}
/**
* 清理失败时的资源(S3文件、本地zip、临时文件、数据库记录)
*/
export async function cleanupOnError(
uploadedPreviewUrls: string[],
zipFilePaths: string | string[] | null,
tempFiles: string[],
materialId: string | null
): Promise<void> {
console.log('[Cleanup] Starting error cleanup...');
// 清理已上传的S3预览图
if (uploadedPreviewUrls.length > 0) {
console.log(`[Cleanup] Cleaning up ${uploadedPreviewUrls.length} S3 preview images...`);
for (const fileUrl of uploadedPreviewUrls) {
try {
await deleteFromS3(fileUrl);
console.log(`[Cleanup] Deleted S3 file: ${fileUrl}`);
} catch (cleanupError) {
console.error(`[Cleanup] Failed to delete S3 file ${fileUrl}:`, cleanupError);
}
}
}
// 清理本地zip文件(支持单个或多个)
if (zipFilePaths) {
const paths = Array.isArray(zipFilePaths) ? zipFilePaths : [zipFilePaths];
console.log(`[Cleanup] Cleaning up ${paths.length} local zip files...`);
for (const zipFilePath of paths) {
try {
const fullPath = path.join(process.cwd(), zipFilePath);
await unlink(fullPath);
console.log(`[Cleanup] Deleted local zip file: ${fullPath}`);
} catch (cleanupError) {
console.error('[Cleanup] Failed to delete local zip file:', cleanupError);
}
}
}
// 清理临时文件
if (tempFiles.length > 0) {
for (const filePath of tempFiles) {
try {
await unlink(filePath);
} catch (cleanupError) {
console.warn(`[Cleanup] Failed to delete temp file: ${filePath}`);
}
}
}
// 删除素材记录
if (materialId) {
try {
await prisma.material.delete({
where: { id: materialId },
});
console.log(`[Cleanup] Deleted material record: ${materialId}`);
} catch (cleanupError) {
console.error('[Cleanup] Failed to delete material record:', cleanupError);
}
}
console.log('[Cleanup] Error cleanup completed');
}
/**
* 保存资产记录到数据库
*/
export async function saveAssets(
materialId: string,
previewUrls: string[],
zipInfo: { filePath: string; fileName: string; fileSize: number } | null
): Promise<void> {
const assetRecords: Array<{
materialId: string;
assetType: string;
fileUrl: string;
fileName?: string;
fileSize?: bigint;
sortOrder?: number;
}> = [];
// 添加预览图资产
if (previewUrls.length > 0) {
previewUrls.forEach((url, index) => {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.PREVIEW,
fileUrl: url,
fileName: `preview-${index}.webp`,
sortOrder: index,
});
});
}
// 添加zip资产
if (zipInfo) {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.ZIP,
fileUrl: zipInfo.filePath,
fileName: zipInfo.fileName,
fileSize: BigInt(zipInfo.fileSize),
sortOrder: 0,
});
}
// 使用事务保存
await prisma.$transaction(async (tx) => {
// 批量创建资产记录
if (assetRecords.length > 0) {
await tx.materialAsset.createMany({
data: assetRecords,
});
}
// 更新素材记录
const firstPreview = previewUrls[0] || '';
await tx.material.update({
where: { id: materialId },
data: {
previewUrl: firstPreview,
zipFilePath: zipInfo?.filePath || '',
zipFileName: zipInfo?.fileName || '',
zipFileSize: zipInfo?.fileSize ? BigInt(zipInfo.fileSize) : BigInt(0),
},
});
});
console.log(`[Database] Saved ${assetRecords.length} asset records`);
}
/**
* 保存任务-详情的资产记录到数据库
* 包括:预览图(S3)、分类zip(本地)、总zip(本地)、PSD(本地)
*/
export async function saveProductPackageAssets(
materialId: string,
previewUrls: {
main: string[];
sku: string[];
detail: string[];
},
zipInfos: {
main?: { filePath: string; fileName: string; fileSize: number };
sku?: { filePath: string; fileName: string; fileSize: number };
detail?: { filePath: string; fileName: string; fileSize: number };
all: { filePath: string; fileName: string; fileSize: number };
},
psdInfos?: Array<{ filePath: string; fileName: string; fileSize: number }>
): Promise<void> {
const assetRecords: Array<{
materialId: string;
assetType: string;
fileUrl: string;
fileName?: string;
fileSize?: bigint;
sortOrder?: number;
}> = [];
// 添加预览图资产(所有预览图)
let previewIndex = 0;
// 主图预览
previewUrls.main.forEach((url, index) => {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.MAIN_PREVIEW,
fileUrl: url,
fileName: `main-preview-${index}.webp`,
sortOrder: previewIndex++,
});
});
// SKU图预览
previewUrls.sku.forEach((url, index) => {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.SKU_PREVIEW,
fileUrl: url,
fileName: `sku-preview-${index}.webp`,
sortOrder: previewIndex++,
});
});
// 详情图预览
previewUrls.detail.forEach((url, index) => {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.DETAIL_PREVIEW,
fileUrl: url,
fileName: `detail-preview-${index}.webp`,
sortOrder: previewIndex++,
});
});
// 添加分类zip资产
if (zipInfos.main) {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.MAIN,
fileUrl: zipInfos.main.filePath,
fileName: zipInfos.main.fileName,
fileSize: BigInt(zipInfos.main.fileSize),
sortOrder: 0,
});
}
if (zipInfos.sku) {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.SKU,
fileUrl: zipInfos.sku.filePath,
fileName: zipInfos.sku.fileName,
fileSize: BigInt(zipInfos.sku.fileSize),
sortOrder: 0,
});
}
if (zipInfos.detail) {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.DETAIL,
fileUrl: zipInfos.detail.filePath,
fileName: zipInfos.detail.fileName,
fileSize: BigInt(zipInfos.detail.fileSize),
sortOrder: 0,
});
}
// 添加总zip资产
assetRecords.push({
materialId,
assetType: ASSET_TYPE.ALL,
fileUrl: zipInfos.all.filePath,
fileName: zipInfos.all.fileName,
fileSize: BigInt(zipInfos.all.fileSize),
sortOrder: 0,
});
// 添加PSD资产
if (psdInfos && psdInfos.length > 0) {
psdInfos.forEach((psdInfo, index) => {
assetRecords.push({
materialId,
assetType: ASSET_TYPE.PSD,
fileUrl: psdInfo.filePath,
fileName: psdInfo.fileName,
fileSize: BigInt(psdInfo.fileSize),
sortOrder: index,
});
});
}
// 使用事务保存
await prisma.$transaction(async (tx) => {
// 批量创建资产记录
if (assetRecords.length > 0) {
await tx.materialAsset.createMany({
data: assetRecords,
});
}
// 更新素材记录(zip信息,previewUrl在Step 6中更新)
await tx.material.update({
where: { id: materialId },
data: {
zipFilePath: zipInfos.all.filePath,
zipFileName: zipInfos.all.fileName,
zipFileSize: BigInt(zipInfos.all.fileSize),
},
});
});
console.log(`[Database] Saved ${assetRecords.length} asset records for product package`);
}