diff --git a/app/api/external/download/route.ts b/app/api/external/download/route.ts index 9dba40ed..8eaeb20d 100644 --- a/app/api/external/download/route.ts +++ b/app/api/external/download/route.ts @@ -15,8 +15,9 @@ import fs from 'fs'; import { createReadStream } from 'fs'; import { access, stat } from 'fs/promises'; import { lookup, extension } from 'mime-types'; -import JSZip from 'jszip'; import { isProductPackageType } from '@/lib/material-type-mapper'; +import { nodeReadableToWebStream } from '@/lib/node-readable-stream'; +import { createArchiveDownloadStream } from '@/lib/archive-download-stream'; import { getVideoDownloadAvailability, getVideoDownloadUnavailableMessage, @@ -293,8 +294,8 @@ async function downloadStandardModelWhiteBackground( const whiteBackgroundGroup = groupWhiteBackgroundAssetsBySku(material.assets || []).get(sku.id); if (whiteBackgroundGroup && hasAnyWhiteBackgroundAssetGroup(whiteBackgroundGroup)) { - const zipBuffer = await buildExternalWhiteBackgroundZipBuffer(whiteBackgroundGroup); - if (zipBuffer.length === 0) { + const zipStream = await buildExternalWhiteBackgroundZipStream(whiteBackgroundGroup); + if (!zipStream) { return NextResponse.json( { success: false, error: { code: 'FILE_NOT_FOUND', message: '白底图不存在' } }, { status: 404 } @@ -302,11 +303,10 @@ async function downloadStandardModelWhiteBackground( } const fileName = getWhiteBackgroundZipFileName(sku.skuName); - return new NextResponse(zipBuffer as any, { + return new NextResponse(zipStream, { headers: { 'Content-Type': 'application/zip', 'Content-Disposition': `attachment; filename="${encodeURIComponent(fileName)}"`, - 'Content-Length': zipBuffer.length.toString(), }, }); } @@ -367,22 +367,7 @@ async function downloadSingleFile(asset: any) { } // 使用流式传输，避免将整个文件加载到内存中（修复 R24） - const fileStream = createReadStream(localPath); - // N6修复：确保文件流在任何情况下都能正确清理 - const stream = new ReadableStream({ - start(controller) { - fileStream.on('data', (chunk) => controller.enqueue(chunk)); - fileStream.on('end', () => controller.close()); - fileStream.on('error', (err) => { - // 错误时也要销毁文件流，防止文件描述符泄漏 - fileStream.destroy(); - controller.error(err); - }); - }, - cancel() { - fileStream.destroy(); - }, - }); + const stream = nodeReadableToWebStream(createReadStream(localPath)); const contentType = getContentType(fileName); const fileStats = await stat(localPath); @@ -410,16 +395,7 @@ async function downloadSingleFile(asset: any) { const contentType = response.headers.get('content-type') || getContentType(fileName); const contentLength = response.headers.get('content-length'); const nodeStream = Readable.fromWeb(response.body as any); - const stream = new ReadableStream({ - start(controller) { - nodeStream.on('data', (chunk) => controller.enqueue(chunk)); - nodeStream.on('end', () => controller.close()); - nodeStream.on('error', (error) => controller.error(error)); - }, - cancel() { - nodeStream.destroy(); - }, - }); + const stream = nodeReadableToWebStream(nodeStream); return new NextResponse(stream as any, { headers: { @@ -443,55 +419,72 @@ async function downloadSingleFile(asset: any) { ); } -async function buildExternalWhiteBackgroundZipBuffer(whiteBackgroundGroup: { +async function buildExternalWhiteBackgroundZipStream(whiteBackgroundGroup: { fixed: Record; other: any[]; -}) { - const zip = new JSZip(); +}): Promise | null> { + const archive = archiver('zip', { zlib: { level: 6 } }); + const archiveDownload = createArchiveDownloadStream(archive); let addedCount = 0 - for (const view of STANDARD_MODEL_WHITE_BG_VIEWS) { - const asset = whiteBackgroundGroup.fixed[view.key]; - if (!asset?.fileUrl) { - continue; + try { + for (const view of STANDARD_MODEL_WHITE_BG_VIEWS) { + const asset = whiteBackgroundGroup.fixed[view.key]; + if (!asset?.fileUrl) { + continue; + } + + const ext = path.extname(asset.fileName || asset.fileUrl) || '.jpg'; + const fileName = asset.fileName || `${view.label}${ext}`; + await appendExternalAssetToArchive( + archive, + archiveDownload.trackSource, + asset.fileUrl, + fileName, + ); + addedCount += 1 } - const fileBuffer = await readExternalAssetBuffer(asset.fileUrl); - const ext = path.extname(asset.fileName || asset.fileUrl) || '.jpg'; - const fileName = asset.fileName || `${view.label}${ext}`; - zip.file(fileName, fileBuffer); - addedCount += 1 - } + for (const asset of whiteBackgroundGroup.other || []) { + if (!asset?.fileUrl) { + continue; + } - for (const asset of whiteBackgroundGroup.other || []) { - if (!asset?.fileUrl) { - continue; + await appendExternalAssetToArchive( + archive, + archiveDownload.trackSource, + asset.fileUrl, + `other/${asset.fileName || path.basename(asset.fileUrl)}`, + ); + addedCount += 1 } - const fileBuffer = await readExternalAssetBuffer(asset.fileUrl); - zip.file(`other/${asset.fileName || path.basename(asset.fileUrl)}`, fileBuffer); - addedCount += 1 - } + if (addedCount === 0) { + archiveDownload.abort(); + return null; + } - if (addedCount === 0) { - return Buffer.alloc(0); + return archiveDownload.finalize(); + } catch (error) { + archiveDownload.abort(); + throw error; } - - return zip.generateAsync({ - type: 'nodebuffer', - compression: 'DEFLATE', - compressionOptions: { level: 6 }, - }); } -async function readExternalAssetBuffer(fileUrl: string): Promise { +async function appendExternalAssetToArchive( + archive: archiver.Archiver, + trackSource: (source: T) => T, + fileUrl: string, + entryName: string, +): Promise { if (isHttpUrl(fileUrl)) { const response = await fetch(fileUrl); - if (!response.ok) { + if (!response.ok || !response.body) { throw new Error(`读取资源失败: ${response.status}`); } - return Buffer.from(await response.arrayBuffer()); + archive.append(trackSource(Readable.fromWeb(response.body as any)), { name: entryName }); + return; } const localPath = resolveAssetLocalPath(fileUrl); @@ -499,7 +492,8 @@ async function readExternalAssetBuffer(fileUrl: string): Promise { throw new Error('资源路径无效'); } - return fs.promises.readFile(localPath); + await access(localPath); + archive.file(localPath, { name: entryName }); } /** @@ -514,6 +508,7 @@ async function downloadAsZip(assets: any[], taskNo: string, downloadType: string }); const archive = archiver('zip', { zlib: { level: 5 } }); + const archiveDownload = createArchiveDownloadStream(archive); let successCount = 0; let failCount = 0; @@ -538,7 +533,10 @@ async function downloadAsZip(assets: any[], taskNo: string, downloadType: string if (isHttpUrl(fileUrl)) { const response = await fetch(fileUrl); if (response.ok && response.body) { - archive.append(Readable.fromWeb(response.body as any), { name: entryName }); + archive.append( + archiveDownload.trackSource(Readable.fromWeb(response.body as any)), + { name: entryName }, + ); successCount++; continue; } @@ -554,39 +552,14 @@ async function downloadAsZip(assets: any[], taskNo: string, downloadType: string } if (successCount === 0) { - archive.destroy(); + archiveDownload.abort(); return NextResponse.json( { success: false, error: { code: 'NO_FILES', message: '没有可下载的文件' } }, { status: 404 } ); } - const stream = new ReadableStream({ - start(controller) { - let settled = false; - const close = () => { - if (!settled) { - settled = true; - console.log(`[downloadAsZip] 打包完成: 成功 ${successCount} 个, 失败 ${failCount} 个`); - controller.close(); - } - }; - const fail = (error: Error) => { - if (!settled) { - settled = true; - controller.error(error); - } - }; - - archive.on('data', (chunk) => controller.enqueue(chunk)); - archive.on('end', close); - archive.on('error', fail); - void archive.finalize().catch(fail); - }, - cancel() { - archive.abort(); - }, - }); + const stream = archiveDownload.finalize(); return new NextResponse(stream as any, { headers: { @@ -703,22 +676,7 @@ async function downloadNonProductPackageMaterial(material: any, downloadType: st console.log(`[external/download] 使用本地 ZIP 文件: ${localZipPath}`); - const fileStream = createReadStream(localZipPath); - // N6修复：确保文件流在任何情况下都能正确清理 - const stream = new ReadableStream({ - start(controller) { - fileStream.on('data', (chunk) => controller.enqueue(chunk)); - fileStream.on('end', () => controller.close()); - fileStream.on('error', (err) => { - // 错误时也要销毁文件流，防止文件描述符泄漏 - fileStream.destroy(); - controller.error(err); - }); - }, - cancel() { - fileStream.destroy(); - }, - }); + const stream = nodeReadableToWebStream(createReadStream(localZipPath)); const ext = path.extname(material.zipFileName || localZipPath) || '.zip'; const mimeType = lookup(ext) || 'application/octet-stream'; @@ -779,16 +737,7 @@ async function downloadNonProductPackageMaterial(material: any, downloadType: st } const nodeStream = Readable.fromWeb(response.body as any); - const stream = new ReadableStream({ - start(controller) { - nodeStream.on('data', (chunk) => controller.enqueue(chunk)); - nodeStream.on('end', () => controller.close()); - nodeStream.on('error', (err) => controller.error(err)); - }, - cancel() { - nodeStream.destroy(); - }, - }); + const stream = nodeReadableToWebStream(nodeStream); return new NextResponse(stream as any, { status: 200, @@ -865,16 +814,7 @@ async function downloadNonProductPackageMaterial(material: any, downloadType: st const filename = `${filenameParts.join('_')}${finalExt}`; const nodeStream = Readable.fromWeb(response.body as any); - const stream = new ReadableStream({ - start(controller) { - nodeStream.on('data', (chunk) => controller.enqueue(chunk)); - nodeStream.on('end', () => controller.close()); - nodeStream.on('error', (err) => controller.error(err)); - }, - cancel() { - nodeStream.destroy(); - }, - }); + const stream = nodeReadableToWebStream(nodeStream); return new NextResponse(stream as any, { status: 200, @@ -989,23 +929,7 @@ async function downloadNonProductPackageMaterial(material: any, downloadType: st } const chunkSize = end - start + 1; - const fileStream = createReadStream(filePath, { start, end }); - - // N6修复：确保文件流在任何情况下都能正确清理 - const stream = new ReadableStream({ - start(controller) { - fileStream.on('data', (chunk) => controller.enqueue(chunk)); - fileStream.on('end', () => controller.close()); - fileStream.on('error', (err) => { - // 错误时也要销毁文件流，防止文件描述符泄漏 - fileStream.destroy(); - controller.error(err); - }); - }, - cancel() { - fileStream.destroy(); - }, - }); + const stream = nodeReadableToWebStream(createReadStream(filePath, { start, end })); return new NextResponse(stream, { status: 206, @@ -1020,23 +944,7 @@ async function downloadNonProductPackageMaterial(material: any, downloadType: st } // 正常下载 - const fileStream = createReadStream(filePath); - - // N6修复：确保文件流在任何情况下都能正确清理 - const stream = new ReadableStream({ - start(controller) { - fileStream.on('data', (chunk) => controller.enqueue(chunk)); - fileStream.on('end', () => controller.close()); - fileStream.on('error', (err) => { - // 错误时也要销毁文件流，防止文件描述符泄漏 - fileStream.destroy(); - controller.error(err); - }); - }, - cancel() { - fileStream.destroy(); - }, - }); + const stream = nodeReadableToWebStream(createReadStream(filePath)); return new NextResponse(stream, { headers: { @@ -1064,6 +972,7 @@ async function buildDynamicZipFromDisplayImages(material: any) { } const archive = archiver('zip', { zlib: { level: 6 } }); + const archiveDownload = createArchiveDownloadStream(archive); let addedCount = 0; for (let i = 0; i < displayImages.length; i++) { @@ -1077,7 +986,10 @@ async function buildDynamicZipFromDisplayImages(material: any) { const urlObj = new URL(imageUrl); const filename = (path.basename(urlObj.pathname) || `image-${i + 1}.webp`).replace(/[^\w\-.]+/g, '_'); - archive.append(Readable.fromWeb(response.body as any), { name: filename }); + archive.append( + archiveDownload.trackSource(Readable.fromWeb(response.body as any)), + { name: filename }, + ); addedCount++; } catch (error) { console.error(`[external/download] Error while adding display image index=${i} url=${imageUrl} to zip:`, error); @@ -1085,7 +997,7 @@ async function buildDynamicZipFromDisplayImages(material: any) { } if (addedCount === 0) { - archive.destroy(); + archiveDownload.abort(); return NextResponse.json( { success: false, error: { code: 'NO_DOWNLOADABLE_FILES', message: '该素材可能只包含图片' } }, { status: 404 } @@ -1102,31 +1014,7 @@ async function buildDynamicZipFromDisplayImages(material: any) { const filename = `${filenameParts.join('_')}.zip`; - const stream = new ReadableStream({ - start(controller) { - let settled = false; - const close = () => { - if (!settled) { - settled = true; - controller.close(); - } - }; - const fail = (error: Error) => { - if (!settled) { - settled = true; - controller.error(error); - } - }; - - archive.on('data', (chunk) => controller.enqueue(chunk)); - archive.on('end', close); - archive.on('error', fail); - void archive.finalize().catch(fail); - }, - cancel() { - archive.abort(); - }, - }); + const stream = archiveDownload.finalize(); return new NextResponse(stream as any, { headers: { diff --git a/app/api/materials/[id]/skus/[skuId]/download/route.ts b/app/api/materials/[id]/skus/[skuId]/download/route.ts index edba8606..f1f79ca6 100644 --- a/app/api/materials/[id]/skus/[skuId]/download/route.ts +++ b/app/api/materials/[id]/skus/[skuId]/download/route.ts @@ -1,10 +1,9 @@ 'use server' -import { promises as fs } from 'fs' import { createReadStream, existsSync } from 'fs' import { stat } from 'fs/promises' import path from 'path' -import JSZip from 'jszip' +import archiver from 'archiver' import { NextRequest, NextResponse } from 'next/server' import { getServerSession } from 'next-auth' import { authOptions } from '@/lib/auth' @@ -21,6 +20,8 @@ import { import { getUploadRootDir, resolveStoredUploadPath } from '@/lib/upload-paths' import { isPathInAllowedDirs } from '@/lib/path-security' import { canDownloadSkuMaterial } from '../../authorization' +import { nodeReadableToWebStream } from '@/lib/node-readable-stream' +import { createArchiveDownloadStream } from '@/lib/archive-download-stream' type DownloadType = 'model' | 'white-background' @@ -182,39 +183,33 @@ export async function GET( }) } - const zip = new JSZip() + const archive = archiver('zip', { zlib: { level: 6 } }) + const archiveDownload = createArchiveDownloadStream(archive) for (const entry of fixedEntries) { const filePath = resolveLocalFilePath(entry.fileUrl) if (!filePath) { return NextResponse.json({ error: '白底图文件路径无效' }, { status: 404 }) } - const fileBuffer = await fs.readFile(filePath) - zip.file(entry.fileName, fileBuffer) + archive.file(filePath, { name: entry.fileName }) } for (const entry of otherEntries) { const filePath = resolveLocalFilePath(entry.fileUrl) if (!filePath) { return NextResponse.json({ error: '白底图文件路径无效' }, { status: 404 }) } - const fileBuffer = await fs.readFile(filePath) - zip.file(`other/${entry.fileName}`, fileBuffer) + archive.file(filePath, { name: `other/${entry.fileName}` }) } - const zipBuffer = await zip.generateAsync({ - type: 'nodebuffer', - compression: 'DEFLATE', - compressionOptions: { level: 6 }, - }) + const stream = archiveDownload.finalize() recordSkuDownloadLog(materialId, userId).catch((error) => { console.error('[SKU Download] 记录下载日志失败:', error) }) - return new NextResponse(zipBuffer as any, { + return new NextResponse(stream, { headers: { 'Content-Type': 'application/zip', 'Content-Disposition': `attachment; filename="${encodeURIComponent(downloadFileName)}"`, - 'Content-Length': zipBuffer.length.toString(), }, }) } @@ -246,20 +241,7 @@ export async function GET( }) } - const fileStream = createReadStream(filePath) - const stream = new ReadableStream({ - start(controller) { - fileStream.on('data', (chunk) => controller.enqueue(chunk)) - fileStream.on('end', () => controller.close()) - fileStream.on('error', (err) => { - fileStream.destroy() - controller.error(err) - }) - }, - cancel() { - fileStream.destroy() - }, - }) + const stream = nodeReadableToWebStream(createReadStream(filePath)) recordSkuDownloadLog(materialId, userId).catch((error) => { console.error('[SKU Download] 记录下载日志失败:', error) diff --git a/app/api/materials/[id]/skus/batch-download/route.ts b/app/api/materials/[id]/skus/batch-download/route.ts index 1cbfd8bd..dbefb6e6 100644 --- a/app/api/materials/[id]/skus/batch-download/route.ts +++ b/app/api/materials/[id]/skus/batch-download/route.ts @@ -16,6 +16,7 @@ import { import { getUploadRootDir, resolveStoredUploadPath } from '@/lib/upload-paths' import { isPathInAllowedDirs } from '@/lib/path-security' import { canDownloadSkuMaterial } from '../authorization' +import { createArchiveDownloadStream } from '@/lib/archive-download-stream' type DownloadType = 'model' | 'white-background' @@ -224,35 +225,12 @@ export async function POST( const filename = `${material.productModel}_${downloadType === 'white-background' ? 'SKU白底图' : 'SKU模型'}_${Date.now()}.zip` const archive = archiver('zip', { zlib: { level: 6 } }) + const archiveDownload = createArchiveDownloadStream(archive) archiveEntries.forEach((entry) => { archive.file(entry.filePath, { name: entry.name }) }) - const stream = new ReadableStream({ - start(controller) { - let settled = false - const close = () => { - if (!settled) { - settled = true - controller.close() - } - } - const fail = (error: Error) => { - if (!settled) { - settled = true - controller.error(error) - } - } - - archive.on('data', (chunk: Buffer) => controller.enqueue(chunk)) - archive.on('end', close) - archive.on('error', fail) - void archive.finalize().catch(fail) - }, - cancel() { - archive.abort() - }, - }) + const stream = archiveDownload.finalize() recordBatchDownloadLog(materialId, userId).catch((error) => { console.error('[SKU Batch Download] 记录下载日志失败:', error) diff --git a/app/api/materials/download/route.ts b/app/api/materials/download/route.ts index 6d578921..56ee236e 100644 --- a/app/api/materials/download/route.ts +++ b/app/api/materials/download/route.ts @@ -8,7 +8,6 @@ import { authOptions } from '@/lib/auth'; import { prisma } from '@/lib/prisma'; import { access, stat } from 'fs/promises'; import { createReadStream } from 'fs'; -import { Readable } from 'stream'; import path from 'path'; import { lookup } from 'mime-types'; import { @@ -26,6 +25,7 @@ import { getVideoDownloadAvailability, getVideoDownloadUnavailableMessage, } from '@/lib/video-download-policy'; +import { nodeReadableToWebStream } from '@/lib/node-readable-stream'; function normalizeStoredLocalPath(filePath: string) { return String(filePath || '').replace(/\\/g, '/'); @@ -264,23 +264,6 @@ async function serveFile(filePath: string, material: any, fileNameSuffix: string try { await access(filePath); const fileStats = await stat(filePath); - const fileStream = createReadStream(filePath); - - // N6修复：确保文件流在任何情况下都能正确清理 - const stream = new ReadableStream({ - start(controller) { - fileStream.on('data', (chunk) => controller.enqueue(chunk)); - fileStream.on('end', () => controller.close()); - fileStream.on('error', (err) => { - // 错误时也要销毁文件流，防止文件描述符泄漏 - fileStream.destroy(); - controller.error(err); - }); - }, - cancel() { - fileStream.destroy(); - }, - }); const ext = path.extname(filePath); const mimeType = lookup(ext) || 'application/octet-stream'; @@ -295,7 +278,6 @@ async function serveFile(filePath: string, material: any, fileNameSuffix: string const filename = `${filenameParts.join('_')}${ext}`; if (validateOnly) { - fileStream.destroy(); return NextResponse.json({ success: true, fileName: filename, @@ -304,6 +286,8 @@ async function serveFile(filePath: string, material: any, fileNameSuffix: string }); } + const stream = nodeReadableToWebStream(createReadStream(filePath)); + console.log(`[Download] 开始下载: ${filename}, size=${fileStats.size} bytes`); // 修复：记录下载日志并更新下载次数（在后台异步执行） diff --git a/app/api/materials/route.ts b/app/api/materials/route.ts index d7ff18f0..f8054212 100644 --- a/app/api/materials/route.ts +++ b/app/api/materials/route.ts @@ -234,6 +234,14 @@ export async function GET(request: NextRequest) { console.log('[Materials API] Using preview from assets:', previewUrl); } } + + // 任务-页面类型：完成上传后预览资源的 assetType 为 preview。 + if (!previewUrl && material.materialType === '任务-页面' && material.assets && material.assets.length > 0) { + const previewAsset = material.assets.find((a: any) => a.assetType === 'preview'); + if (previewAsset) { + previewUrl = previewAsset.fileUrl; + } + } // 其他素材类型：使用 displayImages if (!previewUrl && material.displayImages) { @@ -388,7 +396,7 @@ export async function GET(request: NextRequest) { }, }, where: { - assetType: { in: ['main-preview', 'sku-preview', 'detail-preview'] }, + assetType: { in: ['main-preview', 'sku-preview', 'detail-preview', 'preview'] }, }, orderBy: { sortOrder: 'asc' }, }, @@ -749,6 +757,13 @@ export async function GET(request: NextRequest) { const formattedMaterials = materials.map(material => { // 处理previewUrl的fallback逻辑 let previewUrl = material.previewUrl || material.videoCoverUrl || ''; + + if (!previewUrl && material.materialType === '任务-页面') { + const previewAsset = (material.assets || []).find((asset: any) => asset.assetType === 'preview'); + if (previewAsset) { + previewUrl = previewAsset.fileUrl; + } + } // 任务-详情类型：优先使用从 assets 获取的封面图 if (isProductPackageType(material.materialType)) { diff --git a/app/api/materials/upload/task-page/complete/route.ts b/app/api/materials/upload/task-page/complete/route.ts index e8c93322..151ec26d 100644 --- a/app/api/materials/upload/task-page/complete/route.ts +++ b/app/api/materials/upload/task-page/complete/route.ts @@ -11,11 +11,20 @@ import { saveAssets, } from '@/lib/task-material-processor'; import { createMaterialLog } from '@/lib/material-log'; +import { TASK_DETAIL_UPLOAD_LIMITS } from '@/lib/task-detail-upload-policy'; +import { stat } from 'fs/promises'; export const runtime = 'nodejs' export const dynamic = 'force-dynamic' export const maxDuration = 10800 +class TaskPageUploadValidationError extends Error { + constructor(message: string, readonly status: number = 400) { + super(message) + this.name = 'TaskPageUploadValidationError' + } +} + // POST - 完成任务-页面上传 export async function POST(request: NextRequest) { let materialId: string | null = null; @@ -23,6 +32,8 @@ export async function POST(request: NextRequest) { let zipFilePath: string | null = null; const uploadedPreviewUrls: string[] = []; // 新生成的S3预览图URLs const uploadIds: string[] = []; // 需要清理的uploadId列表 + let claimedForCompletion = false; + let completed = false; try { const session = await getServerSession(authOptions); @@ -42,7 +53,8 @@ export async function POST(request: NextRequest) { ); } - if (!assets.main || assets.main.length === 0) { + const mainAssets = Array.isArray(assets.main) ? assets.main : []; + if (mainAssets.length === 0) { return NextResponse.json( { error: '至少需要上传一张预览图' }, { status: 400 } @@ -50,7 +62,7 @@ export async function POST(request: NextRequest) { } console.log('[Task Page Complete] Material ID:', materialId); - console.log('[Task Page Complete] Preview images count:', assets.main.length); + console.log('[Task Page Complete] Preview images count:', mainAssets.length); // 验证素材是否存在且属于当前用户 const material: any = await prisma.material.findUnique({ @@ -71,27 +83,90 @@ export async function POST(request: NextRequest) { ); } + if (material.materialType !== '任务-页面') { + return NextResponse.json( + { success: false, error: '素材类型不匹配' }, + { status: 400 } + ); + } + + // 只有一个请求能把 uploading 原子地切换为 processing。重复或并发完成请求 + // 不会进入清理分支，更不会误删已经完成的素材。 + const claimResult = await prisma.material.updateMany({ + where: { + id: materialId, + userId: (session.user as any).id, + materialType: '任务-页面', + deletedAt: null, + taskStatus: 'uploading', + }, + data: { taskStatus: 'processing' }, + }); + if (claimResult.count !== 1) { + return NextResponse.json( + { success: false, error: '该素材正在处理或已经完成，请勿重复提交' }, + { status: 409 }, + ); + } + claimedForCompletion = true; + + if (mainAssets.length > TASK_DETAIL_UPLOAD_LIMITS.maxImagesPerGroup) { + throw new TaskPageUploadValidationError( + `预览图最多上传 ${TASK_DETAIL_UPLOAD_LIMITS.maxImagesPerGroup} 张，请分批提交`, + ); + } + + const requestedUploadIds = mainAssets.map((asset: any) => String(asset?.uploadId || '').trim()); + if (requestedUploadIds.some((uploadId: string) => !uploadId)) { + throw new TaskPageUploadValidationError('上传会话编号无效'); + } + if (new Set(requestedUploadIds).size !== requestedUploadIds.length) { + throw new TaskPageUploadValidationError('上传会话编号重复，请重新选择文件'); + } + // 步骤1: 从chunk临时目录读取预览图文件 - console.log(`[Task Page] Processing ${assets.main.length} preview images...`); + console.log(`[Task Page] Processing ${mainAssets.length} preview images...`); const { getOwnedUploadedFile } = await import('@/lib/upload-sessions'); const previewFiles: Array<{ path: string; name: string; size: number }> = []; + let totalSourceBytes = 0; - for (const asset of assets.main) { + for (const asset of mainAssets) { try { - const filePath = await getOwnedUploadedFile(asset.uploadId, (session.user as any).id); + const uploadId = String(asset.uploadId).trim(); + const fileName = String(asset.fileName || 'image'); + const filePath = await getOwnedUploadedFile(uploadId, (session.user as any).id); if (filePath) { + tempFiles.push(filePath); + uploadIds.push(uploadId); + + const fileStats = await stat(filePath); + if (!fileStats.isFile()) { + throw new TaskPageUploadValidationError(`上传文件无效: ${fileName}`); + } + if (fileStats.size > TASK_DETAIL_UPLOAD_LIMITS.maxSingleImageBytes) { + throw new TaskPageUploadValidationError( + `图片「${fileName}」超过单图 ${Math.round(TASK_DETAIL_UPLOAD_LIMITS.maxSingleImageBytes / 1024 / 1024)}MB 限制`, + 413, + ); + } + totalSourceBytes += fileStats.size; + if (totalSourceBytes > TASK_DETAIL_UPLOAD_LIMITS.maxTotalSourceBytes) { + throw new TaskPageUploadValidationError('本次图片总大小超过 1GB，请分批提交', 413); + } + previewFiles.push({ path: filePath, - name: asset.fileName, - size: asset.fileSize, + name: fileName, + size: fileStats.size, }); - tempFiles.push(filePath); - uploadIds.push(asset.uploadId); } else { - throw new Error(`找不到uploadId对应的文件: ${asset.uploadId}`); + throw new Error(`找不到uploadId对应的文件: ${uploadId}`); } } catch (error) { + if (error instanceof TaskPageUploadValidationError) { + throw error; + } console.error(`[Task Page] 获取预览图文件失败 ${asset.uploadId}:`, error); throw new Error(`获取预览图文件失败: ${asset.fileName}`); } @@ -110,7 +185,7 @@ export async function POST(request: NextRequest) { // 步骤3: 生成预览图并上传到S3 const previewUrls = await generatePreviewImages( - assets.main.map((a: any) => ({ uploadId: a.uploadId, fileName: a.fileName })), + mainAssets.map((a: any) => ({ uploadId: a.uploadId, fileName: a.fileName })), '任务-页面' ); uploadedPreviewUrls.push(...previewUrls); @@ -124,9 +199,12 @@ export async function POST(request: NextRequest) { // 步骤4.5: 仅在素材成功完成后写入任务号 await assignTaskNoOnComplete(materialId!, taskNo, dingtalkApprovalNo); + completed = true; // 步骤5: 清理临时文件 - await cleanupTempFiles(tempFiles, uploadIds); + await cleanupTempFiles(tempFiles, uploadIds).catch((cleanupError) => { + console.warn('[Task Page] 上传已完成，但临时文件清理失败:', cleanupError); + }); // 步骤6: 记录上传日志 await createMaterialLog({ @@ -147,6 +225,8 @@ export async function POST(request: NextRequest) { newValue: zipInfo.fileName, }, }, + }).catch((logError) => { + console.error('[Task Page] 上传已完成，但日志记录失败:', logError); }); console.log('[Task Page] Complete success!'); @@ -157,12 +237,23 @@ export async function POST(request: NextRequest) { } catch (error: any) { console.error('[Task Page Complete Error]', error); - // 清理失败时的资源 - await cleanupOnError(uploadedPreviewUrls, zipFilePath, tempFiles, materialId); + // 只有成功抢占 processing 状态的请求，才有资格清理由本请求创建的资源和素材。 + // 重复请求、鉴权失败或查询异常绝不能根据用户传入的 materialId 删除数据。 + if (claimedForCompletion && !completed) { + await cleanupOnError( + uploadedPreviewUrls, + zipFilePath, + tempFiles, + materialId, + uploadIds, + true, + { taskStatus: 'processing' }, + ); + } return NextResponse.json( { error: error.message || '完成上传失败' }, - { status: 500 } + { status: error instanceof TaskPageUploadValidationError ? error.status : 500 } ); } } diff --git a/app/api/materials/upload/task-page/init/route.ts b/app/api/materials/upload/task-page/init/route.ts index e2fc1cbb..7d1bcea7 100644 --- a/app/api/materials/upload/task-page/init/route.ts +++ b/app/api/materials/upload/task-page/init/route.ts @@ -92,6 +92,7 @@ export async function POST(request: NextRequest) { zipFilePath: '', // 临时空值，完成上传时更新 zipFileName: '', // 临时空值，完成上传时更新 renderer: '', // 任务-页面不需要渲染器 + taskStatus: 'uploading', }, }); diff --git a/components/upload/TaskPageUploader.tsx b/components/upload/TaskPageUploader.tsx index 751a6fe9..26c1e267 100644 --- a/components/upload/TaskPageUploader.tsx +++ b/components/upload/TaskPageUploader.tsx @@ -1,8 +1,7 @@ 'use client' import { useState, useRef, useEffect } from 'react'; -import Image from 'next/image'; -import { parseSPU, type SPUParseResult } from '@/lib/spu-parser'; +import { parseSPU } from '@/lib/spu-parser'; import { hasDragLeftDropZone, prepareFileDrag, @@ -10,6 +9,7 @@ import { } from './drag-drop-utils'; import TaskReferenceFields from './TaskReferenceFields'; import { getDingtalkApprovalNoError } from '@/lib/dingtalk-approval-no'; +import { TASK_DETAIL_UPLOAD_LIMITS } from '@/lib/task-detail-upload-policy'; interface AssetFile { file: File; @@ -150,7 +150,26 @@ export default function TaskPageUploader({ } const currentList = previewImages; - const { acceptedFiles, errors } = validateImageFiles(files); + const validation = validateImageFiles(files, { + maxBytes: TASK_DETAIL_UPLOAD_LIMITS.maxSingleImageBytes, + maxCount: TASK_DETAIL_UPLOAD_LIMITS.maxImagesPerGroup, + existingCount: currentList.length, + }); + const currentBytes = currentList.reduce((sum, item) => sum + item.file.size, 0); + let nextTotalBytes = currentBytes; + const rejectedByTotal: string[] = []; + const acceptedFiles = validation.acceptedFiles.filter((file) => { + if (nextTotalBytes + file.size > TASK_DETAIL_UPLOAD_LIMITS.maxTotalSourceBytes) { + rejectedByTotal.push(file.name); + return false; + } + nextTotalBytes += file.size; + return true; + }); + const errors = [...validation.errors]; + if (rejectedByTotal.length > 0) { + errors.push(`本次图片总大小最多 1GB，已忽略：${rejectedByTotal.join('、')}`); + } const newFiles: AssetFile[] = acceptedFiles.map((file, index) => ({ file, preview: URL.createObjectURL(file), @@ -223,8 +242,11 @@ export default function TaskPageUploader({ throw new Error(errorData.error || '初始化失败'); } - const { data: initData } = await initResponse.json(); - const materialId = initData.materialId; + const initData = await initResponse.json(); + const materialId = typeof initData?.materialId === 'string' ? initData.materialId : ''; + if (!materialId) { + throw new Error('初始化失败：服务器未返回素材编号'); + } setUploadProgress(10); diff --git a/components/upload/TaskPageUploader.upload.test.tsx b/components/upload/TaskPageUploader.upload.test.tsx new file mode 100644 index 00000000..b71386d8 --- /dev/null +++ b/components/upload/TaskPageUploader.upload.test.tsx @@ -0,0 +1,115 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' + +import TaskPageUploader from '@/components/upload/TaskPageUploader' +import { chunkUpload } from '@/lib/chunk-upload' +import { TASK_DETAIL_UPLOAD_LIMITS } from '@/lib/task-detail-upload-policy' + +jest.mock('@/lib/chunk-upload', () => ({ + chunkUpload: jest.fn(), +})) + +const originalFetch = global.fetch + +describe('TaskPageUploader upload contract', () => { + beforeEach(() => { + jest.clearAllMocks() + Object.defineProperty(URL, 'createObjectURL', { + configurable: true, + writable: true, + value: jest.fn(() => 'blob:task-page-preview'), + }) + Object.defineProperty(URL, 'revokeObjectURL', { + configurable: true, + writable: true, + value: jest.fn(), + }) + }) + + afterAll(() => { + global.fetch = originalFetch + }) + + test('应读取初始化接口顶层 materialId 并提交完成请求', async () => { + const fetchMock = jest + .fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ success: true, materialId: 'material-task-page-1' }), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ success: true, materialId: 'material-task-page-1' }), + }) + global.fetch = fetchMock as unknown as typeof fetch + ;(chunkUpload as jest.Mock).mockResolvedValue({ uploadId: 'upload-preview-1' }) + + const onSuccess = jest.fn() + const { container } = render( + , + ) + + fireEvent.change(screen.getByLabelText(/SPU型号/), { target: { value: 'SJ0371' } }) + fireEvent.change(screen.getByLabelText(/使用店铺/), { target: { value: '索菲亚' } }) + fireEvent.change(screen.getByLabelText(/场景类别/), { target: { value: '书房' } }) + + const imageInput = container.querySelector('input[type="file"][accept="image/*"]') as HTMLInputElement + const image = new File(['preview'], 'page-1.jpg', { type: 'image/jpeg' }) + fireEvent.change(imageInput, { target: { files: [image] } }) + fireEvent.click(screen.getByRole('button', { name: '提交' })) + + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)) + + expect(fetchMock.mock.calls[0][0]).toBe('/api/materials/upload/task-page/init') + expect(fetchMock.mock.calls[1][0]).toBe('/api/materials/upload/task-page/complete') + expect(JSON.parse(String(fetchMock.mock.calls[1][1]?.body))).toEqual( + expect.objectContaining({ + materialId: 'material-task-page-1', + assets: { + main: [ + expect.objectContaining({ + uploadId: 'upload-preview-1', + fileName: 'page-1.jpg', + }), + ], + }, + }), + ) + + await waitFor(() => expect(onSuccess).toHaveBeenCalledTimes(1), { timeout: 2000 }) + }) + + test('选择文件时应在上传前拒绝超过 100MB 的单张图片', () => { + const { container } = render( + , + ) + const imageInput = container.querySelector('input[type="file"][accept="image/*"]') as HTMLInputElement + const oversized = new File(['small fixture'], 'too-large.jpg', { type: 'image/jpeg' }) + Object.defineProperty(oversized, 'size', { + configurable: true, + value: TASK_DETAIL_UPLOAD_LIMITS.maxSingleImageBytes + 1, + }) + + fireEvent.change(imageInput, { target: { files: [oversized] } }) + + expect(screen.getByText(/too-large\.jpg/)).toHaveTextContent('超过 100MB') + expect(screen.queryByAltText('预览图 1')).not.toBeInTheDocument() + }) +}) diff --git a/docs/未完成问题处理记录-2026-08-01.md b/docs/未完成问题处理记录-2026-08-01.md index af651bbb..45c7d556 100644 --- a/docs/未完成问题处理记录-2026-08-01.md +++ b/docs/未完成问题处理记录-2026-08-01.md @@ -1,6 +1,6 @@ # 素材库未完成问题处理记录（2026-08-01） -> 当前代码基线：`origin/main@776c6bafce9a819fd141339d6a6991c47f15f6ca` +> 当前代码基线：`origin/main@7c609a5ca891e95e23b7e1ad3a7fd6cfa680fb21` > 记录原则：代码、生产运行状态和历史事件分开判断；没有完成真实验收的项目不标记为“已关闭”。 ## 一、当前结论 @@ -173,9 +173,41 @@ Edge 补充边界：真实 Edge 已能登录隔离站点并正常显示已上传预览，但 Codex 的 Edge 扩展未开启“允许访问文件 URL”，自动化工具因此无法把本地测试图片交给网页；这是测试工具权限，不是素材库返回的上传失败。用户此前已经用自己的 Edge 完成人工上传。生产发布后仍需再做一次真实业务账号的小批量上传、预览和下载验收，才能把本项标记为“生产已关闭”。 +### 代码批次 3.1：任务-页面空卡片与大文件下载背压（2026-08-03 本地完成并通过提交门禁，待发布） + +本轮对生产数据库做了只读核查，没有修改或删除任何数据。生产共有 14 条未删除的“任务-页面”素材，其中 13 条同时满足“顶层预览为空、ZIP 路径为空、资源记录为 0”，只有 1 条具备真实预览和文件。这与首页截图中的 13 张“暂无预览图”空卡片和 1 张正常卡片完全一致。 + +已确认根因与处理： + +1. 初始化接口返回的是顶层 `materialId`，前端却按 `data.materialId` 读取；后端刚创建空素材记录，前端便立即报错，后续图片上传和完成接口都没有执行。前端现已按真实接口契约读取，并在素材编号缺失时给出明确错误。 +2. 新建记录会标记为 `uploading`；完成请求必须原子地把本人名下这条记录抢占为 `processing`，只有一个请求能成功。重复点击或并发完成会返回 409，不能进入清理分支，也不能删除已经完成的素材。 +3. 完成接口增加素材类型校验，不能用“任务-页面”接口覆盖用户自己的其它素材。完成阶段失败时，临时文件、上传会话、S3 预览、ZIP 和素材会进入统一清理；数据库删除还必须再次满足 `taskStatus=processing`，不能只根据客户端传来的素材 ID 删除。 +4. 首页列表会从 `assetType=preview` 的任务页面资源回退取封面；没有预览、没有 ZIP、也没有任何资源的任务页面空壳会从正常列表和筛选数量隐藏，但不会被删除，回收站查询也不受影响。 +5. 生产只读验证表明该过滤条件上线后，“任务-页面”的正常列表数量将从 14 变为 1；13 条历史空壳仍保留在数据库中，后续如需物理清理，必须先备份并单独授权。 +6. 任务页面前端和后端统一增加单图 100MB、最多 60 张、总量 1GB 的硬门禁；后端对用户拥有的上传会话去重，并以磁盘实际 `stat` 大小为准，在打包和 `readFile()` 之前拒绝超限文件。图片解码阶段仍受 60MP、最大边长和全局单并发门禁保护。 + +同时修复了下载链路的真实内存风险：原实现虽然使用了 Web Stream，但通过 Node `data` 事件持续推送文件，用户网络慢或多人并发时无法向源文件施加背压，进程仍可能积压大量数据。现已统一为“下游每请求一块，Node 源流才读取一块”的转换器，并覆盖： + +- 素材主下载与 Range 下载； +- SKU 单个模型下载和批量动态 ZIP； +- SKU 白底图动态 ZIP； +- 外部 Token 的本地文件、远程文件、Range、动态 ZIP 和展示图 ZIP。 + +SKU 白底图 ZIP 已从 JSZip 的“读取全部图片并生成完整内存 Buffer”改为 Archiver 流式压缩；动态 ZIP 不再先生成完整压缩包再返回。用户取消动态 ZIP 时会同时销毁显式远程输入流、调用 Archiver `abort()` 清空排队任务并销毁输出流；测试已验证源流销毁、archive 进入 aborted/destroyed 状态且队列归零。 + +验证结果： + +- 任务页面、并发完成、条件清理、下载背压和 ZIP 取消的定向回归全部通过。 +- 全量 Jest 回归共 150 个测试套件、962 项测试全部通过。 +- TypeScript 类型检查通过；相关文件 ESLint 为 0 error，仅保留任务页面既有的动态 Blob `` 性能提示。 +- Next.js 生产编译成功；Impeccable 检测没有发现新增界面问题。 +- 本机隔离工作树通过 junction 复用原项目 `node_modules`，standalone 文件复制阶段仍有 Windows `EPERM` 符号链接提示，因此本次构建只证明代码和页面编译通过，不能替代 Linux 镜像打包与部署后验收。 + +尚未完成：本批尚未部署生产；需要在安全发布窗口完成 Linux 镜像构建、真实“任务-页面”小文件上传、首页卡片预览、单文件/Range/动态 ZIP 下载，以及下载期间容器内存峰值验收后，才能标记为生产关闭。 + ### 后续 P1 -- 普通主上传和部分 ZIP 下载仍会把完整文件或最终压缩包放入内存。 +- 普通主上传的小文件接口仍使用 `formData()`，SKU 模型上传仍会把单个 ZIP 完整放入内存；两者需要改为服务端强制限额或统一分片上传，不能只依赖前端约束。 - 下载 Token 有效期仍接近永久，且外部下载没有完成实时权限复核。 - `canShare` 权限配置未被分享接口真正执行；部分管理员接口角色口径不一致；素材所有者仍可能通过编辑接口转移素材归属。 - 部门改名、移动层级和负责人角色更新仍缺完整事务。 @@ -205,5 +237,6 @@ Edge 补充边界：真实 Edge 已能登录隔离站点并正常显示已上传 5. **代码批次 2.1**：已随 `62295a0` 提交并推送，Linux Docker 构建和 Chromium 大图上传验收通过；待生产门禁恢复后合并、部署，并补做真实业务账号验收。 6. **生产发布门禁**：先处理或等待 TrueNAS 可用内存恢复，禁止在当前不足 2GiB 余量下切换生产。 7. **代码批次 3**：继续处理上传/ZIP 内存、长期下载 Token、权限和数据库一致性。 +8. **代码批次 3.1**：任务-页面上传/首页预览和下载背压已完成本地修复并通过独立提交门禁；生产发布前仍须完成 Linux 镜像和真实运行验收。 当前可以确认“旧影子漏洞入口已不可达”，但在旧凭据失效状态和访问日志核对完成前，不能把历史凭据事件标记为完全结案；`uploads` 的 `0777` 权限问题按用户决定暂缓，仍属于已知未关闭项。 diff --git a/lib/__tests__/archive-download-stream.test.ts b/lib/__tests__/archive-download-stream.test.ts new file mode 100644 index 00000000..db829fc5 --- /dev/null +++ b/lib/__tests__/archive-download-stream.test.ts @@ -0,0 +1,40 @@ +/** @jest-environment node */ + +import archiver from 'archiver' +import { Readable } from 'node:stream' + +import { createArchiveDownloadStream } from '@/lib/archive-download-stream' + +class OpenReadable extends Readable { + private emitted = false + + override _read(): void { + if (!this.emitted) { + this.emitted = true + this.push(Buffer.alloc(64 * 1024, 1)) + } + } +} + +describe('createArchiveDownloadStream', () => { + test('取消动态 ZIP 时应中止 Archiver 并销毁显式输入流', async () => { + const archive = archiver('zip', { zlib: { level: 1 } }) + const abortSpy = jest.spyOn(archive, 'abort') + const source = new OpenReadable() + const sourceDestroySpy = jest.spyOn(source, 'destroy') + const download = createArchiveDownloadStream(archive) + + archive.append(download.trackSource(source), { name: 'open.bin' }) + const reader = download.finalize().getReader() + + await reader.read() + await reader.cancel() + + expect(abortSpy).toHaveBeenCalledTimes(1) + expect(sourceDestroySpy).toHaveBeenCalled() + expect(source.destroyed).toBe(true) + expect(archive.destroyed).toBe(true) + expect((archive as any)._state?.aborted).toBe(true) + expect((archive as any)._queue?.length?.() || 0).toBe(0) + }) +}) diff --git a/lib/__tests__/file-stream-cleanup.test.ts b/lib/__tests__/file-stream-cleanup.test.ts index f08f0f1c..8f357774 100644 --- a/lib/__tests__/file-stream-cleanup.test.ts +++ b/lib/__tests__/file-stream-cleanup.test.ts @@ -1,91 +1,30 @@ /** - * N6修复测试：文件流错误时正确清理 - * - * 验证： - * 1. 所有 fileStream.on('error') 回调中都调用了 fileStream.destroy() - * 2. 确保文件描述符不会泄漏 + * 大文件下载流门禁：路由必须使用统一的背压转换器，不能通过 data 事件 + * 无限制地把源文件推入 Web Stream 队列。 */ -describe('N6: 文件流错误清理测试', () => { - - test('所有下载路由的文件流错误处理应该包含 destroy() 调用', async () => { - const fs = require('fs') - const path = require('path') - - const downloadRoutes = [ - 'app/api/materials/download/route.ts', - 'app/api/materials/[id]/skus/[skuId]/download/route.ts', - 'app/api/external/download/route.ts', - ] - - for (const routePath of downloadRoutes) { - const filePath = path.join(__dirname, '../..', routePath) - - if (!fs.existsSync(filePath)) { - console.log(`跳过测试：文件不存在 ${routePath}`) - continue - } - - const content = fs.readFileSync(filePath, 'utf-8') - - // 查找所有 fileStream.on('error' 模式 - const errorHandlerPattern = /fileStream\.on\('error',\s*\(err\)\s*=>\s*\{[^}]*\}/g - const matches = content.match(errorHandlerPattern) - - if (matches) { - for (const match of matches) { - // 每个错误处理回调中都应该有 destroy() 调用 - expect(match).toContain('destroy()') - } - } - } - }) +describe('大文件下载流清理与背压', () => { + const fs = require('fs') + const path = require('path') + const downloadRoutes = [ + ['app/api/materials/download/route.ts', 'nodeReadableToWebStream'], + ['app/api/materials/[id]/skus/[skuId]/download/route.ts', 'createArchiveDownloadStream'], + ['app/api/materials/[id]/skus/batch-download/route.ts', 'createArchiveDownloadStream'], + ['app/api/external/download/route.ts', 'createArchiveDownloadStream'], + ] - test('不应该有只调用 controller.error 而不调用 destroy 的错误处理', async () => { - const fs = require('fs') - const path = require('path') - - const downloadRoutes = [ - 'app/api/materials/download/route.ts', - 'app/api/materials/[id]/skus/[skuId]/download/route.ts', - 'app/api/external/download/route.ts', - ] - - for (const routePath of downloadRoutes) { - const filePath = path.join(__dirname, '../..', routePath) - - if (!fs.existsSync(filePath)) { - console.log(`跳过测试：文件不存在 ${routePath}`) - continue - } - - const content = fs.readFileSync(filePath, 'utf-8') - - // 查找简单的错误处理模式（只有 controller.error，没有 destroy） - // 这种模式是有问题的：fileStream.on('error', (err) => controller.error(err)) - const badPattern = /fileStream\.on\('error',\s*\(err\)\s*=>\s*controller\.error\(err\)\)/g - const badMatches = content.match(badPattern) - - // 不应该有这种简单的错误处理 - expect(badMatches).toBeNull() - } - }) - - test('所有错误处理应该先调用 destroy 再调用 controller.error', async () => { - const fs = require('fs') - const path = require('path') - - const filePath = path.join(__dirname, '../../app/api/materials/download/route.ts') - - if (!fs.existsSync(filePath)) { - console.log('跳过测试：文件不存在') - return - } - + test.each(downloadRoutes)('%s 应使用统一的背压/取消流转换器', (routePath, helperName) => { + const filePath = path.join(__dirname, '../..', routePath) const content = fs.readFileSync(filePath, 'utf-8') - - // 检查错误处理的顺序：先 destroy() 再 controller.error() - const correctPattern = /fileStream\.destroy\(\)[\s\S]*?controller\.error\(err\)/ - expect(correctPattern.test(content)).toBe(true) + + expect(content).toContain(helperName) + expect(content).not.toMatch(/(?:fileStream|nodeStream|archive)\.on\(['"]data['"]/) + }) + + test('转换器取消时必须销毁 Node 源流', () => { + const helperPath = path.join(__dirname, '..', 'node-readable-stream.ts') + const content = fs.readFileSync(helperPath, 'utf-8') + + expect(content).toMatch(/cancel\([^)]*\)[\s\S]*?source\.destroy\(\)/) }) }) diff --git a/lib/__tests__/material-query.test.ts b/lib/__tests__/material-query.test.ts index 0b3ca403..5922d2ea 100644 --- a/lib/__tests__/material-query.test.ts +++ b/lib/__tests__/material-query.test.ts @@ -113,4 +113,33 @@ describe('material search conditions', () => { expect(context.isSuperAdmin).toBe(true) expect(getUserViewableMaterialTypes).not.toHaveBeenCalled() }) + + test('正常列表应隐藏没有任何文件的任务-页面空记录', async () => { + ;(getUserViewableMaterialTypes as jest.Mock).mockResolvedValue(['任务-页面']) + + const result = await buildMaterialWhereFromSearchParams( + new URLSearchParams('materialType=任务-页面'), + { id: 'user-1', role: 'USER', department: '品牌创意部' }, + ) + + expect(result.empty).toBe(false) + expect(result.where.NOT).toEqual({ + materialType: '任务-页面', + previewUrl: '', + zipFilePath: '', + assets: { none: {} }, + }) + }) + + test('回收站查询不应套用任务-页面空记录隐藏条件', async () => { + ;(isSuperAdmin as jest.Mock).mockResolvedValue(true) + + const result = await buildMaterialWhereFromSearchParams( + new URLSearchParams('showDeleted=true'), + { id: 'super-admin', role: 'ADMIN' }, + ) + + expect(result.where.deletedAt).toEqual({ not: null }) + expect(result.where.NOT).toBeUndefined() + }) }) diff --git a/lib/__tests__/materials-route-ai-generated.test.ts b/lib/__tests__/materials-route-ai-generated.test.ts index bdad0f03..394512c5 100644 --- a/lib/__tests__/materials-route-ai-generated.test.ts +++ b/lib/__tests__/materials-route-ai-generated.test.ts @@ -208,4 +208,81 @@ describe('materials route ai generated', () => { expect(body.totalCount).toBe(0) expect(prisma.material.findMany).not.toHaveBeenCalled() }) + + test('任务-页面缺少顶层 previewUrl 时应使用 preview 资产作为首页封面', async () => { + ;(getUserViewableMaterialTypes as jest.Mock).mockResolvedValue(['任务-页面']) + ;(prisma.material.findMany as jest.Mock).mockResolvedValue([ + { + id: 'task-page-1', + userId: 'user-1', + previewUrl: '', + productModel: 'SJ0371', + categoryName: '索菲亚', + sceneName: '书房', + platformName: '', + productCategoryName: '', + materialType: '任务-页面', + renderer: '', + authorName: '张三', + zipFileName: 'task-page.zip', + zipFileSize: BigInt(1024), + downloadCount: 0, + likeCount: 0, + createdAt: new Date('2026-06-26T08:00:00.000Z'), + videoSourceUrl: null, + displayImages: [], + videoPreview540Url: null, + videoPreviewClipUrl: null, + videoCoverUrl: null, + videoDuration: null, + videoProcessingStatus: null, + transcodeLastError: null, + transcodeAttempts: 0, + isAiGenerated: false, + taskDetailSourceType: null, + taskDetailRequesterUserId: null, + taskDetailRequesterName: null, + user: { + username: 'zhangsan', + realName: '张三', + avatar: null, + }, + assets: [ + { + id: 'preview-1', + assetType: 'preview', + fileUrl: 'https://example.com/task-page-preview.webp', + fileName: 'preview-0.webp', + fileSize: null, + sortOrder: 0, + createdAt: new Date('2026-06-26T08:00:00.000Z'), + updatedAt: new Date('2026-06-26T08:00:00.000Z'), + skuCode: null, + barcodes: [], + }, + ], + }, + ]) + + const response = await GET( + new NextRequest('http://localhost/api/materials?materialType=任务-页面&page=1&pageSize=20'), + ) + const body = await response.json() + + expect(response.status).toBe(200) + expect(prisma.material.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + select: expect.objectContaining({ + assets: expect.objectContaining({ + where: { + assetType: { + in: ['main-preview', 'sku-preview', 'detail-preview', 'preview'], + }, + }, + }), + }), + }), + ) + expect(body.materials[0].previewUrl).toBe('https://example.com/task-page-preview.webp') + }) }) diff --git a/lib/__tests__/node-readable-stream.test.ts b/lib/__tests__/node-readable-stream.test.ts new file mode 100644 index 00000000..ab315936 --- /dev/null +++ b/lib/__tests__/node-readable-stream.test.ts @@ -0,0 +1,111 @@ +/** @jest-environment node */ + +import { Readable } from 'node:stream' + +import { nodeReadableToWebStream } from '@/lib/node-readable-stream' + +async function readAll(stream: ReadableStream): Promise { + const reader = stream.getReader() + const chunks: Buffer[] = [] + + try { + while (true) { + const { value, done } = await reader.read() + if (done) { + return Buffer.concat(chunks) + } + + chunks.push(Buffer.from(value)) + } + } finally { + reader.releaseLock() + } +} + +class CountingReadable extends Readable { + produced = 0 + + private nextChunk = 0 + + constructor(private readonly totalChunks: number) { + // One-byte high water mark keeps Node's own buffering from hiding a + // converter that eagerly drains the source. + super({ highWaterMark: 1 }) + } + + override _read(): void { + if (this.nextChunk >= this.totalChunks) { + this.push(null) + return + } + + this.produced += 1 + this.push(Buffer.from([this.nextChunk])) + this.nextChunk += 1 + } +} + +class CancellableReadable extends Readable { + private emitted = false + + constructor() { + super({ highWaterMark: 1 }) + } + + override _read(): void { + // Keep the source open after the first chunk so cancellation, rather than + // natural EOF, is what must release it. + if (!this.emitted) { + this.emitted = true + this.push(Buffer.from('first chunk')) + } + } +} + +describe('nodeReadableToWebStream', () => { + test('preserves every byte across the Node-to-Web conversion', async () => { + const sourceChunks = [ + Buffer.from('header\0', 'utf8'), + Buffer.from([0, 1, 127, 128, 254, 255]), + Buffer.from('tail', 'utf8'), + ] + const source = Readable.from(sourceChunks) + + const result = await readAll(nodeReadableToWebStream(source)) + + expect(result).toEqual(Buffer.concat(sourceChunks)) + }) + + test('does not pre-read every source chunk before the Web stream is consumed', async () => { + const totalChunks = 64 + const source = new CountingReadable(totalChunks) + const webStream = nodeReadableToWebStream(source) + + try { + // Let the stream's initial pull settle without introducing a time-based + // delay. A correctly backpressured adapter may read one chunk (and Node + // may briefly request one extra), but it must not drain the source. + await Promise.resolve() + + expect(source.produced).toBeLessThan(totalChunks) + } finally { + await webStream.cancel() + } + }) + + test('destroys the Node stream when the Web reader is cancelled', async () => { + const source = new CancellableReadable() + const destroySpy = jest.spyOn(source, 'destroy') + const reader = nodeReadableToWebStream(source).getReader() + + await expect(reader.read()).resolves.toEqual({ + done: false, + value: expect.any(Uint8Array), + }) + + await reader.cancel() + + expect(destroySpy).toHaveBeenCalledTimes(1) + expect(source.destroyed).toBe(true) + }) +}) diff --git a/lib/__tests__/task-page-upload-routes.test.ts b/lib/__tests__/task-page-upload-routes.test.ts new file mode 100644 index 00000000..760a0c8d --- /dev/null +++ b/lib/__tests__/task-page-upload-routes.test.ts @@ -0,0 +1,199 @@ +/** @jest-environment node */ + +import { NextRequest } from 'next/server' +import { getServerSession } from 'next-auth' + +import { POST as initTaskPage } from '@/app/api/materials/upload/task-page/init/route' +import { POST as completeTaskPage } from '@/app/api/materials/upload/task-page/complete/route' +import { prisma } from '@/lib/prisma' +import { cleanupOnError, createZipFromFiles } from '@/lib/task-material-processor' +import { getOwnedUploadedFile } from '@/lib/upload-sessions' +import { stat } from 'fs/promises' +import { TASK_DETAIL_UPLOAD_LIMITS } from '@/lib/task-detail-upload-policy' + +jest.mock('next-auth', () => ({ getServerSession: jest.fn() })) +jest.mock('@/lib/auth', () => ({ authOptions: {} })) +jest.mock('@/lib/prisma', () => ({ + prisma: { + user: { findUnique: jest.fn() }, + material: { + create: jest.fn(), + findFirst: jest.fn(), + findUnique: jest.fn(), + updateMany: jest.fn(), + }, + }, +})) +jest.mock('@/lib/product-package', () => ({ + assignTaskNoOnComplete: jest.fn(), + purgeDeletedMaterialsByTaskNo: jest.fn(), + validateTaskNo: jest.fn(() => true), +})) +jest.mock('@/lib/dingtalk-approval-no', () => ({ + getDingtalkApprovalNoError: jest.fn(() => null), +})) +jest.mock('@/lib/task-material-processor', () => ({ + createZipFromFiles: jest.fn(), + generatePreviewImages: jest.fn(), + cleanupTempFiles: jest.fn(), + cleanupOnError: jest.fn(), + saveAssets: jest.fn(), +})) +jest.mock('@/lib/material-log', () => ({ createMaterialLog: jest.fn() })) +jest.mock('@/lib/upload-sessions', () => ({ getOwnedUploadedFile: jest.fn() })) +jest.mock('fs/promises', () => ({ stat: jest.fn() })) + +describe('任务-页面上传接口', () => { + beforeEach(() => { + jest.clearAllMocks() + jest.spyOn(console, 'log').mockImplementation(() => undefined) + jest.spyOn(console, 'error').mockImplementation(() => undefined) + ;(getServerSession as jest.Mock).mockResolvedValue({ user: { id: 'user-1' } }) + ;(prisma.user.findUnique as jest.Mock).mockResolvedValue({ username: '张三' }) + ;(prisma.material.create as jest.Mock).mockResolvedValue({ id: 'material-1' }) + ;(prisma.material.updateMany as jest.Mock).mockResolvedValue({ count: 1 }) + ;(stat as jest.Mock).mockResolvedValue({ isFile: () => true, size: 10 }) + }) + + afterEach(() => { + jest.restoreAllMocks() + }) + + test('初始化接口应返回顶层 materialId，并标记为 uploading', async () => { + const response = await initTaskPage( + new NextRequest('http://localhost/api/materials/upload/task-page/init', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + productModel: 'SJ0371', + categoryName: '索菲亚', + sceneName: '书房', + }), + }), + ) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body).toEqual({ success: true, materialId: 'material-1' }) + expect(prisma.material.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + materialType: '任务-页面', + taskStatus: 'uploading', + }), + }), + ) + }) + + test('完成接口不能覆盖其它素材类型', async () => { + ;(prisma.material.findUnique as jest.Mock).mockResolvedValue({ + id: 'material-1', + userId: 'user-1', + materialType: '任务-详情', + }) + + const response = await completeTaskPage( + makeCompleteRequest([{ uploadId: 'upload-1', fileName: 'page.jpg', fileSize: 10 }]), + ) + + expect(response.status).toBe(400) + expect(getOwnedUploadedFile).not.toHaveBeenCalled() + }) + + test('打包失败时应把上传会话一并交给统一清理', async () => { + ;(prisma.material.findUnique as jest.Mock).mockResolvedValue({ + id: 'material-1', + userId: 'user-1', + materialType: '任务-页面', + productModel: 'SJ0371', + }) + ;(getOwnedUploadedFile as jest.Mock).mockResolvedValue('uploads/temp/upload-1/merged.jpg') + ;(createZipFromFiles as jest.Mock).mockRejectedValue(new Error('打包失败')) + + const response = await completeTaskPage( + makeCompleteRequest([{ uploadId: 'upload-1', fileName: 'page.jpg', fileSize: 10 }]), + ) + + expect(response.status).toBe(500) + expect(cleanupOnError).toHaveBeenCalledWith( + [], + null, + ['uploads/temp/upload-1/merged.jpg'], + 'material-1', + ['upload-1'], + true, + { taskStatus: 'processing' }, + ) + }) + + test('重复完成请求应返回 409，且不能进入清理删除素材', async () => { + ;(prisma.material.findUnique as jest.Mock).mockResolvedValue({ + id: 'material-1', + userId: 'user-1', + materialType: '任务-页面', + taskStatus: 'completed', + }) + ;(prisma.material.updateMany as jest.Mock).mockResolvedValue({ count: 0 }) + + const response = await completeTaskPage( + makeCompleteRequest([{ uploadId: 'upload-1', fileName: 'page.jpg', fileSize: 10 }]), + ) + + expect(response.status).toBe(409) + expect(cleanupOnError).not.toHaveBeenCalled() + expect(getOwnedUploadedFile).not.toHaveBeenCalled() + }) + + test('应按磁盘实际大小在 readFile 前拒绝超大图片并清理本请求', async () => { + ;(prisma.material.findUnique as jest.Mock).mockResolvedValue({ + id: 'material-1', + userId: 'user-1', + materialType: '任务-页面', + taskStatus: 'uploading', + productModel: 'SJ0371', + }) + ;(getOwnedUploadedFile as jest.Mock).mockResolvedValue('uploads/temp/upload-1/merged.jpg') + ;(stat as jest.Mock).mockResolvedValue({ + isFile: () => true, + size: TASK_DETAIL_UPLOAD_LIMITS.maxSingleImageBytes + 1, + }) + + const response = await completeTaskPage( + makeCompleteRequest([{ uploadId: 'upload-1', fileName: 'large.jpg', fileSize: 1 }]), + ) + + expect(response.status).toBe(413) + expect(createZipFromFiles).not.toHaveBeenCalled() + expect(cleanupOnError).toHaveBeenCalledWith( + [], + null, + ['uploads/temp/upload-1/merged.jpg'], + 'material-1', + ['upload-1'], + true, + { taskStatus: 'processing' }, + ) + }) + + test('素材查询失败时不能按请求中的 materialId 执行删除清理', async () => { + ;(prisma.material.findUnique as jest.Mock).mockRejectedValue(new Error('database unavailable')) + + const response = await completeTaskPage( + makeCompleteRequest([{ uploadId: 'upload-1', fileName: 'page.jpg', fileSize: 10 }]), + ) + + expect(response.status).toBe(500) + expect(cleanupOnError).not.toHaveBeenCalled() + }) +}) + +function makeCompleteRequest(assets: Array<{ uploadId: string; fileName: string; fileSize: number }>) { + return new NextRequest('http://localhost/api/materials/upload/task-page/complete', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + materialId: 'material-1', + assets: { main: assets }, + }), + }) +} diff --git a/lib/archive-download-stream.ts b/lib/archive-download-stream.ts new file mode 100644 index 00000000..f1c11ab3 --- /dev/null +++ b/lib/archive-download-stream.ts @@ -0,0 +1,65 @@ +import type { Archiver } from 'archiver' +import { Readable } from 'node:stream' + +import { nodeReadableToWebStream } from '@/lib/node-readable-stream' + +/** + * 管理动态 ZIP 的输出与外部输入流。 + * + * Archiver 仅销毁输出流时不会保证取消排队任务或销毁已 append 的远程流， + * 因此下载取消必须同时 abort 压缩队列并销毁所有仍存活的显式输入流。 + */ +export function createArchiveDownloadStream(archive: Archiver) { + const trackedSources = new Set() + let stopped = false + let finalized = false + + const destroyTrackedSources = () => { + trackedSources.forEach((source) => { + if (!source.destroyed) { + source.destroy() + } + }) + trackedSources.clear() + } + + const abort = () => { + if (stopped) { + return + } + stopped = true + destroyTrackedSources() + try { + archive.abort() + } catch { + // 已结束或已销毁的 archive 仍继续执行下方 destroy，确保输出关闭。 + } + if (!archive.destroyed) { + archive.destroy() + } + } + + archive.once('error', destroyTrackedSources) + archive.once('end', () => trackedSources.clear()) + + return { + trackSource(source: T): T { + if (!source.destroyed) { + trackedSources.add(source) + source.once('close', () => trackedSources.delete(source)) + } + return source + }, + abort, + finalize(): ReadableStream { + if (finalized) { + throw new Error('ZIP 下载流已经开始生成') + } + finalized = true + + const stream = nodeReadableToWebStream(archive, { onCancel: abort }) + void archive.finalize().catch(() => abort()) + return stream + }, + } +} diff --git a/lib/material-query.ts b/lib/material-query.ts index d8ac05dd..dc11cf96 100644 --- a/lib/material-query.ts +++ b/lib/material-query.ts @@ -276,6 +276,15 @@ export async function buildMaterialWhereFromSearchParams( where.deletedAt = { not: null } } else { where.deletedAt = null + // “任务-页面”会先创建素材记录，再上传并生成预览图。历史上传中断留下的 + // 空记录没有预览、ZIP 或任何资产，不应出现在正常素材列表和筛选计数中。 + // 这里只隐藏可确认没有文件的空壳，不删除数据，也不影响回收站查看。 + where.NOT = { + materialType: '任务-页面', + previewUrl: '', + zipFilePath: '', + assets: { none: {} }, + } } const category = searchParams.get('category') || '' diff --git a/lib/node-readable-stream.ts b/lib/node-readable-stream.ts new file mode 100644 index 00000000..57381665 --- /dev/null +++ b/lib/node-readable-stream.ts @@ -0,0 +1,61 @@ +import { Readable } from 'node:stream' + +export interface NodeReadableToWebStreamOptions { + onCancel?: (reason?: unknown) => void | Promise +} + +/** + * 将 Node.js Readable 转换为受 Web Stream 消费速度约束的流。 + * + * 每次 pull 只向下游交付一个分块，避免 data 事件持续读取大文件、 + * 而浏览器或代理来不及接收时在 Node 进程内堆积大量内存。 + */ +export function nodeReadableToWebStream( + source: Readable, + options: NodeReadableToWebStreamOptions = {}, +): ReadableStream { + const iterator = source[Symbol.asyncIterator]() + let finished = false + + return new ReadableStream( + { + async pull(controller) { + if (finished) { + return + } + + try { + const next = await iterator.next() + if (next.done) { + finished = true + controller.close() + return + } + + const chunk = next.value + controller.enqueue(chunk instanceof Uint8Array ? chunk : Buffer.from(chunk)) + } catch (error) { + finished = true + if (!source.destroyed) { + source.destroy() + } + controller.error(error) + } + }, + async cancel(reason) { + finished = true + try { + await options.onCancel?.(reason) + } finally { + if (!source.destroyed) { + source.destroy() + } + } + }, + }, + { + highWaterMark: 1, + size: () => 1, + }, + ) +} diff --git a/lib/task-material-processor.ts b/lib/task-material-processor.ts index 59cbeb1a..a1f3b628 100644 --- a/lib/task-material-processor.ts +++ b/lib/task-material-processor.ts @@ -562,6 +562,7 @@ export async function cleanupOnError( materialId: string | null, uploadIds: string[] = [], deleteMaterial = true, + deleteMaterialCondition?: Record, ): Promise { console.log('[Cleanup] Starting error cleanup...'); @@ -618,10 +619,21 @@ export async function cleanupOnError( // 删除素材记录 if (materialId && deleteMaterial) { try { - await prisma.material.delete({ - where: { id: materialId }, - }); - console.log(`[Cleanup] Deleted material record: ${materialId}`); + if (deleteMaterialCondition) { + const result = await prisma.material.deleteMany({ + where: { id: materialId, ...deleteMaterialCondition } as any, + }); + if (result.count === 1) { + console.log(`[Cleanup] Deleted conditionally claimed material record: ${materialId}`); + } else { + console.warn(`[Cleanup] Skipped material deletion because claim state changed: ${materialId}`); + } + } else { + await prisma.material.delete({ + where: { id: materialId }, + }); + console.log(`[Cleanup] Deleted material record: ${materialId}`); + } } catch (cleanupError) { console.error('[Cleanup] Failed to delete material record:', cleanupError); }