From e2f49112688376b7fc65726d6fe633ca2b222cdd Mon Sep 17 00:00:00 2001 From: chenpei Date: Fri, 31 Jul 2026 10:40:00 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E9=99=90=E5=88=B6=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E5=A4=84=E7=90=86=E5=86=85=E5=AD=98=E5=B3=B0?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/admin/regenerate-preview/route.ts | 6 +- .../by-barcode/route.ts | 28 +++ .../by-spu/route.ts | 28 +++ app/api/materials/upload/route.ts | 17 +- ...产图片视频内存事故与处理记录-2026-07-31.md | 44 ++++ ...-white-background-by-barcode-route.test.ts | 15 ++ ...odel-white-background-by-spu-route.test.ts | 1 + lib/image-processing-limiter.ts | 46 ++++ lib/image-processor.ts | 225 ++++++++++-------- lib/pdf-processor.ts | 3 + lib/performance-survey-image-processing.ts | 3 + lib/product-package-completion.ts | 10 +- lib/psd-processor.ts | 3 + lib/sharp-runtime.ts | 43 ++++ ...standard-model-white-background-service.ts | 10 + ...ard-model-white-background-upload-guard.ts | 83 ++++--- lib/video-processor.ts | 13 +- lib/video-transcode-queue.ts | 36 +-- lib/white-background-preview.ts | 49 ++-- 19 files changed, 476 insertions(+), 187 deletions(-) create mode 100644 docs/生产图片视频内存事故与处理记录-2026-07-31.md create mode 100644 lib/image-processing-limiter.ts create mode 100644 lib/sharp-runtime.ts diff --git a/app/api/admin/regenerate-preview/route.ts b/app/api/admin/regenerate-preview/route.ts index 678a898..4d68657 100644 --- a/app/api/admin/regenerate-preview/route.ts +++ b/app/api/admin/regenerate-preview/route.ts @@ -100,7 +100,11 @@ export async function POST(request: NextRequest) { // 读取图片 const buffer = await readFile(imagePath) - const metadata = await sharp(buffer).metadata() + const metadata = await sharp(buffer, { + limitInputPixels: 60_000_000, + sequentialRead: true, + failOn: 'error', + }).metadata() console.log('[Regenerate Preview] Image size:', metadata.width, 'x', metadata.height) // 切片处理 diff --git a/app/api/external/standard-model-white-background/by-barcode/route.ts b/app/api/external/standard-model-white-background/by-barcode/route.ts index 568e099..4bfbb75 100644 --- a/app/api/external/standard-model-white-background/by-barcode/route.ts +++ b/app/api/external/standard-model-white-background/by-barcode/route.ts @@ -22,6 +22,10 @@ import { } from '@/lib/standard-model-white-background' import { normalizeStandardModelWhiteBackgroundSourceType } from '@/lib/standard-model-white-background-metadata' import { uploadStandardModelWhiteBackground } from '@/lib/standard-model-white-background-service' +import { + STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS, + WhiteBackgroundUploadValidationError, +} from '@/lib/standard-model-white-background-upload-guard' export const runtime = 'nodejs' @@ -180,6 +184,22 @@ export async function POST(request: NextRequest) { let idempotencyRecordId: string | null = null try { + const contentLength = Number(request.headers.get('content-length')) + if (!Number.isFinite(contentLength) || contentLength <= 0) { + await writeApiLog({ apiKeyId: keyRecord.id, trace, statusCode: 411 }) + return finalizeResponse( + buildErrorResponse(411, 'CONTENT_LENGTH_REQUIRED', '必须提供有效的请求大小'), + trace, + ) + } + if (contentLength > STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS.maxRequestBytes) { + await writeApiLog({ apiKeyId: keyRecord.id, trace, statusCode: 413 }) + return finalizeResponse( + buildErrorResponse(413, 'REQUEST_TOO_LARGE', '白底图上传请求过大'), + trace, + ) + } + const formData = await request.formData() const clientRequestId = String(formData.get('clientRequestId') || '').trim() const barcode = String(formData.get('barcode') || '').trim() @@ -298,6 +318,14 @@ export async function POST(request: NextRequest) { return finalizeResponse(buildErrorResponse(error.status, error.code, error.message), trace) } + if (error instanceof WhiteBackgroundUploadValidationError) { + await writeApiLog({ apiKeyId: keyRecord.id, trace, statusCode: 400 }) + return finalizeResponse( + buildErrorResponse(400, 'INVALID_IMAGE_UPLOAD', error.message), + trace, + ) + } + const message = error instanceof Error ? error.message : '上传白底图失败' console.error('[external/standard-model-white-background/by-barcode] POST error:', error) await writeApiLog({ apiKeyId: keyRecord.id, trace, statusCode: 500 }) diff --git a/app/api/external/standard-model-white-background/by-spu/route.ts b/app/api/external/standard-model-white-background/by-spu/route.ts index 86e4f42..9014c3e 100644 --- a/app/api/external/standard-model-white-background/by-spu/route.ts +++ b/app/api/external/standard-model-white-background/by-spu/route.ts @@ -24,6 +24,10 @@ import { } from '@/lib/standard-model-white-background' import { normalizeStandardModelWhiteBackgroundSourceType } from '@/lib/standard-model-white-background-metadata' import { uploadStandardModelWhiteBackground } from '@/lib/standard-model-white-background-service' +import { + STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS, + WhiteBackgroundUploadValidationError, +} from '@/lib/standard-model-white-background-upload-guard' export const runtime = 'nodejs' @@ -574,6 +578,22 @@ export async function POST(request: NextRequest) { let idempotencyRecordId: string | null = null try { + const contentLength = Number(request.headers.get('content-length')) + if (!Number.isFinite(contentLength) || contentLength <= 0) { + await writeApiLog({ apiKeyId: keyRecord.id, trace, statusCode: 411 }) + return finalizeResponse( + buildErrorResponse(411, 'CONTENT_LENGTH_REQUIRED', '必须提供有效的请求大小'), + trace, + ) + } + if (contentLength > STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS.maxRequestBytes) { + await writeApiLog({ apiKeyId: keyRecord.id, trace, statusCode: 413 }) + return finalizeResponse( + buildErrorResponse(413, 'REQUEST_TOO_LARGE', '白底图上传请求过大'), + trace, + ) + } + const formData = await request.formData() const clientRequestId = getFormText(formData, 'clientRequestId') const materialType = getFormText(formData, 'materialType') @@ -746,6 +766,14 @@ export async function POST(request: NextRequest) { return finalizeResponse(buildErrorResponse(error.status, error.code, error.message), trace) } + if (error instanceof WhiteBackgroundUploadValidationError) { + await writeApiLog({ apiKeyId: keyRecord.id, trace, statusCode: 400 }) + return finalizeResponse( + buildErrorResponse(400, 'INVALID_IMAGE_UPLOAD', error.message), + trace, + ) + } + const message = error instanceof Error ? error.message : '上传白底图失败' console.error('[external/standard-model-white-background/by-spu] POST error:', error) await writeApiLog({ apiKeyId: keyRecord.id, trace, statusCode: 500 }) diff --git a/app/api/materials/upload/route.ts b/app/api/materials/upload/route.ts index 4cd1ad0..98bd08b 100644 --- a/app/api/materials/upload/route.ts +++ b/app/api/materials/upload/route.ts @@ -426,10 +426,11 @@ export async function POST(request: NextRequest) { console.log('[Upload] Preview image uploaded:', previewUrl); } - // 处理其他预览图（作为展示图）- 并行处理以提高速度 + // 处理其他预览图（作为展示图）。图片解码会使用 Sharp 原生内存， + // 这里必须串行处理，避免一个请求内同时解码多张大图。 // 详情页/详情模板：其他预览图也作为展示图处理 if (previewImages.length > 0 && !['模特'].includes(materialType)) { - console.log(`[Upload] Processing ${previewImages.length} preview images in parallel...`); + console.log(`[Upload] Processing ${previewImages.length} preview images sequentially...`); // 验证所有图片大小 for (let i = 0; i < previewImages.length; i++) { @@ -441,8 +442,8 @@ export async function POST(request: NextRequest) { } } - // 并行处理所有预览图 - const uploadPromises = previewImages.map(async (image, index) => { + for (let index = 0; index < previewImages.length; index += 1) { + const image = previewImages[index] console.log(`[Upload] Processing preview image ${index + 2}/${previewImages.length + 1}...`); const buffer = Buffer.from(await image.arrayBuffer()); @@ -464,12 +465,8 @@ export async function POST(request: NextRequest) { ); uploadedFileUrls.push(url); // 记录已上传的文件，用于失败时清理 console.log(`[Upload] Preview image ${index + 2} uploaded:`, url); - return url; - }); - - // 等待所有预览图上传完成 - const urls = await Promise.all(uploadPromises); - displayImages.push(...urls); + displayImages.push(url); + } console.log(`[Upload] All ${previewImages.length} preview images uploaded successfully`); } diff --git a/docs/生产图片视频内存事故与处理记录-2026-07-31.md b/docs/生产图片视频内存事故与处理记录-2026-07-31.md new file mode 100644 index 0000000..04ffa43 --- /dev/null +++ b/docs/生产图片视频内存事故与处理记录-2026-07-31.md @@ -0,0 +1,44 @@ +# 生产图片/视频处理内存事故与处理记录 + +日期：2026-07-31 + +## 结论 + +这次“处理图片后整个服务崩掉”不是单一 Node.js 请求超出 6GB 容器限制，而是生产宿主机发生了全局内存耗尽（`global_oom`）。 + +真正的内存来源分两层： + +1. 宿主机的 TrueNAS `middlewared` 长时间占用异常高的内存，历史峰值约 9.54GiB，并设置了 `oom_score_adj=-1000`，内核不会优先杀它。 +2. 应用容器内的 `libvips`（Sharp 的底层图片库）和 `ffmpeg` 在处理大图/视频时产生原生内存尖峰。它们是把整机推过 OOM 阈值的触发器，内核随后杀掉了 `next-server` 或 MinIO。 + +因此，单纯把 Node.js 堆内存或应用容器上限调大不能解决问题，反而可能更快吃光宿主机余量。 + +## 已核实的生产时间线 + +| 时间（北京时间） | 生产证据 | 业务对应 | +| --- | --- | --- | +| 7 月 30 日 15:18 | `libvips worker` 触发 `global_oom`；内核杀掉 `next-server`，其匿名 RSS 约 1.405GiB | 同时段任务详情完成接口进入 `sharp.metadata()`、`sliceImage/compressImage`，Nginx 随后出现 502 | +| 7 月 30 日 16:18 | 同一应用容器同时有两个 `ffmpeg`，RSS 约 642MiB 和 372–408MiB；之后 MinIO 也被杀 | 同时段连续创建淘系视频任务；转码队列未配置时默认并发为 2 | +| 7 月 30 日 17:39 | `libvips worker` 再次触发 `global_oom`；同时还有一个约 325MiB 的 `ffmpeg`，最终 `next-server` 被杀 | 图片任务详情和抖音视频任务叠加，随后大量 502/504 | + +三次事件都显示：宿主机没有 Swap，且是 `global_oom`，不是应用容器自己的 cgroup OOM。 + +## 本次代码处理 + +- Sharp 默认并发降为 1，限制 libvips 缓存和输入像素，并增加进程级图片处理门闩。 +- 白底图内部和外部接口统一增加请求大小、单图大小、图片数量、格式、像素和动图门禁；服务层再次校验，防止新调用方绕过路由。 +- 主上传的多张预览图改为串行处理，不再通过 `Promise.all` 同时解码多张图片。 +- 任务详情预览在第一次 `sharp.metadata()` 前增加像素上限；长图限制最大尺寸和切片数。 +- 视频转码队列固定为同一容器最多 1 个任务，FFmpeg 默认 1 线程，并限制进度日志缓存。 +- 同批次关闭上传路径越界、任意文件删除、跨用户白底图替换等高危入口，并移除头像菜单中的网络诊断/缓存刷新入口。 + +## 仍需运维处理的风险 + +代码修复不能替代宿主机治理。生产运维需要单独完成： + +- 查明 `middlewared` 从正常内存增长到数 GiB 的原因，并设置可用内存告警。 +- 评估在维护窗口增加 Swap 或等效内存缓冲；这不是永久修复，只是降低瞬时 OOM 概率。 +- 评估把 FFmpeg/Sharp 拆到独立 worker，避免媒体任务与 Next.js、MinIO 共用进程/容器。 +- 持续采集容器 RSS、cgroup `memory.events`、宿主机可用内存和 OOM 日志。 + +本次排查未重启、杀进程、清理生产数据或修改生产部署脚本。 diff --git a/lib/__tests__/standard-model-white-background-by-barcode-route.test.ts b/lib/__tests__/standard-model-white-background-by-barcode-route.test.ts index 94b9326..a36f138 100644 --- a/lib/__tests__/standard-model-white-background-by-barcode-route.test.ts +++ b/lib/__tests__/standard-model-white-background-by-barcode-route.test.ts @@ -82,6 +82,7 @@ describe('external standard-model-white-background by barcode routes', () => { headers: new Headers({ 'X-API-Key': 'test-key', 'user-agent': 'jest', + 'content-length': '1024', }), formData: jest.fn().mockResolvedValue(formData), } as any @@ -154,6 +155,20 @@ describe('external standard-model-white-background by barcode routes', () => { expect(uploadStandardModelWhiteBackground).not.toHaveBeenCalled() }) + test('缺少请求大小时在解析 multipart 前拒绝', async () => { + const request = { + headers: new Headers({ 'X-API-Key': 'test-key' }), + formData: jest.fn(), + } as any + + const response = await uploadByBarcode(request) + const body = await response.json() + + expect(response.status).toBe(411) + expect(body.error.code).toBe('CONTENT_LENGTH_REQUIRED') + expect(request.formData).not.toHaveBeenCalled() + }) + test('cleanup 只按条码解析到的 SKU 调用 SKU 范围清理', async () => { const response = await cleanupByBarcode( createCleanupRequest({ diff --git a/lib/__tests__/standard-model-white-background-by-spu-route.test.ts b/lib/__tests__/standard-model-white-background-by-spu-route.test.ts index b9e5b53..300fb50 100644 --- a/lib/__tests__/standard-model-white-background-by-spu-route.test.ts +++ b/lib/__tests__/standard-model-white-background-by-spu-route.test.ts @@ -95,6 +95,7 @@ describe('external standard-model-white-background by spu route', () => { headers: new Headers({ 'X-API-Key': 'test-key', 'user-agent': 'jest', + 'content-length': '1024', }), formData: jest.fn().mockResolvedValue(formData), } as any diff --git a/lib/image-processing-limiter.ts b/lib/image-processing-limiter.ts new file mode 100644 index 0000000..ee53e50 --- /dev/null +++ b/lib/image-processing-limiter.ts @@ -0,0 +1,46 @@ +const DEFAULT_MAX_CONCURRENT_IMAGE_JOBS = 1 +const MAX_ALLOWED_IMAGE_JOBS = 2 + +function readMaxConcurrentImageJobs(): number { + const configured = Number.parseInt(process.env.IMAGE_PROCESSING_MAX_CONCURRENT || '', 10) + if (!Number.isFinite(configured)) { + return DEFAULT_MAX_CONCURRENT_IMAGE_JOBS + } + + return Math.max(1, Math.min(MAX_ALLOWED_IMAGE_JOBS, configured)) +} + +const maxConcurrentImageJobs = readMaxConcurrentImageJobs() +let activeImageJobs = 0 +const waitingImageJobs: Array<() => void> = [] + +async function acquireImageProcessingSlot(): Promise { + if (activeImageJobs < maxConcurrentImageJobs) { + activeImageJobs += 1 + return + } + + await new Promise((resolve) => waitingImageJobs.push(resolve)) + activeImageJobs += 1 +} + +function releaseImageProcessingSlot(): void { + activeImageJobs = Math.max(0, activeImageJobs - 1) + const next = waitingImageJobs.shift() + if (next) { + next() + } +} + +/** + * 跨请求限制 Sharp 任务数。单个请求内串行并不能阻止多个请求同时解码大图， + * 所以白底图、预览图和长图处理都必须共用这个进程级门闩。 + */ +export async function withImageProcessingSlot(task: () => Promise): Promise { + await acquireImageProcessingSlot() + try { + return await task() + } finally { + releaseImageProcessingSlot() + } +} diff --git a/lib/image-processor.ts b/lib/image-processor.ts index 46f819e..68c0de8 100644 --- a/lib/image-processor.ts +++ b/lib/image-processor.ts @@ -1,4 +1,32 @@ import sharp from 'sharp' +import { withImageProcessingSlot } from '@/lib/image-processing-limiter' +import { configureSharpRuntime } from '@/lib/sharp-runtime' + +configureSharpRuntime() + +const IMAGE_PROCESSING_MAX_PIXELS = 60_000_000 +const IMAGE_PROCESSING_MAX_DIMENSION = 20_000 +const IMAGE_PROCESSING_MAX_SLICES = 60 + +function createSharpInput(buffer: Buffer) { + return sharp(buffer, { + limitInputPixels: IMAGE_PROCESSING_MAX_PIXELS, + sequentialRead: true, + failOn: 'error', + }) +} + +function assertProcessableMetadata(metadata: sharp.Metadata) { + if (!metadata.width || !metadata.height) { + throw new Error('无法获取图片尺寸') + } + if (metadata.width > IMAGE_PROCESSING_MAX_DIMENSION || metadata.height > IMAGE_PROCESSING_MAX_DIMENSION) { + throw new Error('图片尺寸过大') + } + if (Number(metadata.pages || 1) > 1) { + throw new Error('不支持动图或多页图片') + } +} interface ProcessImageOptions { maxSizeKB?: number // 最大文件大小（KB） @@ -24,63 +52,66 @@ export async function processImage( format = 'webp', // 默认webP格式 } = options - try { - // 获取原始图片信息 - const image = sharp(buffer) - const metadata = await image.metadata() + return withImageProcessingSlot(async () => { + try { + // 获取原始图片信息 + const image = createSharpInput(buffer) + const metadata = await image.metadata() + assertProcessableMetadata(metadata) - // 计算新尺寸（保持比例，同时限制宽度和高度） - let width = metadata.width || maxWidth - let height = metadata.height || maxHeight + // 计算新尺寸（保持比例，同时限制宽度和高度） + let width = metadata.width || maxWidth + let height = metadata.height || maxHeight - // 如果宽度超过限制 - if (width > maxWidth) { - const ratio = maxWidth / width - width = maxWidth - height = Math.round(height * ratio) - } + // 如果宽度超过限制 + if (width > maxWidth) { + const ratio = maxWidth / width + width = maxWidth + height = Math.round(height * ratio) + } - // 如果高度超过限制，再次调整 - if (height > maxHeight) { - const ratio = maxHeight / height - height = maxHeight - width = Math.round(width * ratio) - } + // 如果高度超过限制，再次调整 + if (height > maxHeight) { + const ratio = maxHeight / height + height = maxHeight + width = Math.round(width * ratio) + } - // 优化：初始质量从80降到70，在保证视觉质量的前提下减少文件大小 - let quality = 70 + // 优化：初始质量从80降到70，在保证视觉质量的前提下减少文件大小 + let quality = 70 - let processedBuffer: Buffer + let processedBuffer: Buffer - // 第一次处理 - processedBuffer = await image - .resize(width, height, { - fit: 'inside', - withoutEnlargement: true, - }) - .webp({ quality }) - .toBuffer() - - // 如果文件还是太大，逐步降低质量 - const maxBytes = maxSizeKB * 1024 - while (processedBuffer.length > maxBytes && quality > 20) { - quality -= 10 - processedBuffer = await sharp(buffer) + // 第一次处理 + processedBuffer = await createSharpInput(buffer) .resize(width, height, { fit: 'inside', withoutEnlargement: true, }) .webp({ quality }) .toBuffer() + + // 如果文件还是太大，逐步降低质量 + const maxBytes = maxSizeKB * 1024 + while (processedBuffer.length > maxBytes && quality > 20) { + quality -= 10 + processedBuffer = await createSharpInput(buffer) + .resize(width, height, { + fit: 'inside', + withoutEnlargement: true, + }) + .webp({ quality }) + .toBuffer() + } + + console.log(`图片处理完成: ${metadata.width}x${metadata.height} -> ${width}x${height}, ${(buffer.length / 1024).toFixed(2)}KB -> ${(processedBuffer.length / 1024).toFixed(2)}KB, 质量: ${quality}`) + + return processedBuffer + } catch (error) { + console.error('图片处理失败:', error) + throw new Error('图片处理失败') } - - console.log(`图片处理完成: ${metadata.width}x${metadata.height} -> ${width}x${height}, ${(buffer.length / 1024).toFixed(2)}KB -> ${(processedBuffer.length / 1024).toFixed(2)}KB, 质量: ${quality}`) - - return processedBuffer - } catch (error) { - console.error('图片处理失败:', error) - throw new Error('图片处理失败') - } + }) } /** @@ -103,62 +134,50 @@ export async function sliceImage( maxHeight: number = 1500, maxSizeKB: number = 500 ): Promise { - try { - const image = sharp(buffer) - const metadata = await image.metadata() - - if (!metadata.width || !metadata.height) { - throw new Error('无法获取图片尺寸') - } + return withImageProcessingSlot(async () => { + try { + const image = createSharpInput(buffer) + const metadata = await image.metadata() + assertProcessableMetadata(metadata) - const { width, height } = metadata - const slices: Buffer[] = [] - const maxBytes = maxSizeKB * 1024 + const { width, height } = metadata + const slices: Buffer[] = [] + const maxBytes = maxSizeKB * 1024 // 如果图片高度不超过maxHeight，直接返回原图（压缩处理） - if (height <= maxHeight) { - let quality = 90 - let processedBuffer = await image - .webp({ quality }) - .toBuffer() - - // 如果文件还是太大，逐步降低质量 - while (processedBuffer.length > maxBytes && quality > 20) { - quality -= 10 - processedBuffer = await image + if (height <= maxHeight) { + let quality = 90 + let processedBuffer = await createSharpInput(buffer) .webp({ quality }) .toBuffer() + + // 如果文件还是太大，逐步降低质量 + while (processedBuffer.length > maxBytes && quality > 20) { + quality -= 10 + processedBuffer = await createSharpInput(buffer) + .webp({ quality }) + .toBuffer() + } + + console.log(`图片无需切割: ${width}x${height}, 压缩到 ${(processedBuffer.length / 1024).toFixed(2)}KB, 质量: ${quality}`) + return [processedBuffer] } - - console.log(`图片无需切割: ${width}x${height}, 压缩到 ${(processedBuffer.length / 1024).toFixed(2)}KB, 质量: ${quality}`) - return [processedBuffer] - } // 计算需要切割的片数 - const sliceCount = Math.ceil(height / maxHeight) - console.log(`开始切割图片: ${width}x${height} -> ${sliceCount}个片段，每片限制${maxSizeKB}KB`) + const sliceCount = Math.ceil(height / maxHeight) + if (sliceCount > IMAGE_PROCESSING_MAX_SLICES) { + throw new Error(`图片过长，最多支持 ${IMAGE_PROCESSING_MAX_SLICES} 个切片`) + } + console.log(`开始切割图片: ${width}x${height} -> ${sliceCount}个片段，每片限制${maxSizeKB}KB`) // 切割图片 - for (let i = 0; i < sliceCount; i++) { - const top = i * maxHeight - const extractHeight = Math.min(maxHeight, height - top) + for (let i = 0; i < sliceCount; i++) { + const top = i * maxHeight + const extractHeight = Math.min(maxHeight, height - top) // 初始质量 - let quality = 90 - let sliceBuffer = await sharp(buffer) - .extract({ - left: 0, - top: top, - width: width, - height: extractHeight - }) - .webp({ quality }) - .toBuffer() - - // 如果文件还是太大，逐步降低质量 - while (sliceBuffer.length > maxBytes && quality > 20) { - quality -= 10 - sliceBuffer = await sharp(buffer) + let quality = 90 + let sliceBuffer = await createSharpInput(buffer) .extract({ left: 0, top: top, @@ -167,17 +186,31 @@ export async function sliceImage( }) .webp({ quality }) .toBuffer() + + // 如果文件还是太大，逐步降低质量 + while (sliceBuffer.length > maxBytes && quality > 20) { + quality -= 10 + sliceBuffer = await createSharpInput(buffer) + .extract({ + left: 0, + top: top, + width: width, + height: extractHeight + }) + .webp({ quality }) + .toBuffer() + } + + slices.push(sliceBuffer) + console.log(`切割片段 ${i + 1}/${sliceCount}: 高度 ${extractHeight}px, 大小 ${(sliceBuffer.length / 1024).toFixed(2)}KB, 质量: ${quality}`) } - slices.push(sliceBuffer) - console.log(`切割片段 ${i + 1}/${sliceCount}: 高度 ${extractHeight}px, 大小 ${(sliceBuffer.length / 1024).toFixed(2)}KB, 质量: ${quality}`) + return slices + } catch (error) { + console.error('图片切割失败:', error) + throw new Error('图片切割失败') } - - return slices - } catch (error) { - console.error('图片切割失败:', error) - throw new Error('图片切割失败') - } + }) } /** diff --git a/lib/pdf-processor.ts b/lib/pdf-processor.ts index da89005..f5340ba 100644 --- a/lib/pdf-processor.ts +++ b/lib/pdf-processor.ts @@ -11,6 +11,9 @@ import { readFile, unlink, writeFile, open } from 'fs/promises' import { tmpdir } from 'os' import path from 'path' import sharp from 'sharp' +import { configureSharpRuntime } from '@/lib/sharp-runtime' + +configureSharpRuntime() const execFileAsync = promisify(execFile) diff --git a/lib/performance-survey-image-processing.ts b/lib/performance-survey-image-processing.ts index 46d63e5..64ef760 100644 --- a/lib/performance-survey-image-processing.ts +++ b/lib/performance-survey-image-processing.ts @@ -1,4 +1,7 @@ import sharp from 'sharp' +import { configureSharpRuntime } from '@/lib/sharp-runtime' + +configureSharpRuntime() export const PERFORMANCE_SURVEY_IMAGE_LIMITS = { maxSourceCount: 60, diff --git a/lib/product-package-completion.ts b/lib/product-package-completion.ts index 145a3e7..2f96db6 100644 --- a/lib/product-package-completion.ts +++ b/lib/product-package-completion.ts @@ -338,7 +338,15 @@ async function generatePreviewsFromAssets( buffer = await readAssetBuffer(asset, type) - const metadata = await sharp.default(buffer).metadata() + if (buffer.length > 200 * 1024 * 1024) { + throw new Error(`预览源图片 ${asset.fileName} 不能超过 200MB`) + } + + const metadata = await sharp.default(buffer, { + limitInputPixels: 60_000_000, + sequentialRead: true, + failOn: 'error', + }).metadata() const shouldSlice = type === 'detail' && metadata.height && metadata.height > 1500 const buffers = shouldSlice ? await sliceImage(buffer, 1500, 200) diff --git a/lib/psd-processor.ts b/lib/psd-processor.ts index d51925b..2455247 100644 --- a/lib/psd-processor.ts +++ b/lib/psd-processor.ts @@ -1,6 +1,9 @@ import { readPsd, initializeCanvas } from 'ag-psd' import sharp from 'sharp' import fs from 'fs/promises' +import { configureSharpRuntime } from '@/lib/sharp-runtime' + +configureSharpRuntime() // 尝试初始化 canvas（如果可用） try { diff --git a/lib/sharp-runtime.ts b/lib/sharp-runtime.ts new file mode 100644 index 0000000..ba8ddf5 --- /dev/null +++ b/lib/sharp-runtime.ts @@ -0,0 +1,43 @@ +import sharp from 'sharp' + +const DEFAULT_SHARP_CONCURRENCY = 1 +const MAX_SHARP_CONCURRENCY = 2 +const SHARP_CACHE_MEMORY_MB = 128 +const SHARP_CACHE_ITEMS = 64 + +function readConcurrency(): number { + const configured = Number.parseInt(process.env.SHARP_CONCURRENCY || '', 10) + if (!Number.isFinite(configured)) { + return DEFAULT_SHARP_CONCURRENCY + } + + return Math.max(1, Math.min(MAX_SHARP_CONCURRENCY, configured)) +} + +let configured = false + +/** + * Sharp 的原生解码/编码内存不受 Node old-space 完全约束。 + * 生产应用与 Next.js 共用一个容器，因此默认只允许一个 Sharp 任务， + * 并关闭磁盘缓存、限制内存缓存，避免并发图片请求把宿主机推入 OOM。 + */ +export function configureSharpRuntime(): void { + if (configured) { + return + } + + // 测试或极简运行时可能只提供 sharp() 工厂，没有全局调优方法； + // 这时保留原有行为，不让配置本身阻断业务。 + if (typeof sharp.concurrency !== 'function' || typeof sharp.cache !== 'function') { + configured = true + return + } + + sharp.concurrency(readConcurrency()) + sharp.cache({ + memory: SHARP_CACHE_MEMORY_MB, + items: SHARP_CACHE_ITEMS, + files: 0, + }) + configured = true +} diff --git a/lib/standard-model-white-background-service.ts b/lib/standard-model-white-background-service.ts index 25ac289..540f7e8 100644 --- a/lib/standard-model-white-background-service.ts +++ b/lib/standard-model-white-background-service.ts @@ -37,6 +37,9 @@ import { } from '@/lib/standard-model-white-background' import type { StandardModelWhiteBackgroundSourceType } from '@/lib/standard-model-white-background-metadata' import { buildWhiteBackgroundWebpVariants } from '@/lib/white-background-preview' +import { + assertWhiteBackgroundUploadFiles, +} from '@/lib/standard-model-white-background-upload-guard' type WhiteBackgroundAssetRecord = WhiteBackgroundAssetLike & { id: string @@ -197,6 +200,13 @@ export async function uploadStandardModelWhiteBackground( throw new Error('请至少上传 1 张白底图') } + // 服务层再次执行门禁，避免外部 API 或未来新增调用方绕过路由校验， + // 把超大、高像素或动图直接交给 Sharp。 + await assertWhiteBackgroundUploadFiles({ + fixedImages: input.fixedImages, + otherImages, + }) + const existingAssets = await prisma.materialAsset.findMany({ where: { materialId: input.materialId, diff --git a/lib/standard-model-white-background-upload-guard.ts b/lib/standard-model-white-background-upload-guard.ts index 995f299..2f934f5 100644 --- a/lib/standard-model-white-background-upload-guard.ts +++ b/lib/standard-model-white-background-upload-guard.ts @@ -2,6 +2,10 @@ import sharp from 'sharp' import type { StandardModelWhiteBgKey, } from '@/lib/standard-model-white-background' +import { withImageProcessingSlot } from '@/lib/image-processing-limiter' +import { configureSharpRuntime } from '@/lib/sharp-runtime' + +configureSharpRuntime() export const STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS = { maxOtherImages: 12, @@ -10,6 +14,7 @@ export const STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS = { maxTotalBytes: 100 * 1024 * 1024, maxPixelsPerImage: 40_000_000, maxTotalPixels: 160_000_000, + maxRequestBytes: 105 * 1024 * 1024, } as const const ALLOWED_IMAGE_MIME_TYPES = new Set([ @@ -111,47 +116,49 @@ export async function assertWhiteBackgroundUploadFiles(input: { throw new WhiteBackgroundUploadValidationError('单次上传白底图总体积不能超过 100MB') } - let totalPixels = 0 - for (let index = 0; index < files.length; index++) { - const file = files[index] - const label = `图片「${file.name || index + 1}」` + await withImageProcessingSlot(async () => { + let totalPixels = 0 + for (let index = 0; index < files.length; index++) { + const file = files[index] + const label = `图片「${file.name || index + 1}」` - try { - const buffer = Buffer.from(await file.arrayBuffer()) - const metadata = await sharp(buffer, { - limitInputPixels: STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS.maxPixelsPerImage, - sequentialRead: true, - failOn: 'error', - }).metadata() + try { + const buffer = Buffer.from(await file.arrayBuffer()) + const metadata = await sharp(buffer, { + limitInputPixels: STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS.maxPixelsPerImage, + sequentialRead: true, + failOn: 'error', + }).metadata() - if ( - !metadata.width || - !metadata.height || - !metadata.format || - !ALLOWED_IMAGE_FORMATS.has(metadata.format) - ) { - throw new WhiteBackgroundUploadValidationError(`${label}不是支持的图片`) - } - if (Number(metadata.pages || 1) > 1) { - throw new WhiteBackgroundUploadValidationError(`${label}不支持动图或多页图片`) - } + if ( + !metadata.width || + !metadata.height || + !metadata.format || + !ALLOWED_IMAGE_FORMATS.has(metadata.format) + ) { + throw new WhiteBackgroundUploadValidationError(`${label}不是支持的图片`) + } + if (Number(metadata.pages || 1) > 1) { + throw new WhiteBackgroundUploadValidationError(`${label}不支持动图或多页图片`) + } - const pixels = assertWhiteBackgroundImageDimensions( - metadata.width, - metadata.height, - label, - ) - totalPixels += pixels - if (totalPixels > STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS.maxTotalPixels) { - throw new WhiteBackgroundUploadValidationError('单次上传白底图总像素量过大') + const pixels = assertWhiteBackgroundImageDimensions( + metadata.width, + metadata.height, + label, + ) + totalPixels += pixels + if (totalPixels > STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS.maxTotalPixels) { + throw new WhiteBackgroundUploadValidationError('单次上传白底图总像素量过大') + } + } catch (error) { + if (error instanceof WhiteBackgroundUploadValidationError) { + throw error + } + throw new WhiteBackgroundUploadValidationError( + `${label}无法读取或像素总量过大`, + ) } - } catch (error) { - if (error instanceof WhiteBackgroundUploadValidationError) { - throw error - } - throw new WhiteBackgroundUploadValidationError( - `${label}无法读取或像素总量过大`, - ) } - } + }) } diff --git a/lib/video-processor.ts b/lib/video-processor.ts index 8b9eebd..aab3712 100644 --- a/lib/video-processor.ts +++ b/lib/video-processor.ts @@ -38,6 +38,8 @@ export const TRANSCODE_CONFIG = { timeout: parseInt(process.env.VIDEO_TRANSCODE_TIMEOUT || '3600000', 10), // 临时文件大小限制（字节，默认 10GB） maxTempFileSize: parseInt(process.env.VIDEO_MAX_TEMP_FILE_SIZE || '10737418240', 10), + // 与 Next.js 共用容器时限制 ffmpeg 线程，避免单个转码任务占满 CPU 和原生内存 + threads: Math.max(1, Math.min(2, parseInt(process.env.VIDEO_TRANSCODE_THREADS || '1', 10) || 1)), } interface VideoTranscodeOptions { @@ -199,7 +201,7 @@ async function execFFmpegWithTimeout( onProgress?: (progress: number) => void // progress: 0-100 ): Promise<{ stdout: string; stderr: string }> { return new Promise((resolve, reject) => { - const childProcess = exec(command, { maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => { + const childProcess = exec(command, { maxBuffer: 4 * 1024 * 1024 }, (error, stdout, stderr) => { if (error) { // FFmpeg 即使成功也可能返回非零退出码，需要检查 stderr 是否包含错误 if (stderr && !stderr.includes('frame=') && !stderr.includes('Duration:')) { @@ -217,7 +219,8 @@ async function execFFmpegWithTimeout( if (onProgress && childProcess.stderr) { childProcess.stderr.on('data', (chunk: Buffer) => { - stderrBuffer += chunk.toString() + // 只保留最近一段日志；FFmpeg 长任务会持续输出进度，不能让字符串无限增长。 + stderrBuffer = `${stderrBuffer}${chunk.toString()}`.slice(-64 * 1024) // 提取视频时长 const durationMatch = stderrBuffer.match(/Duration:\s*(\d{2}):(\d{2}):(\d{2})\.(\d{2})/) @@ -303,7 +306,7 @@ async function transcodeToPreview( * -preset medium: 平衡质量和速度（H.265 转码时间约为 H.264 的 3-5 倍） * -crf 22: 预览视频质量（中等-高质量，适合 4K 源视频缩放到 720P） */ - const command = `ffmpeg -i "${inputPath}" -vf "scale=-2:${height}:flags=lanczos" -c:v libx265 -preset medium -crf ${TRANSCODE_CONFIG.previewCrf} -c:a aac -b:a 128k -movflags +faststart -y "${outputPath}"` + const command = `ffmpeg -threads ${TRANSCODE_CONFIG.threads} -i "${inputPath}" -vf "scale=-2:${height}:flags=lanczos" -c:v libx265 -preset medium -crf ${TRANSCODE_CONFIG.previewCrf} -c:a aac -b:a 128k -movflags +faststart -y "${outputPath}"` const { stdout, stderr } = await execFFmpegWithTimeout(command, TRANSCODE_CONFIG.timeout, onProgress) // ffmpeg 通常将进度信息输出到 stderr，这是正常的 @@ -330,7 +333,7 @@ async function generateClip( * -an: 移除音轨（卡片不需要声音，节省文件大小） * 使用 libx265 (H.265) 编码器，相比 libx264 能以更低码率获得更好质量 */ - const command = `ffmpeg -i "${inputPath}" -t ${duration} -vf "scale=-2:${height}:flags=lanczos" -c:v libx265 -preset ultrafast -crf ${TRANSCODE_CONFIG.clipCrf} -an -movflags +faststart -y "${outputPath}"` + const command = `ffmpeg -threads ${TRANSCODE_CONFIG.threads} -i "${inputPath}" -t ${duration} -vf "scale=-2:${height}:flags=lanczos" -c:v libx265 -preset ultrafast -crf ${TRANSCODE_CONFIG.clipCrf} -an -movflags +faststart -y "${outputPath}"` const { stdout, stderr } = await execFFmpegWithTimeout(command, TRANSCODE_CONFIG.timeout, onProgress) if (stderr && !stderr.includes('frame=') && !stderr.includes('Duration:')) { @@ -349,7 +352,7 @@ async function extractCoverImage( outputPath: string ): Promise { const height = TRANSCODE_CONFIG.previewResolution - const command = `ffmpeg -i "${videoPath}" -vf "scale=-2:${height}:flags=lanczos" -frames:v 1 -y "${outputPath}"` + const command = `ffmpeg -threads ${TRANSCODE_CONFIG.threads} -i "${videoPath}" -vf "scale=-2:${height}:flags=lanczos" -frames:v 1 -y "${outputPath}"` const { stdout, stderr } = await execFFmpegWithTimeout(command) if (stderr && !stderr.includes('frame=') && !stderr.includes('Duration:')) { diff --git a/lib/video-transcode-queue.ts b/lib/video-transcode-queue.ts index 95f0a07..517c671 100644 --- a/lib/video-transcode-queue.ts +++ b/lib/video-transcode-queue.ts @@ -4,15 +4,13 @@ * 用于控制转码任务的并发数，充分利用多核 CPU * * 优化说明： - * - 默认并发数：2（充分利用多核 CPU） + * - 默认并发数：1（与 Next.js 共用容器，优先保护内存） * - 可以通过环境变量配置：VIDEO_TRANSCODE_MAX_CONCURRENT * - 支持队列管理，避免系统过载 * - 支持部署中断后自动恢复转码任务 * * 并发数建议： - * - H.265 编码：2 并发（CPU 占用 60-80%） - * - H.265 编码：3-4 并发（高性能服务器，CPU 占用 80-100%） - * - H.264 编码：3-4 并发（CPU 占用 60-80%） + * - 当前生产容器只允许 1 个转码任务，避免 ffmpeg 与 Sharp 争抢宿主机内存。 */ import { LEGACY_COMPATIBLE_VIDEO_MATERIAL_TYPES } from './video-material-types' @@ -55,6 +53,8 @@ interface RecoverInterruptedTasksOptions { videoTypes?: string[] } +const SAFE_MAX_CONCURRENT_IN_SHARED_CONTAINER = 1 + class VideoTranscodeQueue { private queue: QueuedTask[] = [] private processing: Set = new Set() @@ -64,21 +64,21 @@ class VideoTranscodeQueue { private prisma: any = null // R1修复：Prisma 客户端延迟初始化 constructor(maxConcurrent?: number) { - // 默认并发数计算逻辑： - // 1. 优先使用环境变量配置 - // 2. 如果没有配置，使用默认值 2（充分利用多核 CPU） - // 3. 可以根据实际负载情况调整到 3-4（高性能服务器） + // 当前 Next.js、Sharp 和 ffmpeg 共用一个容器；即使环境变量误配为 2， + // 也不能让两个 ffmpeg 同时进入同一容器。后续拆出独立 worker 后再放宽。 const envConcurrent = process.env.VIDEO_TRANSCODE_MAX_CONCURRENT - if (envConcurrent) { - this.maxConcurrent = parseInt(envConcurrent, 10) || 2 - } else if (maxConcurrent !== undefined) { - this.maxConcurrent = maxConcurrent - } else { - // 默认值：2 并发（充分利用多核 CPU，提升 CPU 使用率） - // 可以根据实际负载情况调整到 3-4（高性能服务器） - // 注意：使用 H.265 (libx265) 时，2 并发可以达到 60-80% CPU 使用率 - this.maxConcurrent = 2 - } + const requestedConcurrent = envConcurrent + ? parseInt(envConcurrent, 10) + : maxConcurrent + this.maxConcurrent = Math.max( + 1, + Math.min( + SAFE_MAX_CONCURRENT_IN_SHARED_CONTAINER, + typeof requestedConcurrent === 'number' && Number.isFinite(requestedConcurrent) + ? requestedConcurrent + : 1, + ), + ) console.log(`[TranscodeQueue] Initialized with maxConcurrent: ${this.maxConcurrent}`) } diff --git a/lib/white-background-preview.ts b/lib/white-background-preview.ts index 37189d9..182eb97 100644 --- a/lib/white-background-preview.ts +++ b/lib/white-background-preview.ts @@ -6,6 +6,11 @@ import { buildWhiteBackgroundPublicDisplayPathFromAssetPath, } from '@/lib/public-display' import { getWhiteBackgroundPreviewRelativePathFromAssetPath } from '@/lib/standard-model-white-background' +import { withImageProcessingSlot } from '@/lib/image-processing-limiter' +import { configureSharpRuntime } from '@/lib/sharp-runtime' +import { STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS } from '@/lib/standard-model-white-background-upload-guard' + +configureSharpRuntime() export const WHITE_BACKGROUND_PREVIEW_MAX_DIMENSION = 1500 @@ -38,26 +43,34 @@ function resolveAbsoluteLocalPath(fileUrl: string): string { } export async function buildWhiteBackgroundWebpVariants(buffer: Buffer) { - const original = await sharp(buffer) - .rotate() - .webp(WHITE_BACKGROUND_ORIGINAL_WEBP_OPTIONS) - .toBuffer() + return withImageProcessingSlot(async () => { + const sharpOptions = { + limitInputPixels: STANDARD_MODEL_WHITE_BG_UPLOAD_LIMITS.maxPixelsPerImage, + sequentialRead: true, + failOn: 'error' as const, + } - const preview = await sharp(buffer) - .rotate() - .resize({ - width: WHITE_BACKGROUND_PREVIEW_MAX_DIMENSION, - height: WHITE_BACKGROUND_PREVIEW_MAX_DIMENSION, - fit: 'inside', - withoutEnlargement: true, - }) - .webp(WHITE_BACKGROUND_PREVIEW_WEBP_OPTIONS) - .toBuffer() + const original = await sharp(buffer, sharpOptions) + .rotate() + .webp(WHITE_BACKGROUND_ORIGINAL_WEBP_OPTIONS) + .toBuffer() - return { - original, - preview, - } + const preview = await sharp(buffer, sharpOptions) + .rotate() + .resize({ + width: WHITE_BACKGROUND_PREVIEW_MAX_DIMENSION, + height: WHITE_BACKGROUND_PREVIEW_MAX_DIMENSION, + fit: 'inside', + withoutEnlargement: true, + }) + .webp(WHITE_BACKGROUND_PREVIEW_WEBP_OPTIONS) + .toBuffer() + + return { + original, + preview, + } + }) } export function resolveWhiteBackgroundPreviewFileUrl(fileUrl?: string | null): string | null {