From 831ce10a5baa97bd4cefb00c0e4bc2332c91da0e Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 04:28:56 +0800 Subject: [PATCH] fix: use mask edit for product lock scale adjust --- app/api/canvas/local-edit/route.ts | 307 +++++++++++++++++++---------- 1 file changed, 198 insertions(+), 109 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index dec126da7..7dbc354a7 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -127,6 +127,7 @@ type ScaleAdjustPatchReference = { backgroundPlateFullUrl?: string | null; compositeClipMaskUrl?: string | null; productLockMethod?: string | null; + productLockEditMaskUrl?: string | null; productLockOverlayUrl?: string | null; productLockProtectMaskUrl?: string | null; sourceCropUrl: string; @@ -466,7 +467,7 @@ function resolveScaleAdjustPreferredModelId(params: { if (params.preferMaskCapableGpt) { const maskCapableGptImage2 = gptImage2Candidates .filter(isScaleAdjustMaskCapableModel) - .filter(isScaleAdjustHealthyModel); + .filter((entry) => entry.model.healthStatus !== "auto_disabled"); if (maskCapableGptImage2[0]) { return maskCapableGptImage2[0].model.id; } @@ -2232,6 +2233,35 @@ async function buildScaleAdjustCompositeClipMaskBuffer(params: { .toBuffer(); } +async function buildScaleAdjustTransparentEditMaskFromOpacityMask(params: { + height: number; + maskBuffer: Buffer; + width: number; +}) { + const { height, maskBuffer, width } = params; + const editAlpha = await sharp(maskBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .raw() + .toBuffer(); + const rgba = Buffer.alloc(width * height * 4); + for (let index = 0; index < width * height; index += 1) { + const offset = index * 4; + rgba[offset] = 0; + rgba[offset + 1] = 0; + rgba[offset + 2] = 0; + rgba[offset + 3] = clamp(255 - (editAlpha[index] || 0), 0, 255); + } + return sharp(rgba, { + raw: { channels: 4, height, width }, + }) + .png({ compressionLevel: 9 }) + .toBuffer(); +} + function getScaleAdjustAlphaBoundingBox(params: { height: number; rawRgba: Buffer; @@ -3633,50 +3663,59 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { .png({ compressionLevel: 6 }) .toBuffer() : null; - const productLockOverlayBuffer = productLockComposites.length - ? await sharp({ - create: { - background: { alpha: 0, b: 0, g: 0, r: 0 }, + const productLockOverlayBuffer = productLockComposites.length + ? await sharp({ + create: { + background: { alpha: 0, b: 0, g: 0, r: 0 }, channels: 4, height: sourceHeight, width: sourceWidth, }, }) .composite(productLockComposites) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const productLockProtectMaskBuffer = productLockProtectMaskComposites.length - ? await sharp({ - create: { - background: { alpha: 0, b: 255, g: 255, r: 255 }, - channels: 4, - height: sourceHeight, - width: sourceWidth, - }, - }) - .composite(productLockProtectMaskComposites) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const compositeClipMaskBuffer = - compositeClipComposites.length || targetClipComposites.length + .png({ compressionLevel: 6 }) + .toBuffer() + : null; + const productLockProtectMaskBuffer = productLockProtectMaskComposites.length + ? await sharp({ + create: { + background: { alpha: 0, b: 255, g: 255, r: 255 }, + channels: 4, + height: sourceHeight, + width: sourceWidth, + }, + }) + .composite(productLockProtectMaskComposites) + .png({ compressionLevel: 6 }) + .toBuffer() + : null; + const compositeClipMaskBuffer = + compositeClipComposites.length || targetClipComposites.length ? await buildScaleAdjustCompositeClipMaskBuffer({ height: sourceHeight, - sourceMaskComposites: compositeClipComposites, - targetMaskComposites: targetClipComposites, - width: sourceWidth, - }) - : null; + sourceMaskComposites: compositeClipComposites, + targetMaskComposites: targetClipComposites, + width: sourceWidth, + }) + : null; + const productLockEditMaskBuffer = + productLockComposites.length && compositeClipMaskBuffer + ? await buildScaleAdjustTransparentEditMaskFromOpacityMask({ + height: sourceHeight, + maskBuffer: compositeClipMaskBuffer, + width: sourceWidth, + }) + : null; - return { - backgroundPlateBuffer, - buffer, - compositeClipMaskBuffer, - productLockOverlayBuffer, - productLockProtectMaskBuffer, - }; - } + return { + backgroundPlateBuffer, + buffer, + compositeClipMaskBuffer, + productLockEditMaskBuffer, + productLockOverlayBuffer, + productLockProtectMaskBuffer, + }; +} async function createScaleAdjustPatchReference(params: { fullFrameReferenceOnly?: boolean; @@ -3718,13 +3757,14 @@ async function createScaleAdjustPatchReference(params: { const targetDraftBuffer = targetDraftReference.buffer; const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { - const [ - targetDraftFullUpload, - backgroundPlateFullUpload, - productLockOverlayUpload, - productLockProtectMaskUpload, - compositeClipMaskUpload, - ] = await Promise.all([ + const [ + targetDraftFullUpload, + backgroundPlateFullUpload, + productLockOverlayUpload, + productLockProtectMaskUpload, + compositeClipMaskUpload, + productLockEditMaskUpload, + ] = await Promise.all([ uploadFile( `canvas-scale-adjust-full-target-draft-${suffix}.png`, toArrayBuffer(targetDraftBuffer) @@ -3735,33 +3775,40 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.backgroundPlateBuffer) ) : Promise.resolve(null), - targetDraftReference.productLockOverlayBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockOverlayBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockProtectMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.compositeClipMaskBuffer - ? uploadFile( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + targetDraftReference.productLockOverlayBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockOverlayBuffer) + ) + : Promise.resolve(null), + targetDraftReference.productLockProtectMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) + ) + : Promise.resolve(null), + targetDraftReference.compositeClipMaskBuffer + ? uploadFile( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) ) : Promise.resolve(null), + targetDraftReference.productLockEditMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockEditMaskBuffer) + ) + : Promise.resolve(null), ]); return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, - compositeClipMaskUrl: compositeClipMaskUpload?.url || null, - productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, - productLockOverlayUrl: productLockOverlayUpload?.url || null, - productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, + compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, + productLockEditMaskUrl: productLockEditMaskUpload?.url || null, + productLockOverlayUrl: productLockOverlayUpload?.url || null, + productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, + sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, targetDraftUrl: targetDraftFullUpload.url, @@ -3780,15 +3827,16 @@ async function createScaleAdjustPatchReference(params: { .png({ compressionLevel: 6 }) .toBuffer(), ]); - const [ - sourceCropUpload, - targetDraftUpload, - targetDraftFullUpload, - backgroundPlateFullUpload, - productLockOverlayUpload, - productLockProtectMaskUpload, - compositeClipMaskUpload, - ] = await Promise.all([ + const [ + sourceCropUpload, + targetDraftUpload, + targetDraftFullUpload, + backgroundPlateFullUpload, + productLockOverlayUpload, + productLockProtectMaskUpload, + compositeClipMaskUpload, + productLockEditMaskUpload, + ] = await Promise.all([ uploadFile( `canvas-scale-adjust-workspace-source-${suffix}.png`, toArrayBuffer(sourceCropBuffer) @@ -3807,33 +3855,40 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.backgroundPlateBuffer) ) : Promise.resolve(null), - targetDraftReference.productLockOverlayBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockOverlayBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockProtectMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.compositeClipMaskBuffer - ? uploadFile( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + targetDraftReference.productLockOverlayBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockOverlayBuffer) + ) + : Promise.resolve(null), + targetDraftReference.productLockProtectMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) + ) + : Promise.resolve(null), + targetDraftReference.compositeClipMaskBuffer + ? uploadFile( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) ) : Promise.resolve(null), + targetDraftReference.productLockEditMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockEditMaskBuffer) + ) + : Promise.resolve(null), ]); return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, - compositeClipMaskUrl: compositeClipMaskUpload?.url || null, - productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, - productLockOverlayUrl: productLockOverlayUpload?.url || null, - productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl: sourceCropUpload.url, + compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, + productLockEditMaskUrl: productLockEditMaskUpload?.url || null, + productLockOverlayUrl: productLockOverlayUpload?.url || null, + productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, + sourceCropUrl: sourceCropUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, targetDraftUrl: targetDraftUpload.url, @@ -4821,7 +4876,8 @@ function buildScaleAdjustFullFramePatchPrompt( function buildScaleAdjustProductLockBackgroundPrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, - sourceSize: { width: number; height: number } + sourceSize: { width: number; height: number }, + options: { maskEdit?: boolean } = {} ) { const regionLines = regions .map((region) => { @@ -4836,12 +4892,16 @@ function buildScaleAdjustProductLockBackgroundPrompt( return [ "Full-frame background repair for product-locked scale adjustment.", "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.", - "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", + options.maskEdit + ? "Input image 1 is the original clean full photo. The supplied edit mask marks the old selected furniture/product footprint that must be repaired as natural background; do not edit outside the masked area." + : "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the final image. The system will composite the locked original product pixels into the final target footprint after your background image is returned.", "Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.", "Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.", "Keep all non-target content stable: sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", - "Remove draft artifacts from input image 2: pale wash blocks, rough masks, missing patches, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, and old-size residues. They are only edit guides, not final content.", + options.maskEdit + ? "The edit mask is only an invisible repair boundary. Do not output pale wash blocks, rough mask shapes, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, or old-size residues." + : "Remove draft artifacts from input image 2: pale wash blocks, rough masks, missing patches, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, and old-size residues. They are only edit guides, not final content.", "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, crop borders, or any annotation artifacts.", "中文硬约束：这是产品结构锁定流程。你只生成背景底图，不要生成、移动、缩放或重绘目标家具/商品；目标家具/商品会在你输出后由系统用原图锁定像素叠回目标位置。必须擦除旧位置的目标家具/商品并自然补全墙面、窗帘、地面、阴影和背景。非目标人物必须保持原样，不能变脸、变形、改变衣服、手、腿、脚或姿势。最终背景图不能有矩形雾块、贴片边、黑洞、残影、标注框，也不能出现第二个目标家具/商品。", "", @@ -5201,6 +5261,10 @@ export async function POST(request: NextRequest) { useScaleAdjustFullFrameStructurePatch && Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) && Boolean(scaleAdjustPatchReference?.productLockOverlayUrl); + const useScaleAdjustProductLockMaskEdit = + useScaleAdjustProductLockComposite && + isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry) && + Boolean(scaleAdjustPatchReference?.productLockEditMaskUrl); const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const includeScaleAdjustPatchSourceReference = Boolean(scaleAdjustPatchReference); @@ -5215,7 +5279,8 @@ export async function POST(request: NextRequest) { ? buildScaleAdjustProductLockBackgroundPrompt( regions, scalePercent, - { width: sourceWidth, height: sourceHeight } + { width: sourceWidth, height: sourceHeight }, + { maskEdit: useScaleAdjustProductLockMaskEdit } ) : buildScaleAdjustFullFramePatchPrompt( regions, @@ -5286,7 +5351,13 @@ export async function POST(request: NextRequest) { ? scaleAdjustPatchReference ? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustProductLockComposite - ? [ + ? useScaleAdjustProductLockMaskEdit + ? [ + sourceImageUrl, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] + : [ sourceImageUrl, scaleAdjustPatchReference.backgroundPlateFullUrl as string, ...scaleAdjustCropReferenceImageUrls, @@ -5389,7 +5460,9 @@ export async function POST(request: NextRequest) { annotatedReferenceType: isScaleAdjustOperation ? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustProductLockComposite - ? "scale-adjust-product-lock-background-plate" + ? useScaleAdjustProductLockMaskEdit + ? "scale-adjust-product-lock-mask-edit" + : "scale-adjust-product-lock-background-plate" : "scale-adjust-full-frame-structure-target-draft" : useProviderTextOnlyCoordinatePrompt ? "scale-adjust-text-coordinate-only-debug-map" @@ -5437,7 +5510,9 @@ export async function POST(request: NextRequest) { isScaleAdjustFalNanoBananaEdit && hasScaleAdjustShrink, scaleAdjustPatchReferenceOrder: scaleAdjustPatchReference && useScaleAdjustFullFrameStructurePatch - ? "full-source-full-target-draft" + ? useScaleAdjustProductLockMaskEdit + ? "full-source-product-lock-edit-mask" + : "full-source-full-target-draft" : scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary ? includeScaleAdjustPatchSourceReference ? "target-draft-source-identity" @@ -5457,12 +5532,19 @@ export async function POST(request: NextRequest) { useScaleAdjustProductLockComposite, scaleAdjustProductLockMethod: scaleAdjustPatchReference?.productLockMethod || null, - scaleAdjustProductLockOverlayUrl: - scaleAdjustPatchReference?.productLockOverlayUrl || null, - scaleAdjustProductLockProtectMaskUrl: - scaleAdjustPatchReference?.productLockProtectMaskUrl || null, - scaleAdjustPatchCompositeClipMaskUrl: - scaleAdjustPatchReference?.compositeClipMaskUrl || null, + scaleAdjustProductLockMaskEditApplied: + useScaleAdjustProductLockMaskEdit, + scaleAdjustProductLockEditMaskUrl: + scaleAdjustPatchReference?.productLockEditMaskUrl || null, + scaleAdjustProductLockOverlayUrl: + scaleAdjustPatchReference?.productLockOverlayUrl || null, + scaleAdjustProductLockProtectMaskUrl: + scaleAdjustPatchReference?.productLockProtectMaskUrl || null, + scaleAdjustPatchCompositeClipMaskUrl: + scaleAdjustPatchReference?.compositeClipMaskUrl || null, + scaleAdjustProviderMaskImageUrl: useScaleAdjustProductLockMaskEdit + ? scaleAdjustPatchReference?.productLockEditMaskUrl || null + : scaleAdjustEditMaskUpload?.url || null, scaleAdjustPatchWorkspacePixelRect: scaleAdjustPatchReference?.workspacePixelRect || null, scaleAdjustPatchWorkspaceRect: @@ -5516,7 +5598,9 @@ export async function POST(request: NextRequest) { scaleAdjustPromptStrategy: isScaleAdjustOperation ? scaleAdjustPatchReference ? useScaleAdjustProductLockComposite - ? "product-lock-background-plate-rmbg-v2-source-patch-one-shot" + ? useScaleAdjustProductLockMaskEdit + ? "product-lock-mask-edit-rmbg-source-patch-one-shot" + : "product-lock-background-plate-rmbg-v2-source-patch-one-shot" : useScaleAdjustFullFrameStructurePatch ? "full-frame-source-locked-target-guide-v8-one-shot" : scaleAdjustPatchReference.structureLockedDraft @@ -5575,9 +5659,14 @@ export async function POST(request: NextRequest) { : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, - providerImageParams: scaleAdjustEditMaskUpload - ? { maskImageUrl: scaleAdjustEditMaskUpload.url } - : undefined, + providerImageParams: useScaleAdjustProductLockMaskEdit + ? { + maskImageUrl: + scaleAdjustPatchReference?.productLockEditMaskUrl as string, + } + : scaleAdjustEditMaskUpload + ? { maskImageUrl: scaleAdjustEditMaskUpload.url } + : undefined, referenceImages, scaleAdjustPatchComposite: useScaleAdjustProductLockComposite ? {