diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 1aeaa2e21..1901eeb9e 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -126,6 +126,7 @@ type ScaleAdjustCropReference = { type ScaleAdjustPatchReference = { backgroundPlateFullUrl?: string | null; compositeClipMaskUrl?: string | null; + productLockBackgroundRepairMaskUrl?: string | null; productLockMethod?: string | null; productLockEditMaskUrl?: string | null; productLockFalEditMaskUrl?: string | null; @@ -2294,6 +2295,176 @@ async function buildScaleAdjustOpaqueEditMaskFromOpacityMask(params: { .toBuffer(); } +async function buildScaleAdjustProductLockBackgroundRepairMaskBuffer(params: { + productSourceMaskBuffer: Buffer; + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; + sourceHeight: number; + sourceWidth: number; +}) { + const { + productSourceMaskBuffer, + regions, + scalePercent, + sourceHeight, + sourceWidth, + } = params; + const productAlpha = await sharp(productSourceMaskBuffer, { failOn: "none" }) + .rotate() + .resize(sourceWidth, sourceHeight, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .raw() + .toBuffer(); + const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight)); + const blur = Math.max(8, Math.min(34, Math.round(minSide * 0.018))); + const repairRects: string[] = []; + const protectRects: string[] = []; + + for (const region of regions) { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const unionBox = rectToPixels( + getRectUnion(region.rect, targetRect), + sourceWidth, + sourceHeight + ); + const sourceBox = region.pixelRect; + const protectedPersonSide = getScaleAdjustProtectedPersonSide(region.prompt); + const leftPad = Math.max(10, Math.round(sourceBox.width * 0.09)); + const rightPad = + protectedPersonSide === "right" + ? Math.max(6, Math.round(sourceBox.width * 0.025)) + : Math.max(10, Math.round(sourceBox.width * 0.09)); + const topPad = Math.max(10, Math.round(sourceBox.height * 0.055)); + const bottomPad = Math.max(10, Math.round(sourceBox.height * 0.035)); + const repairLeft = clamp( + Math.min(unionBox.left, sourceBox.left) - leftPad, + 0, + Math.max(0, sourceWidth - 1) + ); + const repairTop = clamp( + Math.min(unionBox.top, sourceBox.top) - topPad, + 0, + Math.max(0, sourceHeight - 1) + ); + const repairRight = clamp( + Math.max(unionBox.left + unionBox.width, sourceBox.left + sourceBox.width) + + rightPad, + repairLeft + 1, + sourceWidth + ); + const repairBottom = clamp( + Math.max( + unionBox.top + unionBox.height, + sourceBox.top + sourceBox.height + ) + bottomPad, + repairTop + 1, + sourceHeight + ); + repairRects.push( + `` + ); + + const workspaceBox = rectToPixels( + getScaleAdjustPatchWorkspaceRect([region], scalePercent), + sourceWidth, + sourceHeight + ); + const panelBottom = clamp( + sourceBox.top + Math.round(sourceBox.height * 0.38), + repairTop + 1, + sourceHeight + ); + repairRects.push( + `` + ); + + if (protectedPersonSide === "right") { + const protectLeft = clamp( + sourceBox.left + Math.round(sourceBox.width * 0.73), + 0, + Math.max(0, sourceWidth - 1) + ); + const protectTop = clamp( + sourceBox.top + Math.round(sourceBox.height * 0.26), + 0, + Math.max(0, sourceHeight - 1) + ); + protectRects.push( + `` + ); + } else if (protectedPersonSide === "left") { + const protectRight = clamp( + sourceBox.left + Math.round(sourceBox.width * 0.27), + 1, + sourceWidth + ); + const protectTop = clamp( + sourceBox.top + Math.round(sourceBox.height * 0.26), + 0, + Math.max(0, sourceHeight - 1) + ); + protectRects.push( + `` + ); + } + } + + const repairSvg = Buffer.from( + ` + + + + + + + + + + ${repairRects.join("\n")} + ${protectRects.join("\n")} + ` + ); + const repairAlpha = await sharp(repairSvg, { failOn: "none" }) + .resize(sourceWidth, sourceHeight, { fit: "fill" }) + .greyscale() + .raw() + .toBuffer(); + const rgba = Buffer.alloc(sourceWidth * sourceHeight * 4); + for (let index = 0; index < sourceWidth * sourceHeight; index += 1) { + const offset = index * 4; + const value = Math.max(repairAlpha[index] || 0, productAlpha[index] || 0); + rgba[offset] = 255; + rgba[offset + 1] = 255; + rgba[offset + 2] = 255; + rgba[offset + 3] = value; + } + return sharp(rgba, { + raw: { channels: 4, height: sourceHeight, width: sourceWidth }, + }) + .png({ compressionLevel: 9 }) + .toBuffer(); +} + function getScaleAdjustAlphaBoundingBox(params: { height: number; rawRgba: Buffer; @@ -3749,19 +3920,29 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { width: sourceWidth, }) : null; - const productLockEditMaskBuffer = + const productLockBackgroundRepairMaskBuffer = productLockComposites.length && compositeClipMaskBuffer + ? await buildScaleAdjustProductLockBackgroundRepairMaskBuffer({ + productSourceMaskBuffer: compositeClipMaskBuffer, + regions, + scalePercent, + sourceHeight, + sourceWidth, + }) + : null; + const productLockEditMaskBuffer = + productLockComposites.length && productLockBackgroundRepairMaskBuffer ? await buildScaleAdjustTransparentEditMaskFromOpacityMask({ height: sourceHeight, - maskBuffer: compositeClipMaskBuffer, + maskBuffer: productLockBackgroundRepairMaskBuffer, width: sourceWidth, }) : null; const productLockFalEditMaskBuffer = - productLockComposites.length && compositeClipMaskBuffer + productLockComposites.length && productLockBackgroundRepairMaskBuffer ? await buildScaleAdjustOpaqueEditMaskFromOpacityMask({ height: sourceHeight, - maskBuffer: compositeClipMaskBuffer, + maskBuffer: productLockBackgroundRepairMaskBuffer, width: sourceWidth, }) : null; @@ -3770,6 +3951,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { backgroundPlateBuffer, buffer, compositeClipMaskBuffer, + productLockBackgroundRepairMaskBuffer, productLockEditMaskBuffer, productLockFalEditMaskBuffer, productLockOverlayBuffer, @@ -3824,6 +4006,7 @@ async function createScaleAdjustPatchReference(params: { productLockOverlayUpload, productLockProtectMaskUpload, compositeClipMaskUpload, + productLockBackgroundRepairMaskUpload, productLockEditMaskUpload, productLockFalEditMaskUpload, ] = await Promise.all([ @@ -3864,6 +4047,12 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) ) : Promise.resolve(null), + targetDraftReference.productLockBackgroundRepairMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-background-repair-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockBackgroundRepairMaskBuffer) + ) + : Promise.resolve(null), targetDraftReference.productLockEditMaskBuffer ? uploadFile( `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, @@ -3881,6 +4070,8 @@ async function createScaleAdjustPatchReference(params: { return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockBackgroundRepairMaskUrl: + productLockBackgroundRepairMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, productLockEditMaskUrl: productLockEditMaskUpload?.url || null, productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, @@ -3914,6 +4105,7 @@ async function createScaleAdjustPatchReference(params: { productLockOverlayUpload, productLockProtectMaskUpload, compositeClipMaskUpload, + productLockBackgroundRepairMaskUpload, productLockEditMaskUpload, productLockFalEditMaskUpload, ] = await Promise.all([ @@ -3953,6 +4145,12 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) ) : Promise.resolve(null), + targetDraftReference.productLockBackgroundRepairMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-background-repair-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockBackgroundRepairMaskBuffer) + ) + : Promise.resolve(null), targetDraftReference.productLockEditMaskBuffer ? uploadFile( `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, @@ -3970,6 +4168,8 @@ async function createScaleAdjustPatchReference(params: { return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockBackgroundRepairMaskUrl: + productLockBackgroundRepairMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, productLockEditMaskUrl: productLockEditMaskUpload?.url || null, productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, @@ -4972,29 +5172,32 @@ function buildScaleAdjustProductLockBackgroundPrompt( const targetPixelRect = region.targetPixelRect || rectToPixels(targetRect, sourceSize.width, sourceSize.height); - return `Region ${region.index}: remove the selected furniture/product from the original area x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; keep the final target footprint x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height} visually clean and empty because the locked product pixels will be composited there after generation. User request: ${region.prompt}.`; + return `Region ${region.index}: the selected furniture/product originally occupies x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}. The final locked product will later occupy x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}. In this edit call, remove the old product and any old rectangular haze/ghost/plate inside the white mask; synthesize only the room background. User request: ${region.prompt}.`; }) .join("\n"); return [ - "Full-frame background repair for product-locked scale adjustment.", - "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.", + "Full-frame masked background repair for product-locked scale adjustment.", + "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, resize, or redesign the whole photo.", options.maskEdit ? options.draftInput ? "Input image 1 is a full-photo rough background plate from the original photo. The old selected furniture/product footprint has been roughly wiped, and the target product is intentionally absent. The supplied edit mask marks only that rough old footprint area; repair it into natural background and do not edit outside the masked area." - : "Input image 1 is the original clean full photo. The supplied edit mask marks the old selected furniture/product footprint that must be repaired as natural background; do not edit outside the masked area." + : "Input image 1 is the current full photo. The supplied edit mask is the only allowed edit area: white or transparent pixels must be repaired; black/opaque pixels outside the mask must be copied exactly from input image 1." : "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", - "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the final image. The system will composite the locked original product pixels into the final target footprint after your background image is returned.", + "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the returned image. The returned image must be a background plate only; the system will composite the locked original product pixels into the final target footprint after your background image is returned.", options.draftInput ? "Your job is only to turn the rough wiped footprint and any visible draft haze, gray wash, ghost product silhouette, or edge patch into natural wall, curtain, floor, rug, shadow, sofa edge, and background. Keep the target footprint empty and clean for the locked product layer; do not invent a product, second product, or old-size remnant." - : "Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.", - "Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.", - "Keep all non-target content stable: sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", + : "Your job is only to erase the old selected furniture/product, erase any translucent old-size silhouette, erase any rectangular haze/panel/plate left by previous edits, and complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally. Leave no placeholder where the product used to be.", options.maskEdit - ? "The edit mask is only an invisible repair boundary. Do not output pale wash blocks, rough mask shapes, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, or old-size residues." + ? "The mask may be wider than the object silhouette because it also covers old edit artifacts. Do not interpret the mask shape as a box or object to render. Fill the entire editable white/transparent area with continuous real room background." + : "Use the background plate reference only as a repair guide, not as visible final content.", + "Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.", + "Keep all non-target content stable outside the editable mask: sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", + options.maskEdit + ? "The edit mask is only an invisible repair permission map. Do not output pale wash blocks, rough mask shapes, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, old-size residues, or a gray copy of the removed product." : "Remove draft artifacts from input image 2: pale wash blocks, rough masks, missing patches, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, and old-size residues. They are only edit guides, not final content.", "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, crop borders, or any annotation artifacts.", - "中文硬约束：这是产品结构锁定流程。你只生成背景底图，不要生成、移动、缩放或重绘目标家具/商品；目标家具/商品会在你输出后由系统用原图锁定像素叠回目标位置。必须擦除旧位置的目标家具/商品并自然补全墙面、窗帘、地面、阴影和背景。非目标人物必须保持原样，不能变脸、变形、改变衣服、手、腿、脚或姿势。最终背景图不能有矩形雾块、贴片边、黑洞、残影、标注框，也不能出现第二个目标家具/商品。", + "中文硬约束：这是产品结构锁定流程。你只生成背景底图，不要生成、移动、缩放或重绘目标家具/商品；目标家具/商品会在你输出后由系统用原图锁定像素叠回目标位置。白色/透明 mask 内必须清空旧产品、旧尺寸残影、灰色雾块、矩形贴片和旧标注痕迹，只补成连续自然的墙面、窗帘、地面、阴影和背景。黑色/不透明 mask 外必须保持原图不变。非目标人物必须保持原样，不能变脸、变形、改变衣服、手、腿、脚或姿势。最终背景图不能有矩形雾块、贴片边、黑洞、残影、标注框，也不能出现第二个目标家具/商品。", "", "Region instructions:", regionLines, @@ -5635,6 +5838,8 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference?.productLockEditMaskUrl || null, scaleAdjustProductLockFalEditMaskUrl: scaleAdjustPatchReference?.productLockFalEditMaskUrl || null, + scaleAdjustProductLockBackgroundRepairMaskUrl: + scaleAdjustPatchReference?.productLockBackgroundRepairMaskUrl || null, scaleAdjustProductLockOverlayUrl: scaleAdjustPatchReference?.productLockOverlayUrl || null, scaleAdjustProductLockProtectMaskUrl: