diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index fa9893a85..ae2224bdf 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -126,6 +126,7 @@ type ScaleAdjustCropReference = { type ScaleAdjustPatchReference = { backgroundPlateFullUrl?: string | null; compositeClipMaskUrl?: string | null; + repairBlendMaskUrl?: string | null; productLockMethod?: string | null; productLockEditMaskUrl?: string | null; productLockFalEditMaskUrl?: string | null; @@ -509,14 +510,6 @@ function resolveScaleAdjustPreferredModelId(params: { params.requestedModelId || params.configuredModelId ) ); - if (params.preferMaskCapableGpt) { - const maskCapableGptImage2 = gptImage2Candidates - .filter(isScaleAdjustMaskCapableModel) - .filter((entry) => entry.model.healthStatus !== "auto_disabled"); - if (maskCapableGptImage2[0]) { - return maskCapableGptImage2[0].model.id; - } - } for (const providerType of params.preferProviderTypes || []) { const providerCandidate = gptImage2Candidates.find( (entry) => @@ -526,6 +519,14 @@ function resolveScaleAdjustPreferredModelId(params: { return providerCandidate.model.id; } } + if (params.preferMaskCapableGpt) { + const maskCapableGptImage2 = gptImage2Candidates + .filter(isScaleAdjustMaskCapableModel) + .filter((entry) => entry.model.healthStatus !== "auto_disabled"); + if (maskCapableGptImage2[0]) { + return maskCapableGptImage2[0].model.id; + } + } if (params.preferFalNanoBananaEdit) { const falEditCandidate = params.models .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) @@ -2278,6 +2279,81 @@ async function buildScaleAdjustCompositeClipMaskBuffer(params: { .toBuffer(); } +async function buildScaleAdjustProductLockRepairBlendMask(params: { + height: number; + productLockOverlayBuffer: Buffer | null; + productLockProtectMaskBuffer: Buffer | null; + sourceRepairMaskBuffer: Buffer | null; + width: number; +}) { + const { + height, + productLockOverlayBuffer, + productLockProtectMaskBuffer, + sourceRepairMaskBuffer, + width, + } = params; + if (!sourceRepairMaskBuffer) return null; + + const repairAlpha = await sharp(sourceRepairMaskBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .raw() + .toBuffer(); + const overlayAlpha = productLockOverlayBuffer + ? await sharp(productLockOverlayBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .threshold(8) + .blur(Math.max(6, Math.round(Math.min(width, height) * 0.012))) + .greyscale() + .raw() + .toBuffer() + : null; + const protectAlpha = productLockProtectMaskBuffer + ? await sharp(productLockProtectMaskBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .threshold(8) + .blur(Math.max(4, Math.round(Math.min(width, height) * 0.008))) + .greyscale() + .raw() + .toBuffer() + : null; + + const rgba = Buffer.alloc(width * height * 4); + let markedPixels = 0; + for (let index = 0; index < width * height; index += 1) { + const offset = index * 4; + const protectedValue = Math.max( + overlayAlpha?.[index] || 0, + protectAlpha?.[index] || 0 + ); + const alpha = clamp((repairAlpha[index] || 0) - protectedValue, 0, 255); + if (alpha > 8) markedPixels += 1; + rgba[offset] = 255; + rgba[offset + 1] = 255; + rgba[offset + 2] = 255; + rgba[offset + 3] = alpha; + } + if (markedPixels === 0) return sourceRepairMaskBuffer; + + return sharp(rgba, { + raw: { channels: 4, height, width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + async function buildScaleAdjustTransparentEditMaskFromOpacityMask(params: { height: number; maskBuffer: Buffer; @@ -3805,6 +3881,15 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { width: sourceWidth, }) : null; + const repairBlendMaskBuffer = productLockComposites.length + ? await buildScaleAdjustProductLockRepairBlendMask({ + height: sourceHeight, + productLockOverlayBuffer, + productLockProtectMaskBuffer, + sourceRepairMaskBuffer: compositeClipMaskBuffer, + width: sourceWidth, + }) + : compositeClipMaskBuffer; const productLockEditMaskBuffer = productLockComposites.length && compositeClipMaskBuffer ? await buildScaleAdjustTransparentEditMaskFromOpacityMask({ @@ -3826,6 +3911,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { backgroundPlateBuffer, buffer, compositeClipMaskBuffer, + repairBlendMaskBuffer, productLockEditMaskBuffer, productLockFalEditMaskBuffer, productLockOverlayBuffer, @@ -3912,7 +3998,12 @@ async function createScaleAdjustPatchReference(params: { targetDraftReference.productLockFalEditMaskBuffer ), ]); - const [backgroundPlateFullUpload, productLockProtectMaskUpload, compositeClipMaskUpload] = + const [ + backgroundPlateFullUpload, + productLockProtectMaskUpload, + compositeClipMaskUpload, + repairBlendMaskUpload, + ] = await Promise.all([ uploadOptionalScaleAdjustAsset( `canvas-scale-adjust-background-plate-${suffix}.jpg`, @@ -3926,11 +4017,16 @@ async function createScaleAdjustPatchReference(params: { `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, targetDraftReference.compositeClipMaskBuffer ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-repair-blend-mask-${suffix}.png`, + targetDraftReference.repairBlendMaskBuffer + ), ]); return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + repairBlendMaskUrl: repairBlendMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, productLockEditMaskUrl: productLockEditMaskUpload?.url || null, productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, @@ -3975,6 +4071,7 @@ async function createScaleAdjustPatchReference(params: { productLockOverlayUpload, productLockProtectMaskUpload, compositeClipMaskUpload, + repairBlendMaskUpload, productLockEditMaskUpload, productLockFalEditMaskUpload, ] = await Promise.all([ @@ -3994,6 +4091,10 @@ async function createScaleAdjustPatchReference(params: { `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, targetDraftReference.compositeClipMaskBuffer ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-repair-blend-mask-${suffix}.png`, + targetDraftReference.repairBlendMaskBuffer + ), uploadOptionalScaleAdjustAsset( `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, targetDraftReference.productLockEditMaskBuffer @@ -4013,6 +4114,7 @@ async function createScaleAdjustPatchReference(params: { return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + repairBlendMaskUrl: repairBlendMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, productLockEditMaskUrl: productLockEditMaskUpload?.url || null, productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, @@ -5178,7 +5280,9 @@ export async function POST(request: NextRequest) { const preferScaleAdjustFalNanoBananaEdit = hasScaleAdjustShrink && !hasScaleAdjustEnlarge; const preferScaleAdjustProviderTypes = - hasScaleAdjustShrink && !hasScaleAdjustEnlarge ? ["grsai-gpt-image"] : []; + hasScaleAdjustShrink && !hasScaleAdjustEnlarge + ? ["snapgen-image", "grsai-gpt-image"] + : []; const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); const resolvedAspectRatio = resolveLocalEditAspectRatio( body.aspectRatio, @@ -5687,6 +5791,8 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference?.productLockProtectMaskUrl || null, scaleAdjustPatchCompositeClipMaskUrl: scaleAdjustPatchReference?.compositeClipMaskUrl || null, + scaleAdjustRepairBlendMaskUrl: + scaleAdjustPatchReference?.repairBlendMaskUrl || null, scaleAdjustProviderMaskImageUrl: useScaleAdjustProductLockMaskEdit ? scaleAdjustPatchReference?.productLockEditMaskUrl || null : scaleAdjustEditMaskUpload?.url || null, @@ -5867,7 +5973,9 @@ export async function POST(request: NextRequest) { protectMaskImageUrl: scaleAdjustPatchReference?.productLockProtectMaskUrl || undefined, repairMaskImageUrl: - scaleAdjustPatchReference?.compositeClipMaskUrl || undefined, + scaleAdjustPatchReference?.repairBlendMaskUrl || + scaleAdjustPatchReference?.compositeClipMaskUrl || + undefined, sourceImageUrl, sourceHeight, sourceWidth, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 4e28a8204..3e31bef1d 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -2833,6 +2833,81 @@ async function applyRawAlphaToImage(params: { .toBuffer(); } +async function matchImageColorToSourceNearAlphaMask(params: { + alphaMaskRaw: Buffer; + height: number; + imageBuffer: Buffer; + sourceBuffer: Buffer; + width: number; +}) { + const { alphaMaskRaw, height, imageBuffer, sourceBuffer, width } = params; + if (alphaMaskRaw.length !== width * height) return imageBuffer; + + const [imageRaw, sourceRaw] = await Promise.all([ + sharp(imageBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + sharp(sourceBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + ]); + + const sourceSum = [0, 0, 0]; + const imageSum = [0, 0, 0]; + let count = 0; + for (let index = 0; index < alphaMaskRaw.length; index += 1) { + const alpha = alphaMaskRaw[index] || 0; + if (alpha < 4 || alpha > 110) continue; + const offset = index * 3; + const sourceLuma = + (sourceRaw[offset] || 0) * 0.299 + + (sourceRaw[offset + 1] || 0) * 0.587 + + (sourceRaw[offset + 2] || 0) * 0.114; + const imageLuma = + (imageRaw[offset] || 0) * 0.299 + + (imageRaw[offset + 1] || 0) * 0.587 + + (imageRaw[offset + 2] || 0) * 0.114; + if (sourceLuma < 24 || sourceLuma > 248 || imageLuma < 24 || imageLuma > 248) { + continue; + } + for (let channel = 0; channel < 3; channel += 1) { + sourceSum[channel] += sourceRaw[offset + channel] || 0; + imageSum[channel] += imageRaw[offset + channel] || 0; + } + count += 1; + } + + if (count < 500) return imageBuffer; + const offsets = sourceSum.map((sum, channel) => + Math.max(-32, Math.min(32, sum / count - imageSum[channel] / count)) + ); + const correctedRaw = Buffer.from(imageRaw); + for (let index = 0; index < correctedRaw.length; index += 3) { + correctedRaw[index] = Math.max( + 0, + Math.min(255, Math.round((correctedRaw[index] || 0) + offsets[0])) + ); + correctedRaw[index + 1] = Math.max( + 0, + Math.min(255, Math.round((correctedRaw[index + 1] || 0) + offsets[1])) + ); + correctedRaw[index + 2] = Math.max( + 0, + Math.min(255, Math.round((correctedRaw[index + 2] || 0) + offsets[2])) + ); + } + + return sharp(correctedRaw, { raw: { channels: 3, height, width } }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + async function applyScaleAdjustProductProtectMaskToOverlay(params: { height: number; overlayBuffer: Buffer; @@ -3356,18 +3431,32 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { .png({ compressionLevel: 6 }) .toBuffer() : normalizedGeneratedBuffer; - const generatedRepairOverlay = + const repairAlphaMaskRaw = sourceBaseBuffer && repairMaskBuffer - ? await applyRawAlphaToImage({ - alphaMaskRaw: await sharp(repairMaskBuffer, { failOn: "none" }) - .resize(generatedWidth, generatedHeight, { fit: "fill" }) - .ensureAlpha() - .extractChannel(3) - .blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006))) - .raw() - .toBuffer(), + ? await sharp(repairMaskBuffer, { failOn: "none" }) + .resize(generatedWidth, generatedHeight, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006))) + .raw() + .toBuffer() + : null; + const colorMatchedGeneratedBuffer = + sourceBaseBuffer && repairAlphaMaskRaw + ? await matchImageColorToSourceNearAlphaMask({ + alphaMaskRaw: repairAlphaMaskRaw, height: generatedHeight, imageBuffer: normalizedGeneratedBuffer, + sourceBuffer: baseForProductLock, + width: generatedWidth, + }) + : normalizedGeneratedBuffer; + const generatedRepairOverlay = + repairAlphaMaskRaw + ? await applyRawAlphaToImage({ + alphaMaskRaw: repairAlphaMaskRaw, + height: generatedHeight, + imageBuffer: colorMatchedGeneratedBuffer, width: generatedWidth, }) : null;