From 9ee2d2b50b82241dc75aa914177de9b3c26abad5 Mon Sep 17 00:00:00 2001 From: 134 Date: Sat, 4 Jul 2026 17:13:39 +0800 Subject: [PATCH] fix: clip scale adjust patch composites --- app/api/canvas/local-edit/route.ts | 220 +++++++++++++++++++++++++---- lib/canvas/generate-image.ts | 22 ++- 2 files changed, 211 insertions(+), 31 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 357c49d8b..4338b9a4c 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -124,6 +124,7 @@ type ScaleAdjustCropReference = { url: string; }; type ScaleAdjustPatchReference = { + compositeClipMaskUrl?: string | null; sourceCropUrl: string; structureLockedDraft?: boolean; targetDraftUrl: string; @@ -136,6 +137,7 @@ type ScaleAdjustPatchReference = { }; }; type ScaleAdjustStructureLockedComposite = { + compositeClip?: { input: Buffer; left: number; top: number }; method?: "rmbg-cutout" | "pixel-mask"; overlay: { input: Buffer; left: number; top: number }; wash: { input: Buffer; left: number; top: number }; @@ -1865,6 +1867,124 @@ async function buildScaleAdjustMaskWashPatch(params: { .toBuffer(); } +async function buildScaleAdjustCompositeClipMaskPatch(params: { + box: { width: number; height: number }; + maskBuffer: Buffer | Uint8Array; +}) { + const { box, maskBuffer } = params; + const alpha = await sharp(Buffer.from(maskBuffer), { + raw: { channels: 1, height: box.height, width: box.width }, + }) + .blur(Math.max(1.4, Math.min(box.width, box.height) * 0.012)) + .raw() + .toBuffer(); + const rgba = Buffer.alloc(box.width * box.height * 4); + for (let index = 0; index < box.width * box.height; index += 1) { + const offset = index * 4; + rgba[offset] = 255; + rgba[offset + 1] = 255; + rgba[offset + 2] = 255; + rgba[offset + 3] = alpha[index] || 0; + } + return sharp(rgba, { + raw: { channels: 4, height: box.height, width: box.width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + +function buildScaleAdjustCompositeClipTargetSvg(params: { + height: number; + placements: Array>; + width: number; +}) { + const { height, placements, width } = params; + const minSide = Math.max(1, Math.min(width, height)); + const blur = Math.max(4, Math.min(24, Math.round(minSide * 0.014))); + const pad = Math.max(3, Math.round(blur * 0.7)); + const rects = placements + .map((placement) => { + const left = clamp( + Math.min(placement.placedLeft, placement.targetBox.left) - pad, + 0, + Math.max(0, width - 1) + ); + const top = clamp( + Math.min(placement.placedTop, placement.targetBox.top) - pad, + 0, + Math.max(0, height - 1) + ); + const right = clamp( + Math.max( + placement.placedLeft + placement.placedWidth, + placement.targetBox.left + placement.targetBox.width + ) + pad, + left + 1, + width + ); + const bottom = clamp( + Math.max( + placement.placedTop + placement.placedHeight, + placement.targetBox.top + placement.targetBox.height + ) + pad, + top + 1, + height + ); + const rectWidth = Math.max(1, right - left); + const rectHeight = Math.max(1, bottom - top); + const radius = Math.max(4, Math.round(blur * 1.2)); + return ``; + }) + .join(""); + + return Buffer.from( + ` + + + + + + ${rects} + ` + ); +} + +async function buildScaleAdjustCompositeClipMaskBuffer(params: { + height: number; + placements: Array>; + sourceMaskComposites: Array<{ input: Buffer; left: number; top: number }>; + width: number; +}) { + const { height, placements, sourceMaskComposites, width } = params; + if (placements.length === 0 && sourceMaskComposites.length === 0) return null; + const targetMask = + placements.length > 0 + ? [ + { + input: buildScaleAdjustCompositeClipTargetSvg({ + height, + placements, + width, + }), + left: 0, + top: 0, + }, + ] + : []; + + return sharp({ + create: { + background: { alpha: 0, b: 255, g: 255, r: 255 }, + channels: 4, + height, + width, + }, + }) + .composite([...sourceMaskComposites, ...targetMask]) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + function getScaleAdjustAlphaBoundingBox(params: { height: number; rawRgba: Buffer; @@ -2241,6 +2361,10 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { box: placement.sourceBox, maskBuffer: Buffer.from(sourceWashMask), }); + const compositeClipMaskPatch = await buildScaleAdjustCompositeClipMaskPatch({ + box: placement.sourceBox, + maskBuffer: sourceWashMask, + }); const subjectCutout = await sharp(cleanedCutout, { failOn: "none" }) .extract(alphaBbox) @@ -2320,6 +2444,11 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { }); return { + compositeClip: { + input: compositeClipMaskPatch, + left: placement.sourceBox.left, + top: placement.sourceBox.top, + }, method: "rmbg-cutout", overlay: { input: structureOverlay, @@ -2435,6 +2564,10 @@ async function buildScaleAdjustStructureLockedComposite(params: { box: placement.sourceBox, maskBuffer: Buffer.from(sourceWashMask), }); + const compositeClipMaskPatch = await buildScaleAdjustCompositeClipMaskPatch({ + box: placement.sourceBox, + maskBuffer: sourceWashMask, + }); const subjectCrop = await sharp(sourceBuffer, { failOn: "none" }) .extract(sourceSubjectBox) .removeAlpha() @@ -2483,6 +2616,11 @@ async function buildScaleAdjustStructureLockedComposite(params: { .toBuffer(); return { + compositeClip: { + input: compositeClipMaskPatch, + left: placement.sourceBox.left, + top: placement.sourceBox.top, + }, method: "pixel-mask", overlay: { input: structureOverlay, @@ -2627,6 +2765,11 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { left: number; top: number; }> = []; + const compositeClipComposites: Array<{ + input: Buffer; + left: number; + top: number; + }> = []; for (const region of regions) { const placement = getScaleAdjustVisualGuidePlacement({ @@ -2669,19 +2812,10 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { if (structureLockAppliedRef) { structureLockAppliedRef.value = true; } - sourceFootprintBoxes.push({ - ...placement.sourceFootprintBox, - background: sourceRaw - ? estimateScaleAdjustBoxBackground({ - box: placement.sourceFootprintBox, - raw: sourceRaw, - width: sourceWidth, - }) - : null, - innerOpacity: 0.36, - opacity: 0.72, - }); subjectWashComposites.push(structureLockedComposite.wash); + if (structureLockedComposite.compositeClip) { + compositeClipComposites.push(structureLockedComposite.compositeClip); + } composites.push(structureLockedComposite.overlay); continue; } @@ -2776,7 +2910,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { ] : []; - return sharp(sourceBuffer, { failOn: "none" }) + const buffer = await sharp(sourceBuffer, { failOn: "none" }) .composite([ ...washComposite, ...subjectWashComposites, @@ -2785,6 +2919,16 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { ]) .png({ compressionLevel: 6 }) .toBuffer(); + const compositeClipMaskBuffer = compositeClipComposites.length + ? await buildScaleAdjustCompositeClipMaskBuffer({ + height: sourceHeight, + placements, + sourceMaskComposites: compositeClipComposites, + width: sourceWidth, + }) + : null; + + return { buffer, compositeClipMaskBuffer }; } async function createScaleAdjustPatchReference(params: { @@ -2812,7 +2956,7 @@ async function createScaleAdjustPatchReference(params: { scalePercent ); const structureLockAppliedRef = { value: false }; - const targetDraftBuffer = await buildScaleAdjustVisualLayoutGuideBuffer({ + const targetDraftReference = await buildScaleAdjustVisualLayoutGuideBuffer({ regions, scalePercent, sourceBuffer, @@ -2823,6 +2967,7 @@ async function createScaleAdjustPatchReference(params: { structureLockedDraft: shouldUseStructureLockedDraft, userId, }); + const targetDraftBuffer = targetDraftReference.buffer; const [sourceCropBuffer, targetDraftCropBuffer] = await Promise.all([ sharp(sourceBuffer, { failOn: "none" }) .extract(workspacePixelRect) @@ -2834,7 +2979,7 @@ async function createScaleAdjustPatchReference(params: { .toBuffer(), ]); const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - const [sourceCropUpload, targetDraftUpload] = await Promise.all([ + const [sourceCropUpload, targetDraftUpload, compositeClipMaskUpload] = await Promise.all([ uploadFile( `canvas-scale-adjust-workspace-source-${suffix}.png`, toArrayBuffer(sourceCropBuffer) @@ -2843,9 +2988,16 @@ async function createScaleAdjustPatchReference(params: { `canvas-scale-adjust-workspace-target-draft-${suffix}.png`, toArrayBuffer(targetDraftCropBuffer) ), + targetDraftReference.compositeClipMaskBuffer + ? uploadFile( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) + ) + : Promise.resolve(null), ]); return { + compositeClipMaskUrl: compositeClipMaskUpload?.url || null, sourceCropUrl: sourceCropUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftUrl: targetDraftUpload.url, @@ -4003,13 +4155,15 @@ export async function POST(request: NextRequest) { const annotatedBuffer = isScaleAdjustOperation ? useScaleAdjustCoordinatePrompt ? useScaleAdjustVisualLayoutGuide - ? await buildScaleAdjustVisualLayoutGuideBuffer({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceWidth, - }) + ? ( + await buildScaleAdjustVisualLayoutGuideBuffer({ + regions, + scalePercent, + sourceBuffer, + sourceHeight, + sourceWidth, + }) + ).buffer : useScaleAdjustMarkedPhotoGuide ? await buildScaleAdjustMarkedPhotoGuideBuffer({ height: sourceHeight, @@ -4029,13 +4183,15 @@ export async function POST(request: NextRequest) { ) .png({ compressionLevel: 6 }) .toBuffer() - : await buildScaleAdjustVisualLayoutGuideBuffer({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceWidth, - }) + : ( + await buildScaleAdjustVisualLayoutGuideBuffer({ + regions, + scalePercent, + sourceBuffer, + sourceHeight, + sourceWidth, + }) + ).buffer : await sharp(sourceBuffer, { failOn: "none" }) .composite([ { @@ -4304,6 +4460,8 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference?.sourceCropUrl || null, scaleAdjustPatchTargetDraftUrl: scaleAdjustPatchReference?.targetDraftUrl || null, + scaleAdjustPatchCompositeClipMaskUrl: + scaleAdjustPatchReference?.compositeClipMaskUrl || null, scaleAdjustPatchWorkspacePixelRect: scaleAdjustPatchReference?.workspacePixelRect || null, scaleAdjustPatchWorkspaceRect: @@ -4357,7 +4515,7 @@ export async function POST(request: NextRequest) { scaleAdjustPromptStrategy: isScaleAdjustOperation ? scaleAdjustPatchReference ? scaleAdjustPatchReference.structureLockedDraft - ? "ai-patch-workspace-structure-locked-draft-v85-one-shot" + ? "ai-patch-workspace-structure-locked-draft-v86-mask-clipped-one-shot" : "ai-patch-workspace-draft-v75-soft-wipe-erase-ghost-one-shot" : useScaleAdjustEditMask ? "mask-edit-gpt-image-2-scale-shrink-v74-one-shot" @@ -4425,6 +4583,8 @@ export async function POST(request: NextRequest) { maskMode: "diff", regions: scaleAdjustPatchCompositeRegions, sourceImageUrl, + clipMaskImageUrl: + scaleAdjustPatchReference.compositeClipMaskUrl || undefined, workspaceRegion: scaleAdjustPatchReference.workspaceRect, } : undefined, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index fd71b56aa..67eaff122 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -54,6 +54,7 @@ export interface CanvasLocalEditCompositeConfig { } export interface CanvasScaleAdjustPatchCompositeConfig { + clipMaskImageUrl?: string; sourceImageUrl: string; workspaceRegion: CanvasLocalEditCompositeRegion; regions?: CanvasLocalEditCompositeRegion[]; @@ -403,6 +404,9 @@ function normalizeScaleAdjustPatchCompositeConfig( : []; return { + ...(/^https?:\/\//i.test(String(value.clipMaskImageUrl || "").trim()) + ? { clipMaskImageUrl: String(value.clipMaskImageUrl || "").trim() } + : {}), sourceImageUrl, workspaceRegion, ...(regions.length > 0 ? { regions } : {}), @@ -2913,6 +2917,21 @@ async function applyScaleAdjustPatchCompositeToImages(params: { .extract(workspaceBox) .png({ compressionLevel: 6 }) .toBuffer(); + const customClipPatchMaskRaw = composite.clipMaskImageUrl + ? await normalizeOpacityMaskToRaw({ + height: workspaceBox.height, + maskBuffer: await sharp(await fetchImageBufferFromUrl(composite.clipMaskImageUrl), { + failOn: "none", + }) + .rotate() + .resize(sourceWidth, sourceHeight, { fit: "fill" }) + .extract(workspaceBox) + .png({ compressionLevel: 6 }) + .toBuffer(), + width: workspaceBox.width, + }) + : null; + const activeClipPatchMaskRaw = customClipPatchMaskRaw || regionClipPatchMaskRaw; const diffRegions = (maskRegionBoxes?.length ? maskRegionBoxes : [{ left: 0, top: 0, width: workspaceBox.width, height: workspaceBox.height }] @@ -2968,7 +2987,7 @@ async function applyScaleAdjustPatchCompositeToImages(params: { maskBuffer: diffPatchMask, width: workspaceBox.width, }), - regionClipPatchMaskRaw || + activeClipPatchMaskRaw || Buffer.alloc(workspaceBox.width * workspaceBox.height, 255) ), typeof composite.diffAlphaCutoff === "number" @@ -3025,6 +3044,7 @@ async function applyScaleAdjustPatchCompositeToImages(params: { debugLabel, featherPx, imageCount: mergedImages.length, + customClipMaskApplied: Boolean(customClipPatchMaskRaw), diffRegionClipApplied: Boolean(regionClipPatchMask), maskMode: composite.maskMode || "workspace", maskRegionCount: maskRegionBoxes?.length || 0,