fix: clip scale adjust patch composites
This commit is contained in:
134
parent
40fa7b1e26
commit
9ee2d2b50b
@ -124,6 +124,7 @@ type ScaleAdjustCropReference = {
|
||||
url: string;
|
||||
};
|
||||
type ScaleAdjustPatchReference = {
|
||||
compositeClipMaskUrl?: string | null;
|
||||
sourceCropUrl: string;
|
||||
structureLockedDraft?: boolean;
|
||||
targetDraftUrl: string;
|
||||
@ -136,6 +137,7 @@ type ScaleAdjustPatchReference = {
|
||||
};
|
||||
};
|
||||
type ScaleAdjustStructureLockedComposite = {
|
||||
compositeClip?: { input: Buffer; left: number; top: number };
|
||||
method?: "rmbg-cutout" | "pixel-mask";
|
||||
overlay: { input: Buffer; left: number; top: number };
|
||||
wash: { input: Buffer; left: number; top: number };
|
||||
@ -1865,6 +1867,124 @@ async function buildScaleAdjustMaskWashPatch(params: {
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function buildScaleAdjustCompositeClipMaskPatch(params: {
|
||||
box: { width: number; height: number };
|
||||
maskBuffer: Buffer | Uint8Array;
|
||||
}) {
|
||||
const { box, maskBuffer } = params;
|
||||
const alpha = await sharp(Buffer.from(maskBuffer), {
|
||||
raw: { channels: 1, height: box.height, width: box.width },
|
||||
})
|
||||
.blur(Math.max(1.4, Math.min(box.width, box.height) * 0.012))
|
||||
.raw()
|
||||
.toBuffer();
|
||||
const rgba = Buffer.alloc(box.width * box.height * 4);
|
||||
for (let index = 0; index < box.width * box.height; index += 1) {
|
||||
const offset = index * 4;
|
||||
rgba[offset] = 255;
|
||||
rgba[offset + 1] = 255;
|
||||
rgba[offset + 2] = 255;
|
||||
rgba[offset + 3] = alpha[index] || 0;
|
||||
}
|
||||
return sharp(rgba, {
|
||||
raw: { channels: 4, height: box.height, width: box.width },
|
||||
})
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
function buildScaleAdjustCompositeClipTargetSvg(params: {
|
||||
height: number;
|
||||
placements: Array<ReturnType<typeof getScaleAdjustVisualGuidePlacement>>;
|
||||
width: number;
|
||||
}) {
|
||||
const { height, placements, width } = params;
|
||||
const minSide = Math.max(1, Math.min(width, height));
|
||||
const blur = Math.max(4, Math.min(24, Math.round(minSide * 0.014)));
|
||||
const pad = Math.max(3, Math.round(blur * 0.7));
|
||||
const rects = placements
|
||||
.map((placement) => {
|
||||
const left = clamp(
|
||||
Math.min(placement.placedLeft, placement.targetBox.left) - pad,
|
||||
0,
|
||||
Math.max(0, width - 1)
|
||||
);
|
||||
const top = clamp(
|
||||
Math.min(placement.placedTop, placement.targetBox.top) - pad,
|
||||
0,
|
||||
Math.max(0, height - 1)
|
||||
);
|
||||
const right = clamp(
|
||||
Math.max(
|
||||
placement.placedLeft + placement.placedWidth,
|
||||
placement.targetBox.left + placement.targetBox.width
|
||||
) + pad,
|
||||
left + 1,
|
||||
width
|
||||
);
|
||||
const bottom = clamp(
|
||||
Math.max(
|
||||
placement.placedTop + placement.placedHeight,
|
||||
placement.targetBox.top + placement.targetBox.height
|
||||
) + pad,
|
||||
top + 1,
|
||||
height
|
||||
);
|
||||
const rectWidth = Math.max(1, right - left);
|
||||
const rectHeight = Math.max(1, bottom - top);
|
||||
const radius = Math.max(4, Math.round(blur * 1.2));
|
||||
return `<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${radius}" ry="${radius}" fill="#ffffff" filter="url(#target-soft)" /><rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" rx="${radius}" ry="${radius}" fill="#ffffff" fill-opacity="0.96" />`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
return Buffer.from(
|
||||
`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}">
|
||||
<defs>
|
||||
<filter id="target-soft" x="-45%" y="-45%" width="190%" height="190%">
|
||||
<feGaussianBlur stdDeviation="${blur}" />
|
||||
</filter>
|
||||
</defs>
|
||||
${rects}
|
||||
</svg>`
|
||||
);
|
||||
}
|
||||
|
||||
async function buildScaleAdjustCompositeClipMaskBuffer(params: {
|
||||
height: number;
|
||||
placements: Array<ReturnType<typeof getScaleAdjustVisualGuidePlacement>>;
|
||||
sourceMaskComposites: Array<{ input: Buffer; left: number; top: number }>;
|
||||
width: number;
|
||||
}) {
|
||||
const { height, placements, sourceMaskComposites, width } = params;
|
||||
if (placements.length === 0 && sourceMaskComposites.length === 0) return null;
|
||||
const targetMask =
|
||||
placements.length > 0
|
||||
? [
|
||||
{
|
||||
input: buildScaleAdjustCompositeClipTargetSvg({
|
||||
height,
|
||||
placements,
|
||||
width,
|
||||
}),
|
||||
left: 0,
|
||||
top: 0,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
|
||||
return sharp({
|
||||
create: {
|
||||
background: { alpha: 0, b: 255, g: 255, r: 255 },
|
||||
channels: 4,
|
||||
height,
|
||||
width,
|
||||
},
|
||||
})
|
||||
.composite([...sourceMaskComposites, ...targetMask])
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
function getScaleAdjustAlphaBoundingBox(params: {
|
||||
height: number;
|
||||
rawRgba: Buffer;
|
||||
@ -2241,6 +2361,10 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: {
|
||||
box: placement.sourceBox,
|
||||
maskBuffer: Buffer.from(sourceWashMask),
|
||||
});
|
||||
const compositeClipMaskPatch = await buildScaleAdjustCompositeClipMaskPatch({
|
||||
box: placement.sourceBox,
|
||||
maskBuffer: sourceWashMask,
|
||||
});
|
||||
|
||||
const subjectCutout = await sharp(cleanedCutout, { failOn: "none" })
|
||||
.extract(alphaBbox)
|
||||
@ -2320,6 +2444,11 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: {
|
||||
});
|
||||
|
||||
return {
|
||||
compositeClip: {
|
||||
input: compositeClipMaskPatch,
|
||||
left: placement.sourceBox.left,
|
||||
top: placement.sourceBox.top,
|
||||
},
|
||||
method: "rmbg-cutout",
|
||||
overlay: {
|
||||
input: structureOverlay,
|
||||
@ -2435,6 +2564,10 @@ async function buildScaleAdjustStructureLockedComposite(params: {
|
||||
box: placement.sourceBox,
|
||||
maskBuffer: Buffer.from(sourceWashMask),
|
||||
});
|
||||
const compositeClipMaskPatch = await buildScaleAdjustCompositeClipMaskPatch({
|
||||
box: placement.sourceBox,
|
||||
maskBuffer: sourceWashMask,
|
||||
});
|
||||
const subjectCrop = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.extract(sourceSubjectBox)
|
||||
.removeAlpha()
|
||||
@ -2483,6 +2616,11 @@ async function buildScaleAdjustStructureLockedComposite(params: {
|
||||
.toBuffer();
|
||||
|
||||
return {
|
||||
compositeClip: {
|
||||
input: compositeClipMaskPatch,
|
||||
left: placement.sourceBox.left,
|
||||
top: placement.sourceBox.top,
|
||||
},
|
||||
method: "pixel-mask",
|
||||
overlay: {
|
||||
input: structureOverlay,
|
||||
@ -2627,6 +2765,11 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
|
||||
left: number;
|
||||
top: number;
|
||||
}> = [];
|
||||
const compositeClipComposites: Array<{
|
||||
input: Buffer;
|
||||
left: number;
|
||||
top: number;
|
||||
}> = [];
|
||||
|
||||
for (const region of regions) {
|
||||
const placement = getScaleAdjustVisualGuidePlacement({
|
||||
@ -2669,19 +2812,10 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
|
||||
if (structureLockAppliedRef) {
|
||||
structureLockAppliedRef.value = true;
|
||||
}
|
||||
sourceFootprintBoxes.push({
|
||||
...placement.sourceFootprintBox,
|
||||
background: sourceRaw
|
||||
? estimateScaleAdjustBoxBackground({
|
||||
box: placement.sourceFootprintBox,
|
||||
raw: sourceRaw,
|
||||
width: sourceWidth,
|
||||
})
|
||||
: null,
|
||||
innerOpacity: 0.36,
|
||||
opacity: 0.72,
|
||||
});
|
||||
subjectWashComposites.push(structureLockedComposite.wash);
|
||||
if (structureLockedComposite.compositeClip) {
|
||||
compositeClipComposites.push(structureLockedComposite.compositeClip);
|
||||
}
|
||||
composites.push(structureLockedComposite.overlay);
|
||||
continue;
|
||||
}
|
||||
@ -2776,7 +2910,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
|
||||
]
|
||||
: [];
|
||||
|
||||
return sharp(sourceBuffer, { failOn: "none" })
|
||||
const buffer = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.composite([
|
||||
...washComposite,
|
||||
...subjectWashComposites,
|
||||
@ -2785,6 +2919,16 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
|
||||
])
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const compositeClipMaskBuffer = compositeClipComposites.length
|
||||
? await buildScaleAdjustCompositeClipMaskBuffer({
|
||||
height: sourceHeight,
|
||||
placements,
|
||||
sourceMaskComposites: compositeClipComposites,
|
||||
width: sourceWidth,
|
||||
})
|
||||
: null;
|
||||
|
||||
return { buffer, compositeClipMaskBuffer };
|
||||
}
|
||||
|
||||
async function createScaleAdjustPatchReference(params: {
|
||||
@ -2812,7 +2956,7 @@ async function createScaleAdjustPatchReference(params: {
|
||||
scalePercent
|
||||
);
|
||||
const structureLockAppliedRef = { value: false };
|
||||
const targetDraftBuffer = await buildScaleAdjustVisualLayoutGuideBuffer({
|
||||
const targetDraftReference = await buildScaleAdjustVisualLayoutGuideBuffer({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
@ -2823,6 +2967,7 @@ async function createScaleAdjustPatchReference(params: {
|
||||
structureLockedDraft: shouldUseStructureLockedDraft,
|
||||
userId,
|
||||
});
|
||||
const targetDraftBuffer = targetDraftReference.buffer;
|
||||
const [sourceCropBuffer, targetDraftCropBuffer] = await Promise.all([
|
||||
sharp(sourceBuffer, { failOn: "none" })
|
||||
.extract(workspacePixelRect)
|
||||
@ -2834,7 +2979,7 @@ async function createScaleAdjustPatchReference(params: {
|
||||
.toBuffer(),
|
||||
]);
|
||||
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const [sourceCropUpload, targetDraftUpload] = await Promise.all([
|
||||
const [sourceCropUpload, targetDraftUpload, compositeClipMaskUpload] = await Promise.all([
|
||||
uploadFile(
|
||||
`canvas-scale-adjust-workspace-source-${suffix}.png`,
|
||||
toArrayBuffer(sourceCropBuffer)
|
||||
@ -2843,9 +2988,16 @@ async function createScaleAdjustPatchReference(params: {
|
||||
`canvas-scale-adjust-workspace-target-draft-${suffix}.png`,
|
||||
toArrayBuffer(targetDraftCropBuffer)
|
||||
),
|
||||
targetDraftReference.compositeClipMaskBuffer
|
||||
? uploadFile(
|
||||
`canvas-scale-adjust-composite-clip-mask-${suffix}.png`,
|
||||
toArrayBuffer(targetDraftReference.compositeClipMaskBuffer)
|
||||
)
|
||||
: Promise.resolve(null),
|
||||
]);
|
||||
|
||||
return {
|
||||
compositeClipMaskUrl: compositeClipMaskUpload?.url || null,
|
||||
sourceCropUrl: sourceCropUpload.url,
|
||||
structureLockedDraft: structureLockAppliedRef.value,
|
||||
targetDraftUrl: targetDraftUpload.url,
|
||||
@ -4003,13 +4155,15 @@ export async function POST(request: NextRequest) {
|
||||
const annotatedBuffer = isScaleAdjustOperation
|
||||
? useScaleAdjustCoordinatePrompt
|
||||
? useScaleAdjustVisualLayoutGuide
|
||||
? await buildScaleAdjustVisualLayoutGuideBuffer({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
? (
|
||||
await buildScaleAdjustVisualLayoutGuideBuffer({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
).buffer
|
||||
: useScaleAdjustMarkedPhotoGuide
|
||||
? await buildScaleAdjustMarkedPhotoGuideBuffer({
|
||||
height: sourceHeight,
|
||||
@ -4029,13 +4183,15 @@ export async function POST(request: NextRequest) {
|
||||
)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer()
|
||||
: await buildScaleAdjustVisualLayoutGuideBuffer({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
: (
|
||||
await buildScaleAdjustVisualLayoutGuideBuffer({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
).buffer
|
||||
: await sharp(sourceBuffer, { failOn: "none" })
|
||||
.composite([
|
||||
{
|
||||
@ -4304,6 +4460,8 @@ export async function POST(request: NextRequest) {
|
||||
scaleAdjustPatchReference?.sourceCropUrl || null,
|
||||
scaleAdjustPatchTargetDraftUrl:
|
||||
scaleAdjustPatchReference?.targetDraftUrl || null,
|
||||
scaleAdjustPatchCompositeClipMaskUrl:
|
||||
scaleAdjustPatchReference?.compositeClipMaskUrl || null,
|
||||
scaleAdjustPatchWorkspacePixelRect:
|
||||
scaleAdjustPatchReference?.workspacePixelRect || null,
|
||||
scaleAdjustPatchWorkspaceRect:
|
||||
@ -4357,7 +4515,7 @@ export async function POST(request: NextRequest) {
|
||||
scaleAdjustPromptStrategy: isScaleAdjustOperation
|
||||
? scaleAdjustPatchReference
|
||||
? scaleAdjustPatchReference.structureLockedDraft
|
||||
? "ai-patch-workspace-structure-locked-draft-v85-one-shot"
|
||||
? "ai-patch-workspace-structure-locked-draft-v86-mask-clipped-one-shot"
|
||||
: "ai-patch-workspace-draft-v75-soft-wipe-erase-ghost-one-shot"
|
||||
: useScaleAdjustEditMask
|
||||
? "mask-edit-gpt-image-2-scale-shrink-v74-one-shot"
|
||||
@ -4425,6 +4583,8 @@ export async function POST(request: NextRequest) {
|
||||
maskMode: "diff",
|
||||
regions: scaleAdjustPatchCompositeRegions,
|
||||
sourceImageUrl,
|
||||
clipMaskImageUrl:
|
||||
scaleAdjustPatchReference.compositeClipMaskUrl || undefined,
|
||||
workspaceRegion: scaleAdjustPatchReference.workspaceRect,
|
||||
}
|
||||
: undefined,
|
||||
|
||||
@ -54,6 +54,7 @@ export interface CanvasLocalEditCompositeConfig {
|
||||
}
|
||||
|
||||
export interface CanvasScaleAdjustPatchCompositeConfig {
|
||||
clipMaskImageUrl?: string;
|
||||
sourceImageUrl: string;
|
||||
workspaceRegion: CanvasLocalEditCompositeRegion;
|
||||
regions?: CanvasLocalEditCompositeRegion[];
|
||||
@ -403,6 +404,9 @@ function normalizeScaleAdjustPatchCompositeConfig(
|
||||
: [];
|
||||
|
||||
return {
|
||||
...(/^https?:\/\//i.test(String(value.clipMaskImageUrl || "").trim())
|
||||
? { clipMaskImageUrl: String(value.clipMaskImageUrl || "").trim() }
|
||||
: {}),
|
||||
sourceImageUrl,
|
||||
workspaceRegion,
|
||||
...(regions.length > 0 ? { regions } : {}),
|
||||
@ -2913,6 +2917,21 @@ async function applyScaleAdjustPatchCompositeToImages(params: {
|
||||
.extract(workspaceBox)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const customClipPatchMaskRaw = composite.clipMaskImageUrl
|
||||
? await normalizeOpacityMaskToRaw({
|
||||
height: workspaceBox.height,
|
||||
maskBuffer: await sharp(await fetchImageBufferFromUrl(composite.clipMaskImageUrl), {
|
||||
failOn: "none",
|
||||
})
|
||||
.rotate()
|
||||
.resize(sourceWidth, sourceHeight, { fit: "fill" })
|
||||
.extract(workspaceBox)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer(),
|
||||
width: workspaceBox.width,
|
||||
})
|
||||
: null;
|
||||
const activeClipPatchMaskRaw = customClipPatchMaskRaw || regionClipPatchMaskRaw;
|
||||
const diffRegions = (maskRegionBoxes?.length
|
||||
? maskRegionBoxes
|
||||
: [{ left: 0, top: 0, width: workspaceBox.width, height: workspaceBox.height }]
|
||||
@ -2968,7 +2987,7 @@ async function applyScaleAdjustPatchCompositeToImages(params: {
|
||||
maskBuffer: diffPatchMask,
|
||||
width: workspaceBox.width,
|
||||
}),
|
||||
regionClipPatchMaskRaw ||
|
||||
activeClipPatchMaskRaw ||
|
||||
Buffer.alloc(workspaceBox.width * workspaceBox.height, 255)
|
||||
),
|
||||
typeof composite.diffAlphaCutoff === "number"
|
||||
@ -3025,6 +3044,7 @@ async function applyScaleAdjustPatchCompositeToImages(params: {
|
||||
debugLabel,
|
||||
featherPx,
|
||||
imageCount: mergedImages.length,
|
||||
customClipMaskApplied: Boolean(customClipPatchMaskRaw),
|
||||
diffRegionClipApplied: Boolean(regionClipPatchMask),
|
||||
maskMode: composite.maskMode || "workspace",
|
||||
maskRegionCount: maskRegionBoxes?.length || 0,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user