fix: lock scale adjust product over repaired background

This commit is contained in:
134 2026-07-06 01:08:37 +08:00
parent deee79c5c2
commit 26aa4fb12f
2 changed files with 354 additions and 23 deletions

View File

@ -124,7 +124,10 @@ type ScaleAdjustCropReference = {
url: string; url: string;
}; };
type ScaleAdjustPatchReference = { type ScaleAdjustPatchReference = {
backgroundPlateFullUrl?: string | null;
compositeClipMaskUrl?: string | null; compositeClipMaskUrl?: string | null;
productLockMethod?: string | null;
productLockOverlayUrl?: string | null;
sourceCropUrl: string; sourceCropUrl: string;
structureLockedDraft?: boolean; structureLockedDraft?: boolean;
targetDraftFullUrl?: string | null; targetDraftFullUrl?: string | null;
@ -139,8 +142,9 @@ type ScaleAdjustPatchReference = {
}; };
type ScaleAdjustStructureLockedComposite = { type ScaleAdjustStructureLockedComposite = {
compositeClip?: { input: Buffer; left: number; top: number }; compositeClip?: { input: Buffer; left: number; top: number };
method?: "pixel-mask" | "rect-crop-guide" | "rmbg-cutout"; method?: "pixel-mask" | "rect-crop-guide" | "rect-crop-guide-rmbg-lock" | "rmbg-cutout";
overlay: { input: Buffer; left: number; top: number }; overlay: { input: Buffer; left: number; top: number };
productLock?: { input: Buffer; left: number; top: number };
targetClip?: { input: Buffer; left: number; top: number }; targetClip?: { input: Buffer; left: number; top: number };
wash: { input: Buffer; left: number; top: number }; wash: { input: Buffer; left: number; top: number };
}; };
@ -2312,6 +2316,7 @@ async function resizeScaleAdjustAlphaMask(params: {
async function buildScaleAdjustRmbgStructureLockedComposite(params: { async function buildScaleAdjustRmbgStructureLockedComposite(params: {
placement: ReturnType<typeof getScaleAdjustVisualGuidePlacement>; placement: ReturnType<typeof getScaleAdjustVisualGuidePlacement>;
protectedPersonSide?: ScaleAdjustProtectedPersonSide | null;
raw: Buffer; raw: Buffer;
sourceBuffer: Buffer; sourceBuffer: Buffer;
sourceImageUrl?: string; sourceImageUrl?: string;
@ -2320,6 +2325,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: {
}): Promise<ScaleAdjustStructureLockedComposite | null> { }): Promise<ScaleAdjustStructureLockedComposite | null> {
const { const {
placement, placement,
protectedPersonSide = null,
raw, raw,
sourceBuffer, sourceBuffer,
sourceImageUrl, sourceImageUrl,
@ -2372,8 +2378,24 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: {
rawRgba: cutoutRaw, rawRgba: cutoutRaw,
width: cutoutWidth, width: cutoutWidth,
}); });
const protectedCleanedAlpha =
protectedPersonSide && sameSizeCutout && sourceCropRaw
? suppressScaleAdjustProtectedFurnitureDarkAlpha({
alphaMask: Buffer.from(cleanedAlpha),
bbox: { left: 0, top: 0, width: cutoutWidth, height: cutoutHeight },
clearProtectedOcclusionZones: true,
protectedPersonSide,
rawRgb: sourceCropRaw,
sourceBox: placement.sourceBox,
})
: cleanedAlpha;
if (protectedCleanedAlpha !== cleanedAlpha) {
for (let index = 0; index < protectedCleanedAlpha.length; index += 1) {
cleanedRaw[index * 4 + 3] = protectedCleanedAlpha[index] || 0;
}
}
const highConfidenceCutoutMask = thresholdScaleAdjustAlphaMask({ const highConfidenceCutoutMask = thresholdScaleAdjustAlphaMask({
maskBuffer: cleanedAlpha, maskBuffer: protectedCleanedAlpha,
threshold: 96, threshold: 96,
}); });
const highConfidenceCutoutBbox = getScaleAdjustMaskBoundingBox({ const highConfidenceCutoutBbox = getScaleAdjustMaskBoundingBox({
@ -2562,6 +2584,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: {
cutoutWidth, cutoutWidth,
placedHeight, placedHeight,
placedWidth, placedWidth,
protectedPersonSide,
sameSizeCutout, sameSizeCutout,
alphaBandingStats, alphaBandingStats,
sourceHeight: placement.sourceBox.height, sourceHeight: placement.sourceBox.height,
@ -2580,6 +2603,11 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: {
left: Math.round(targetCenterX - placedWidth / 2), left: Math.round(targetCenterX - placedWidth / 2),
top: Math.round(targetCenterY - placedHeight / 2), top: Math.round(targetCenterY - placedHeight / 2),
}, },
productLock: {
input: structureOverlay,
left: Math.round(targetCenterX - placedWidth / 2),
top: Math.round(targetCenterY - placedHeight / 2),
},
targetClip: { targetClip: {
input: targetClipMaskPatch, input: targetClipMaskPatch,
left: Math.round(targetCenterX - placedWidth / 2), left: Math.round(targetCenterX - placedWidth / 2),
@ -2633,11 +2661,19 @@ function validateScaleAdjustProtectedPersonMask(params: {
function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: {
alphaMask: Buffer; alphaMask: Buffer;
bbox: { left: number; top: number; width: number; height: number }; bbox: { left: number; top: number; width: number; height: number };
clearProtectedOcclusionZones?: boolean;
protectedPersonSide: ScaleAdjustProtectedPersonSide | null; protectedPersonSide: ScaleAdjustProtectedPersonSide | null;
rawRgb: Buffer; rawRgb: Buffer;
sourceBox: { width: number; height: number }; sourceBox: { width: number; height: number };
}) { }) {
const { alphaMask, bbox, protectedPersonSide, rawRgb, sourceBox } = params; const {
alphaMask,
bbox,
clearProtectedOcclusionZones = false,
protectedPersonSide,
rawRgb,
sourceBox,
} = params;
if (!protectedPersonSide) return alphaMask; if (!protectedPersonSide) return alphaMask;
const output = Buffer.from(alphaMask); const output = Buffer.from(alphaMask);
@ -2665,6 +2701,26 @@ function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: {
} }
return sourceY >= sourceBox.height * 0.58; return sourceY >= sourceBox.height * 0.58;
}; };
const isProtectedOcclusionArea = (sourceX: number, sourceY: number) => {
if (!clearProtectedOcclusionZones) return false;
if (protectedPersonSide === "right") {
return (
(sourceX >= sourceBox.width * 0.69 && sourceY >= sourceBox.height * 0.54) ||
(sourceX >= sourceBox.width * 0.58 && sourceY >= sourceBox.height * 0.7) ||
(sourceX >= sourceBox.width * 0.78 && sourceY >= sourceBox.height * 0.4) ||
(sourceX >= sourceBox.width * 0.91 && sourceY >= sourceBox.height * 0.26)
);
}
if (protectedPersonSide === "left") {
return (
(sourceX <= sourceBox.width * 0.31 && sourceY >= sourceBox.height * 0.54) ||
(sourceX <= sourceBox.width * 0.42 && sourceY >= sourceBox.height * 0.7) ||
(sourceX <= sourceBox.width * 0.22 && sourceY >= sourceBox.height * 0.4) ||
(sourceX <= sourceBox.width * 0.09 && sourceY >= sourceBox.height * 0.26)
);
}
return sourceY >= sourceBox.height * 0.68;
};
let clearedPixels = 0; let clearedPixels = 0;
let markedPixels = 0; let markedPixels = 0;
@ -2694,6 +2750,7 @@ function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: {
(saturation < 0.55 || (r < 110 && g < 110 && b < 110)); (saturation < 0.55 || (r < 110 && g < 110 && b < 110));
const lowContrastDark = luma < 132 && saturation < 0.32; const lowContrastDark = luma < 132 && saturation < 0.32;
const shouldClear = const shouldClear =
isProtectedOcclusionArea(sourceX, sourceY) ||
(inProtectedSideArea && likelyHumanPixel) || (inProtectedSideArea && likelyHumanPixel) ||
(farProtectedSideArea && (farProtectedSideArea &&
(likelyHumanPixel || (likelyHumanPixel ||
@ -2743,16 +2800,17 @@ async function buildScaleAdjustStructureLockedComposite(params: {
sourceWidth, sourceWidth,
userId, userId,
} = params; } = params;
// The current RMBG provider can bake horizontal matte stripes into furniture cutouts.
const allowRmbgStructureLock = const allowRmbgStructureLock =
!preferPixelStructure && process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_PRODUCT_LOCK !== "0" &&
(process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_STRUCTURE === "1" || ((preferPixelStructure && Boolean(protectedPersonSide)) ||
process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_STRUCTURE === "1" ||
(includePersonPixels && (includePersonPixels &&
process.env.CANVAS_SCALE_ADJUST_DISABLE_PERSON_RMBG !== "1")); process.env.CANVAS_SCALE_ADJUST_DISABLE_PERSON_RMBG !== "1"));
const rmbgComposite = !allowRmbgStructureLock const rmbgComposite = !allowRmbgStructureLock
? null ? null
: await buildScaleAdjustRmbgStructureLockedComposite({ : await buildScaleAdjustRmbgStructureLockedComposite({
placement, placement,
protectedPersonSide,
raw, raw,
sourceBuffer, sourceBuffer,
sourceImageUrl, sourceImageUrl,
@ -3133,6 +3191,11 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
left: number; left: number;
top: number; top: number;
}> = []; }> = [];
const productLockComposites: Array<{
input: Buffer;
left: number;
top: number;
}> = [];
for (const region of regions) { for (const region of regions) {
const placement = getScaleAdjustVisualGuidePlacement({ const placement = getScaleAdjustVisualGuidePlacement({
@ -3195,6 +3258,9 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
if (structureLockedComposite.targetClip) { if (structureLockedComposite.targetClip) {
targetClipComposites.push(structureLockedComposite.targetClip); targetClipComposites.push(structureLockedComposite.targetClip);
} }
if (structureLockedComposite.productLock) {
productLockComposites.push(structureLockedComposite.productLock);
}
composites.push(structureLockedComposite.overlay); composites.push(structureLockedComposite.overlay);
continue; continue;
} }
@ -3208,6 +3274,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
const shrinkCutoutComposite = shouldUseShrinkCutoutDraft const shrinkCutoutComposite = shouldUseShrinkCutoutDraft
? await buildScaleAdjustRmbgStructureLockedComposite({ ? await buildScaleAdjustRmbgStructureLockedComposite({
placement, placement,
protectedPersonSide,
raw: sourceRaw, raw: sourceRaw,
sourceBuffer, sourceBuffer,
sourceImageUrl, sourceImageUrl,
@ -3224,6 +3291,9 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
if (shrinkCutoutComposite.targetClip) { if (shrinkCutoutComposite.targetClip) {
targetClipComposites.push(shrinkCutoutComposite.targetClip); targetClipComposites.push(shrinkCutoutComposite.targetClip);
} }
if (shrinkCutoutComposite.productLock) {
productLockComposites.push(shrinkCutoutComposite.productLock);
}
composites.push(shrinkCutoutComposite.overlay); composites.push(shrinkCutoutComposite.overlay);
continue; continue;
} }
@ -3304,6 +3374,28 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
]) ])
.png({ compressionLevel: 6 }) .png({ compressionLevel: 6 })
.toBuffer(); .toBuffer();
const backgroundPlateBuffer = productLockComposites.length
? await sharp(sourceBuffer, { failOn: "none" })
.composite([
...washComposite,
...subjectWashComposites,
])
.png({ compressionLevel: 6 })
.toBuffer()
: null;
const productLockOverlayBuffer = productLockComposites.length
? await sharp({
create: {
background: { alpha: 0, b: 0, g: 0, r: 0 },
channels: 4,
height: sourceHeight,
width: sourceWidth,
},
})
.composite(productLockComposites)
.png({ compressionLevel: 6 })
.toBuffer()
: null;
const compositeClipMaskBuffer = const compositeClipMaskBuffer =
compositeClipComposites.length || targetClipComposites.length compositeClipComposites.length || targetClipComposites.length
? await buildScaleAdjustCompositeClipMaskBuffer({ ? await buildScaleAdjustCompositeClipMaskBuffer({
@ -3314,7 +3406,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
}) })
: null; : null;
return { buffer, compositeClipMaskBuffer }; return { backgroundPlateBuffer, buffer, compositeClipMaskBuffer, productLockOverlayBuffer };
} }
async function createScaleAdjustPatchReference(params: { async function createScaleAdjustPatchReference(params: {
@ -3357,11 +3449,28 @@ async function createScaleAdjustPatchReference(params: {
const targetDraftBuffer = targetDraftReference.buffer; const targetDraftBuffer = targetDraftReference.buffer;
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) {
const [targetDraftFullUpload, compositeClipMaskUpload] = await Promise.all([ const [
targetDraftFullUpload,
backgroundPlateFullUpload,
productLockOverlayUpload,
compositeClipMaskUpload,
] = await Promise.all([
uploadFile( uploadFile(
`canvas-scale-adjust-full-target-draft-${suffix}.png`, `canvas-scale-adjust-full-target-draft-${suffix}.png`,
toArrayBuffer(targetDraftBuffer) toArrayBuffer(targetDraftBuffer)
), ),
targetDraftReference.backgroundPlateBuffer
? uploadFile(
`canvas-scale-adjust-background-plate-${suffix}.png`,
toArrayBuffer(targetDraftReference.backgroundPlateBuffer)
)
: Promise.resolve(null),
targetDraftReference.productLockOverlayBuffer
? uploadFile(
`canvas-scale-adjust-product-lock-overlay-${suffix}.png`,
toArrayBuffer(targetDraftReference.productLockOverlayBuffer)
)
: Promise.resolve(null),
targetDraftReference.compositeClipMaskBuffer targetDraftReference.compositeClipMaskBuffer
? uploadFile( ? uploadFile(
`canvas-scale-adjust-composite-clip-mask-${suffix}.png`, `canvas-scale-adjust-composite-clip-mask-${suffix}.png`,
@ -3371,7 +3480,10 @@ async function createScaleAdjustPatchReference(params: {
]); ]);
return { return {
backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null,
compositeClipMaskUrl: compositeClipMaskUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null,
productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null,
productLockOverlayUrl: productLockOverlayUpload?.url || null,
sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url,
structureLockedDraft: structureLockAppliedRef.value, structureLockedDraft: structureLockAppliedRef.value,
targetDraftFullUrl: targetDraftFullUpload.url, targetDraftFullUrl: targetDraftFullUpload.url,
@ -3395,6 +3507,8 @@ async function createScaleAdjustPatchReference(params: {
sourceCropUpload, sourceCropUpload,
targetDraftUpload, targetDraftUpload,
targetDraftFullUpload, targetDraftFullUpload,
backgroundPlateFullUpload,
productLockOverlayUpload,
compositeClipMaskUpload, compositeClipMaskUpload,
] = await Promise.all([ ] = await Promise.all([
uploadFile( uploadFile(
@ -3409,6 +3523,18 @@ async function createScaleAdjustPatchReference(params: {
`canvas-scale-adjust-full-target-draft-${suffix}.png`, `canvas-scale-adjust-full-target-draft-${suffix}.png`,
toArrayBuffer(targetDraftBuffer) toArrayBuffer(targetDraftBuffer)
), ),
targetDraftReference.backgroundPlateBuffer
? uploadFile(
`canvas-scale-adjust-background-plate-${suffix}.png`,
toArrayBuffer(targetDraftReference.backgroundPlateBuffer)
)
: Promise.resolve(null),
targetDraftReference.productLockOverlayBuffer
? uploadFile(
`canvas-scale-adjust-product-lock-overlay-${suffix}.png`,
toArrayBuffer(targetDraftReference.productLockOverlayBuffer)
)
: Promise.resolve(null),
targetDraftReference.compositeClipMaskBuffer targetDraftReference.compositeClipMaskBuffer
? uploadFile( ? uploadFile(
`canvas-scale-adjust-composite-clip-mask-${suffix}.png`, `canvas-scale-adjust-composite-clip-mask-${suffix}.png`,
@ -3418,7 +3544,10 @@ async function createScaleAdjustPatchReference(params: {
]); ]);
return { return {
backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null,
compositeClipMaskUrl: compositeClipMaskUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null,
productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null,
productLockOverlayUrl: productLockOverlayUpload?.url || null,
sourceCropUrl: sourceCropUpload.url, sourceCropUrl: sourceCropUpload.url,
structureLockedDraft: structureLockAppliedRef.value, structureLockedDraft: structureLockAppliedRef.value,
targetDraftFullUrl: targetDraftFullUpload.url, targetDraftFullUrl: targetDraftFullUpload.url,
@ -4404,6 +4533,40 @@ function buildScaleAdjustFullFramePatchPrompt(
].join("\n"); ].join("\n");
} }
function buildScaleAdjustProductLockBackgroundPrompt(
regions: NormalizedLocalEditRegion[],
scalePercent: ScaleAdjustPercent | null | undefined,
sourceSize: { width: number; height: number }
) {
const regionLines = regions
.map((region) => {
const targetRect = getScaleAdjustTargetRect(region, scalePercent);
const targetPixelRect =
region.targetPixelRect ||
rectToPixels(targetRect, sourceSize.width, sourceSize.height);
return `Region ${region.index}: remove the selected furniture/product from the original area x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; keep the final target footprint x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height} visually clean and empty because the locked product pixels will be composited there after generation. User request: ${region.prompt}.`;
})
.join("\n");
return [
"Full-frame background repair for product-locked scale adjustment.",
"Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.",
"Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.",
"Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the final image. The system will composite the locked original product pixels into the final target footprint after your background image is returned.",
"Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.",
"Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.",
"Keep all non-target content stable: sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.",
"Remove draft artifacts from input image 2: pale wash blocks, rough masks, missing patches, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, and old-size residues. They are only edit guides, not final content.",
"Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, crop borders, or any annotation artifacts.",
"中文硬约束:这是产品结构锁定流程。你只生成背景底图,不要生成、移动、缩放或重绘目标家具/商品;目标家具/商品会在你输出后由系统用原图锁定像素叠回目标位置。必须擦除旧位置的目标家具/商品并自然补全墙面、窗帘、地面、阴影和背景。非目标人物必须保持原样,不能变脸、变形、改变衣服、手、腿、脚或姿势。最终背景图不能有矩形雾块、贴片边、黑洞、残影、标注框,也不能出现第二个目标家具/商品。",
"",
"Region instructions:",
regionLines,
"",
"Success check before returning: the selected furniture/product is absent from the generated background plate, the old footprint is naturally repaired, the target footprint is clean, and the full image still looks like one natural photograph.",
].join("\n");
}
export async function POST(request: NextRequest) { export async function POST(request: NextRequest) {
try { try {
const session = await auth(); const session = await auth();
@ -4749,6 +4912,10 @@ export async function POST(request: NextRequest) {
regions.some((region) => regions.some((region) =>
isScaleAdjustFurnitureProductLikePrompt(region.prompt) isScaleAdjustFurnitureProductLikePrompt(region.prompt)
); );
const useScaleAdjustProductLockComposite =
useScaleAdjustFullFrameStructurePatch &&
Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) &&
Boolean(scaleAdjustPatchReference?.productLockOverlayUrl);
const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference);
const includeScaleAdjustPatchSourceReference = const includeScaleAdjustPatchSourceReference =
Boolean(scaleAdjustPatchReference); Boolean(scaleAdjustPatchReference);
@ -4759,11 +4926,17 @@ export async function POST(request: NextRequest) {
const prompt = isScaleAdjustOperation const prompt = isScaleAdjustOperation
? scaleAdjustPatchReference ? scaleAdjustPatchReference
? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustFullFrameStructurePatch
? buildScaleAdjustFullFramePatchPrompt( ? useScaleAdjustProductLockComposite
regions, ? buildScaleAdjustProductLockBackgroundPrompt(
scalePercent, regions,
{ width: sourceWidth, height: sourceHeight } scalePercent,
) { width: sourceWidth, height: sourceHeight }
)
: buildScaleAdjustFullFramePatchPrompt(
regions,
scalePercent,
{ width: sourceWidth, height: sourceHeight }
)
: buildScaleAdjustPatchPrompt( : buildScaleAdjustPatchPrompt(
regions, regions,
scalePercent, scalePercent,
@ -4816,6 +4989,8 @@ export async function POST(request: NextRequest) {
"full room image, boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, ghost copy, old-size subject, unchanged subject, duplicate subject, stretched subject, distorted product, changed product structure"; "full room image, boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, ghost copy, old-size subject, unchanged subject, duplicate subject, stretched subject, distorted product, changed product structure";
const scaleAdjustFullFramePatchNegativePrompt = const scaleAdjustFullFramePatchNegativePrompt =
"boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, ghost copy, old-size subject, unchanged subject, duplicate subject, stretched subject, distorted product, changed product structure, changed person, changed face, deformed person, extra limbs, changed hands, changed feet, whole-image zoom, crop, pan"; "boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, ghost copy, old-size subject, unchanged subject, duplicate subject, stretched subject, distorted product, changed product structure, changed person, changed face, deformed person, extra limbs, changed hands, changed feet, whole-image zoom, crop, pan";
const scaleAdjustProductLockBackgroundNegativePrompt =
"selected furniture, selected product, moved furniture, scaled furniture, recreated product, duplicate furniture, duplicate product, old-size furniture, target object visible, product ghost, product remnant, boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, black hole, changed person, changed face, deformed person, extra limbs, changed hands, changed feet, whole-image zoom, crop, pan";
const manualRegionReferenceImageUrls = regions.flatMap((region) => const manualRegionReferenceImageUrls = regions.flatMap((region) =>
region.referenceImages.map((reference) => reference.url) region.referenceImages.map((reference) => reference.url)
); );
@ -4825,12 +5000,19 @@ export async function POST(request: NextRequest) {
const referenceImages = isScaleAdjustOperation const referenceImages = isScaleAdjustOperation
? scaleAdjustPatchReference ? scaleAdjustPatchReference
? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustFullFrameStructurePatch
? [ ? useScaleAdjustProductLockComposite
sourceImageUrl, ? [
scaleAdjustPatchReference.targetDraftFullUrl as string, sourceImageUrl,
...scaleAdjustCropReferenceImageUrls, scaleAdjustPatchReference.backgroundPlateFullUrl as string,
...manualRegionReferenceImageUrls, ...scaleAdjustCropReferenceImageUrls,
] ...manualRegionReferenceImageUrls,
]
: [
sourceImageUrl,
scaleAdjustPatchReference.targetDraftFullUrl as string,
...scaleAdjustCropReferenceImageUrls,
...manualRegionReferenceImageUrls,
]
: useScaleAdjustPatchTargetDraftPrimary : useScaleAdjustPatchTargetDraftPrimary
? [ ? [
scaleAdjustPatchReference.targetDraftUrl, scaleAdjustPatchReference.targetDraftUrl,
@ -4921,7 +5103,9 @@ export async function POST(request: NextRequest) {
annotatedReferenceSentToModel: sendScaleAdjustAnnotatedReferenceToModel, annotatedReferenceSentToModel: sendScaleAdjustAnnotatedReferenceToModel,
annotatedReferenceType: isScaleAdjustOperation annotatedReferenceType: isScaleAdjustOperation
? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustFullFrameStructurePatch
? "scale-adjust-full-frame-structure-target-draft" ? useScaleAdjustProductLockComposite
? "scale-adjust-product-lock-background-plate"
: "scale-adjust-full-frame-structure-target-draft"
: useProviderTextOnlyCoordinatePrompt : useProviderTextOnlyCoordinatePrompt
? "scale-adjust-text-coordinate-only-debug-map" ? "scale-adjust-text-coordinate-only-debug-map"
: useScaleAdjustCoordinatePrompt : useScaleAdjustCoordinatePrompt
@ -4982,6 +5166,14 @@ export async function POST(request: NextRequest) {
scaleAdjustPatchReference?.targetDraftUrl || null, scaleAdjustPatchReference?.targetDraftUrl || null,
scaleAdjustPatchTargetDraftFullUrl: scaleAdjustPatchTargetDraftFullUrl:
scaleAdjustPatchReference?.targetDraftFullUrl || null, scaleAdjustPatchReference?.targetDraftFullUrl || null,
scaleAdjustBackgroundPlateFullUrl:
scaleAdjustPatchReference?.backgroundPlateFullUrl || null,
scaleAdjustProductLockApplied:
useScaleAdjustProductLockComposite,
scaleAdjustProductLockMethod:
scaleAdjustPatchReference?.productLockMethod || null,
scaleAdjustProductLockOverlayUrl:
scaleAdjustPatchReference?.productLockOverlayUrl || null,
scaleAdjustPatchCompositeClipMaskUrl: scaleAdjustPatchCompositeClipMaskUrl:
scaleAdjustPatchReference?.compositeClipMaskUrl || null, scaleAdjustPatchReference?.compositeClipMaskUrl || null,
scaleAdjustPatchWorkspacePixelRect: scaleAdjustPatchWorkspacePixelRect:
@ -5036,7 +5228,9 @@ export async function POST(request: NextRequest) {
.length, .length,
scaleAdjustPromptStrategy: isScaleAdjustOperation scaleAdjustPromptStrategy: isScaleAdjustOperation
? scaleAdjustPatchReference ? scaleAdjustPatchReference
? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustProductLockComposite
? "product-lock-background-plate-rmbg-v1-one-shot"
: useScaleAdjustFullFrameStructurePatch
? "full-frame-source-locked-target-guide-v8-one-shot" ? "full-frame-source-locked-target-guide-v8-one-shot"
: scaleAdjustPatchReference.structureLockedDraft : scaleAdjustPatchReference.structureLockedDraft
? "ai-patch-workspace-structure-locked-draft-v87-subject-alpha-clipped-one-shot" ? "ai-patch-workspace-structure-locked-draft-v87-subject-alpha-clipped-one-shot"
@ -5079,7 +5273,9 @@ export async function POST(request: NextRequest) {
isScaleAdjustOperation isScaleAdjustOperation
? scaleAdjustPatchReference ? scaleAdjustPatchReference
? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustFullFrameStructurePatch
? scaleAdjustFullFramePatchNegativePrompt ? useScaleAdjustProductLockComposite
? scaleAdjustProductLockBackgroundNegativePrompt
: scaleAdjustFullFramePatchNegativePrompt
: useFalScaleAdjustPatchPrompt : useFalScaleAdjustPatchPrompt
? scaleAdjustFalPatchNegativePrompt ? scaleAdjustFalPatchNegativePrompt
: scaleAdjustPatchNegativePrompt : scaleAdjustPatchNegativePrompt
@ -5119,6 +5315,14 @@ export async function POST(request: NextRequest) {
workspaceRegion: scaleAdjustPatchReference.workspaceRect, workspaceRegion: scaleAdjustPatchReference.workspaceRect,
} }
: undefined, : undefined,
scaleAdjustProductLockComposite: useScaleAdjustProductLockComposite
? {
overlayImageUrl:
scaleAdjustPatchReference?.productLockOverlayUrl as string,
sourceHeight,
sourceWidth,
}
: undefined,
scaleAdjustVerification: scaleAdjustVerificationConfig, scaleAdjustVerification: scaleAdjustVerificationConfig,
localEditComposite: isScaleAdjustOperation localEditComposite: isScaleAdjustOperation
? undefined ? undefined

View File

@ -71,6 +71,12 @@ export interface CanvasScaleAdjustPatchCompositeConfig {
featherRatio?: number; featherRatio?: number;
} }
export interface CanvasScaleAdjustProductLockCompositeConfig {
overlayImageUrl: string;
sourceHeight: number;
sourceWidth: number;
}
export interface CanvasScaleAdjustVerificationConfig { export interface CanvasScaleAdjustVerificationConfig {
allowDeterministicFallback?: boolean; allowDeterministicFallback?: boolean;
allowOutsideChangedWhenChangedInside?: boolean; allowOutsideChangedWhenChangedInside?: boolean;
@ -99,6 +105,7 @@ export interface CanvasGenerateImageRetryVariant {
referenceImage?: string; referenceImage?: string;
referenceImages?: string[]; referenceImages?: string[];
scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig;
scaleAdjustProductLockComposite?: CanvasScaleAdjustProductLockCompositeConfig;
scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
} }
@ -123,6 +130,7 @@ export interface CanvasGenerateImageRequest {
debugMeta?: Record<string, unknown>; debugMeta?: Record<string, unknown>;
localEditComposite?: CanvasLocalEditCompositeConfig; localEditComposite?: CanvasLocalEditCompositeConfig;
scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig;
scaleAdjustProductLockComposite?: CanvasScaleAdjustProductLockCompositeConfig;
scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
retryVariants?: CanvasGenerateImageRetryVariant[]; retryVariants?: CanvasGenerateImageRetryVariant[];
internalRetryVariantIndex?: number; internalRetryVariantIndex?: number;
@ -541,6 +549,23 @@ function normalizeScaleAdjustPatchCompositeConfig(
}; };
} }
function normalizeScaleAdjustProductLockCompositeConfig(
value: CanvasGenerateImageRequest["scaleAdjustProductLockComposite"]
): CanvasScaleAdjustProductLockCompositeConfig | null {
if (!value || typeof value !== "object") return null;
const overlayImageUrl = String(value.overlayImageUrl || "").trim();
if (!/^https?:\/\//i.test(overlayImageUrl)) return null;
const sourceWidth = Math.round(Number(value.sourceWidth || 0));
const sourceHeight = Math.round(Number(value.sourceHeight || 0));
if (sourceWidth <= 0 || sourceHeight <= 0) return null;
return {
overlayImageUrl,
sourceHeight,
sourceWidth,
};
}
function normalizeCanvasRegion(region: unknown): CanvasLocalEditCompositeRegion | null { function normalizeCanvasRegion(region: unknown): CanvasLocalEditCompositeRegion | null {
if (!region || typeof region !== "object") return null; if (!region || typeof region !== "object") return null;
const raw = region as Partial<CanvasLocalEditCompositeRegion>; const raw = region as Partial<CanvasLocalEditCompositeRegion>;
@ -3163,6 +3188,96 @@ async function applyScaleAdjustPatchCompositeToImages(params: {
return mergedImages; return mergedImages;
} }
async function applyScaleAdjustProductLockCompositeToImages(params: {
images: ParsedImage[];
composite: CanvasScaleAdjustProductLockCompositeConfig | null;
debugLabel: string;
}) {
const { images, composite, debugLabel } = params;
if (!composite || images.length === 0) return images;
const overlayBuffer = await sharp(
await fetchImageBufferFromUrl(composite.overlayImageUrl),
{ failOn: "none" }
)
.rotate()
.ensureAlpha()
.png({ compressionLevel: 6 })
.toBuffer();
const overlayMetadata = await sharp(overlayBuffer, { failOn: "none" }).metadata();
const overlayWidth = Math.round(Number(overlayMetadata.width || 0));
const overlayHeight = Math.round(Number(overlayMetadata.height || 0));
if (overlayWidth <= 0 || overlayHeight <= 0) return images;
const mergedImages: ParsedImage[] = [];
for (const [index, image] of images.entries()) {
const generatedUrl = image.originalUrl || image.thumbnailUrl;
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
const normalizedGeneratedBuffer = await sharp(generatedBuffer, {
failOn: "none",
})
.rotate()
.ensureAlpha()
.png({ compressionLevel: 6 })
.toBuffer();
const generatedMetadata = await sharp(normalizedGeneratedBuffer, {
failOn: "none",
}).metadata();
const generatedWidth = Math.round(Number(generatedMetadata.width || 0));
const generatedHeight = Math.round(Number(generatedMetadata.height || 0));
if (generatedWidth <= 0 || generatedHeight <= 0) {
mergedImages.push(image);
continue;
}
const resizedOverlayBuffer = await sharp(overlayBuffer, { failOn: "none" })
.resize(generatedWidth, generatedHeight, {
fit: "fill",
kernel: sharp.kernel.lanczos3,
})
.png({ compressionLevel: 6 })
.toBuffer();
const mergedBuffer = await sharp(normalizedGeneratedBuffer, {
failOn: "none",
})
.composite([{ input: resizedOverlayBuffer, left: 0, top: 0 }])
.png({ compressionLevel: 6 })
.toBuffer();
const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" })
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
.png({ compressionLevel: 6 })
.toBuffer();
const suffix = `${Date.now()}-${index + 1}-${Math.random()
.toString(36)
.slice(2, 8)}`;
const [originalUpload, thumbnailUpload] = await Promise.all([
uploadFile(
`canvas-scale-adjust-product-lock-${suffix}.png`,
bufferToArrayBuffer(mergedBuffer)
),
uploadFile(
`canvas-scale-adjust-product-lock-thumb-${suffix}.png`,
bufferToArrayBuffer(thumbnailBuffer)
),
]);
mergedImages.push({
originalUrl: originalUpload.url,
thumbnailUrl: thumbnailUpload.url,
});
}
console.info("[Canvas Generate Image] Scale adjust product lock composite applied:", {
debugLabel,
imageCount: mergedImages.length,
overlayHeight,
overlayWidth,
sourceHeight: composite.sourceHeight,
sourceWidth: composite.sourceWidth,
});
return mergedImages;
}
function getProviderBatchLimit(providerType: string) { function getProviderBatchLimit(providerType: string) {
switch (providerType) { switch (providerType) {
case "ruxa-image": case "ruxa-image":
@ -3661,6 +3776,10 @@ export async function generateCanvasImage(
const scaleAdjustPatchComposite = normalizeScaleAdjustPatchCompositeConfig( const scaleAdjustPatchComposite = normalizeScaleAdjustPatchCompositeConfig(
request.scaleAdjustPatchComposite request.scaleAdjustPatchComposite
); );
const scaleAdjustProductLockComposite =
normalizeScaleAdjustProductLockCompositeConfig(
request.scaleAdjustProductLockComposite
);
const scaleAdjustVerification = normalizeScaleAdjustVerificationConfig( const scaleAdjustVerification = normalizeScaleAdjustVerificationConfig(
request.scaleAdjustVerification request.scaleAdjustVerification
); );
@ -3887,6 +4006,7 @@ export async function generateCanvasImage(
...debugMeta, ...debugMeta,
hasLocalEditComposite: Boolean(localEditComposite), hasLocalEditComposite: Boolean(localEditComposite),
hasScaleAdjustPatchComposite: Boolean(scaleAdjustPatchComposite), hasScaleAdjustPatchComposite: Boolean(scaleAdjustPatchComposite),
hasScaleAdjustProductLockComposite: Boolean(scaleAdjustProductLockComposite),
hasScaleAdjustVerification: Boolean(scaleAdjustVerification), hasScaleAdjustVerification: Boolean(scaleAdjustVerification),
}); });
@ -4240,9 +4360,16 @@ export async function generateCanvasImage(
images: compositedImages, images: compositedImages,
}); });
const scaleAdjustProductLockedImages =
await applyScaleAdjustProductLockCompositeToImages({
composite: scaleAdjustProductLockComposite,
debugLabel,
images: scaleAdjustPatchCompositedImages,
});
const finalImages = await applyScaleAdjustVerificationToImages({ const finalImages = await applyScaleAdjustVerificationToImages({
debugLabel, debugLabel,
images: scaleAdjustPatchCompositedImages, images: scaleAdjustProductLockedImages,
verification: scaleAdjustVerification, verification: scaleAdjustVerification,
}); });
console.info("[Canvas Generate Image] Completed:", { console.info("[Canvas Generate Image] Completed:", {