From 2241e0f27e65f1779e5bd520ad475766a0926f7b Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 14:48:51 +0800 Subject: [PATCH 01/73] fix: guide scale adjust with visual target layout --- app/api/canvas/local-edit/route.ts | 186 ++++++++++++++++++++++++----- lib/canvas/generate-image.ts | 9 +- 2 files changed, 155 insertions(+), 40 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 33cbf15e6..448d16e2e 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -100,6 +100,7 @@ const MAX_REFERENCE_IMAGES_PER_REGION = 3; const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; const SCALE_ADJUST_CROP_MAX_EDGE = 1536; +const SCALE_ADJUST_LAYOUT_GUIDE_BACKGROUND = "#f5f7fb"; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; @@ -761,6 +762,123 @@ async function createScaleAdjustCropReferences(params: { return references; } +function getScaleAdjustVisualGuidePlacement(params: { + height: number; + region: NormalizedLocalEditRegion; + scalePercent?: ScaleAdjustPercent | null; + width: number; +}) { + const { height, region, scalePercent, width } = params; + const source = region.pixelRect; + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const target = region.targetPixelRect || rectToPixels(targetRect, width, height); + const sourceLeft = clamp(source.left, 0, Math.max(0, width - 1)); + const sourceTop = clamp(source.top, 0, Math.max(0, height - 1)); + const sourceRight = clamp( + source.left + source.width, + sourceLeft + 1, + width + ); + const sourceBottom = clamp( + source.top + source.height, + sourceTop + 1, + height + ); + const sourceWidth = Math.max(1, Math.round(sourceRight - sourceLeft)); + const sourceHeight = Math.max(1, Math.round(sourceBottom - sourceTop)); + const widthScale = target.width / Math.max(1, sourceWidth); + const heightScale = target.height / Math.max(1, sourceHeight); + const uniformScale = Math.max( + 0.05, + Math.min( + Number.isFinite(widthScale) && widthScale > 0 ? widthScale : 1, + Number.isFinite(heightScale) && heightScale > 0 ? heightScale : 1 + ) + ); + const placedWidth = clamp( + Math.round(sourceWidth * uniformScale), + 1, + Math.max(1, width) + ); + const placedHeight = clamp( + Math.round(sourceHeight * uniformScale), + 1, + Math.max(1, height) + ); + const targetCenterX = target.left + target.width / 2; + const targetCenterY = target.top + target.height / 2; + const placedLeft = clamp( + Math.round(targetCenterX - placedWidth / 2), + 0, + Math.max(0, width - placedWidth) + ); + const placedTop = clamp( + Math.round(targetCenterY - placedHeight / 2), + 0, + Math.max(0, height - placedHeight) + ); + + return { + placedHeight: Math.max(1, Math.round(placedHeight)), + placedLeft: Math.max(0, Math.round(placedLeft)), + placedTop: Math.max(0, Math.round(placedTop)), + placedWidth: Math.max(1, Math.round(placedWidth)), + sourceBox: { + left: Math.round(sourceLeft), + top: Math.round(sourceTop), + width: sourceWidth, + height: sourceHeight, + }, + targetBox: target, + uniformScale, + }; +} + +async function buildScaleAdjustVisualLayoutGuideBuffer(params: { + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; + sourceBuffer: Buffer; + sourceHeight: number; + sourceWidth: number; +}) { + const { regions, scalePercent, sourceBuffer, sourceHeight, sourceWidth } = + params; + const composites: Array<{ input: Buffer; left: number; top: number }> = []; + + for (const region of regions) { + const placement = getScaleAdjustVisualGuidePlacement({ + height: sourceHeight, + region, + scalePercent, + width: sourceWidth, + }); + const cropBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .extract(placement.sourceBox) + .resize(placement.placedWidth, placement.placedHeight, { + fit: "fill", + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + composites.push({ + input: cropBuffer, + left: placement.placedLeft, + top: placement.placedTop, + }); + } + + return sharp({ + create: { + background: SCALE_ADJUST_LAYOUT_GUIDE_BACKGROUND, + channels: 3, + height: sourceHeight, + width: sourceWidth, + }, + }) + .composite(composites) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + function wrapText(text: string, maxChars: number, maxLines = 3) { const compact = text.replace(/\s+/g, " ").trim(); if (!compact) return []; @@ -1067,7 +1185,7 @@ function buildScaleAdjustPrompt( cropIndex >= 0 ? `；输入图片${cropIndex + 3}是该区域的干净局部裁切参考图，只用于识别目标主体的身份、轮廓与原始比例` : ""; - return `区域 ${region.index}${cropNote}：请在输入图片1中找到源框范围内最主要、可独立调整的商品/物体主体。${sourcePixelLine}；${targetPixelLine}；${scaleLine}${targetPositionNote}源框只是“寻找主体”的范围，不是最终边界，也不是要缩放或贴回的矩形补丁。目标框只是主体调整后的视觉占位参考，不要求主体填满矩形，更不能改变主体自身宽高比、透视或形体。请像真实图片编辑一样先保留锁定的原始画布，再只把这个主体整体等比变大/变小/移动到目标视觉占位；主体缩放后露出或遮挡的极小背景区域需要自然重建。严禁把输入图片1整张图、输入图片2坐标图、任意裁切图、源框矩形区域或目标框矩形区域当成图片块整体缩放、平移、复制、裁切或贴回。如果源框中只覆盖到主体的一部分，请根据局部裁切参考和可见结构推断该主体的完整可见轮廓，只调整主体，不调整矩形框内背景。`; + return `区域 ${region.index}${cropNote}：请在输入图片1中找到源框范围内最主要、可独立调整的商品/物体主体。${sourcePixelLine}；${targetPixelLine}；${scaleLine}${targetPositionNote}源框只是“寻找主体”的范围，不是最终边界，也不是要缩放或贴回的矩形补丁。目标框只是主体调整后的视觉占位参考，不要求主体填满矩形，更不能改变主体自身宽高比、透视或形体。请像真实图片编辑一样先保留锁定的原始画布，再只把这个主体整体等比变大/变小/移动到目标视觉占位；主体缩放后露出或遮挡的极小背景区域需要自然重建。严禁把输入图片1整张图、输入图片2目标布局参考图、任意裁切图、源框矩形区域或目标框矩形区域当成图片块整体缩放、平移、复制、裁切或贴回。如果源框中只覆盖到主体的一部分，请根据局部裁切参考和可见结构推断该主体的完整可见轮廓，只调整主体，不调整矩形框内背景。`; }) .join("\n"); const cropLines = cropReferences @@ -1081,17 +1199,17 @@ function buildScaleAdjustPrompt( "这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。", sourceSizeLine, "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", - "输入图片2是一张与输入图片1尺寸完全相同的灰度坐标指令图，不是照片，也不是输出底图。它只用细线源框和目标框说明坐标关系；不得学习、复刻或输出其中的线框、灰底或 UI 标记。", + "输入图片2是一张与输入图片1尺寸完全相同的中性背景目标布局参考图，不是照片，也不是输出底图。它把源框里的目标主体粗略缩放并摆放到用户希望的最终位置和大小；请只读取其中主体的最终视觉占位，不得学习、复刻或输出中性背景、矩形裁切边缘或参考图痕迹。", cropLines ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` : "", - "输入图片2中的源框表示输入图片1里目标主体当前大致位置；它只用于定位主体，不限制调整后的空间。目标框表示用户希望主体调整后的视觉占位和中心位置。最终画面不得出现这些线框或灰底；主体也不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", + "输入图片2中的目标布局是强约束：它展示目标主体调整后的大致视觉占位和中心位置。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", "如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", "系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是放大后的空间上限，更不是要整体缩放的矩形图层。", "最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度；严禁整张图整体缩放、裁切、平移、重新排版或重新生成。", "Highest priority rule: do NOT zoom, crop, resize, reframe, pan, or regenerate the full canvas. Treat image 1 as a locked background plate; edit only the selected subject and the smallest necessary background around its old and new footprint.", "请不要生成任何彩色框、矩形边框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件或半透明色块。", - "请同时使用下方每个区域的源框/目标框像素坐标和输入图片2的坐标图来理解目标占位。坐标只用于指导主体位置和大小，不是要求把某个矩形图块缩放后贴回。", + "请同时使用下方每个区域的源框/目标框像素坐标和输入图片2的目标布局参考来理解目标占位。坐标和参考图只用于指导主体位置和大小，不是要求把某个矩形图块缩放后贴回。", "", "任务：先在输入图片1中识别源框范围内最主要、可独立调整的主体物体，再只对这个“主体物体”做局部比例调整。调整后的主体大小和位置必须接近目标框表示的视觉占位；如果输出看起来和原图一样，就是失败。", regionLines, @@ -1170,11 +1288,11 @@ function buildScaleAdjustCoordinatePrompt( "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具必须消失并补成自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；如果最终看起来和原图一样，就是失败。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a separate neutral coordinate instruction map, not a photo and not output content. The light gray OLD rectangle/label locates the current furniture in input image 1; the dark TARGET rectangle/label gives the required final size and center. The map may include arrows, scale-factor text, OLD OUTSIDE = BACKGROUND, and TARGET = FINAL FOOTPRINT labels: read them as instructions only, never as visual content.", + "Input image 2 is a separate neutral-background target layout reference, not a photo and not output content. It places a rough resized crop of the selected furniture at the required final position and size. Read only the final furniture footprint from input image 2; never copy its neutral background, rectangular crop edges, rough patch quality, or guide artifacts into the output.", cropLines ? `${cropLines}` : "", "", "Task:", - "Find the main movable furniture subject inside each OLD box in input image 1. Resize and, if needed, move that subject only so its final visible footprint matches the TARGET box.", + "Find the main movable furniture subject inside each OLD box in input image 1. Resize and, if needed, move that subject only so its final visible footprint matches the target layout shown in input image 2 and the numeric TARGET coordinates below.", "Do a real subject-scale edit, not a full image zoom. The camera, room layout, windows, wall decor, text, floor, rug, and all non-target furniture must remain stable.", "If an OLD box contains more than one object, choose the dominant furniture/product inside that box, usually the largest or most central movable item. Keep adjacent lamps, plants, side tables, pillows, wall decor, windows, text, and background objects at their original size unless they are clearly part of the same furniture subject.", "Keep the furniture identity, material, perspective, objects on top, and natural aspect ratio from input image 1. Do not create a second copy or extra parts. Do not preserve the original furniture size just because the scene looks realistic; the user explicitly wants a visible scale change.", @@ -1186,8 +1304,8 @@ function buildScaleAdjustCoordinatePrompt( "Region requirements:", regionLines, "", - "Success check: compare the final furniture footprint with the OLD and TARGET boxes from input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", - "Do not paste rectangular image areas. Do not output coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", + "Success check: compare the final furniture footprint with input image 1 and the target layout reference in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", + "Do not paste rectangular image areas. Do not output layout-reference background, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", "Return only the clean edited photo.", ].join("\n"); } @@ -1195,13 +1313,13 @@ function buildScaleAdjustCoordinatePrompt( function buildScaleAdjustCoordinateSystemPrompt() { return [ "You are an image editing model for local furniture scale adjustment.", - "Input image 1 is the only final photo base. Input image 2 is a coordinate instruction map, not a photo layer.", + "Input image 1 is the only final photo base. Input image 2 is a rough target layout reference, not a photo layer.", "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding unchanged furniture and full-canvas zoom.", "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is a failed edit.", "When shrinking, remove the old full-size furniture footprint outside the target footprint and heal it as natural background. Large furniture is editable and must not remain locked at original size.", "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", - "Never output guide artifacts from the coordinate reference: no lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", + "Never output guide artifacts from the layout reference: no neutral background, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", ].join("\n"); } @@ -1369,29 +1487,30 @@ export async function POST(request: NextRequest) { const scaleAdjustModelFamily = getScaleAdjustModelFamily( resolvedScaleAdjustModelEntry ); - const annotatedBuffer = await sharp(sourceBuffer, { failOn: "none" }) - .composite([ - { - input: isScaleAdjustOperation - ? buildScaleAdjustAnnotationSvg({ - height: sourceHeight, - regions, - scalePercent, - width: sourceWidth, - }) - : buildAnnotationSvg({ + const annotatedBuffer = isScaleAdjustOperation + ? await buildScaleAdjustVisualLayoutGuideBuffer({ + regions, + scalePercent, + sourceBuffer, + sourceHeight, + sourceWidth, + }) + : await sharp(sourceBuffer, { failOn: "none" }) + .composite([ + { + input: buildAnnotationSvg({ height: sourceHeight, hidePromptText: false, regions, width: sourceWidth, }), - left: 0, - top: 0, - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer(); - const annotatedFilename = `${isScaleAdjustOperation ? "canvas-scale-adjust-guide" : "canvas-local-edit"}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; + left: 0, + top: 0, + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const annotatedFilename = `${isScaleAdjustOperation ? "canvas-scale-adjust-layout-guide" : "canvas-local-edit"}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; const annotatedUpload = await uploadFile( annotatedFilename, toArrayBuffer(annotatedBuffer) @@ -1424,7 +1543,7 @@ export async function POST(request: NextRequest) { ) : buildLocalEditPrompt(regions); const scaleAdjustCoordinateNegativePrompt = - "visible annotation, guide overlay, translucent overlay, text label, arrows, boxes, rectangle border, UI marks, pasted patch, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; + "visible annotation, guide overlay, translucent overlay, neutral layout background, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -1480,7 +1599,7 @@ export async function POST(request: NextRequest) { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation - ? "scale-adjust-neutral-coordinate-guide-map" + ? "scale-adjust-rough-visual-layout-guide" : "source-overlay", cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, @@ -1495,6 +1614,7 @@ export async function POST(request: NextRequest) { resolvedScaleAdjustModelEntry?.provider?.type || null, scaleAdjustModelFamily: scaleAdjustModelFamily || null, scaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide, + scaleAdjustVisualLayoutGuideApplied: isScaleAdjustOperation, falScaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image", @@ -1504,7 +1624,7 @@ export async function POST(request: NextRequest) { : "default-model-failover" : undefined, scaleAdjustQualityFailoverPolicy: isScaleAdjustOperation - ? "disabled-return-first-ai-candidate" + ? "disabled-reject-unchanged-ai-candidate" : undefined, scaleAdjustPreferSameNameProviderFailover: isScaleAdjustOperation && Boolean(scaleAdjustModelFamily), @@ -1523,8 +1643,8 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-locked-source-neutral-coordinate-map-v31-one-shot" - : "locked-plate-neutral-coordinate-guide-map-pixel-anchors-v8" + ? "same-name-locked-source-visual-layout-guide-v32-one-shot" + : "locked-plate-visual-layout-guide-v9" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 6709efd5c..988592daf 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -2671,7 +2671,7 @@ export async function generateCanvasImage( } if (scaleAdjustRejection?.images?.length) { - console.warn("[Canvas Generate Image] Returning rejected scale adjust candidate because no fallback remains:", { + console.warn("[Canvas Generate Image] Rejecting scale adjust candidate because no effective edit was detected:", { debugLabel, imageCount: scaleAdjustRejection.images.length, reason: scaleAdjustRejection.reason, @@ -2679,12 +2679,7 @@ export async function generateCanvasImage( selectedModelId: selectedModel.model.modelId, ...debugMeta, }); - return { - images: scaleAdjustRejection.images, - model: selectedModel.model.displayName, - modelRef: selectedModel.model.id, - prompt: resultPrompt, - }; + throw caughtError; } throw caughtError; From 62e3bc461140b656b4a1c0b4c5c5daf11877e442 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 15:06:30 +0800 Subject: [PATCH 02/73] fix: constrain scale adjust full-frame guide --- app/api/canvas/local-edit/route.ts | 134 +++++++++++++++++++++++------ 1 file changed, 108 insertions(+), 26 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 448d16e2e..697eaebf1 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -100,7 +100,7 @@ const MAX_REFERENCE_IMAGES_PER_REGION = 3; const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; const SCALE_ADJUST_CROP_MAX_EDGE = 1536; -const SCALE_ADJUST_LAYOUT_GUIDE_BACKGROUND = "#f5f7fb"; +const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb"; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; @@ -834,6 +834,57 @@ function getScaleAdjustVisualGuidePlacement(params: { }; } +function buildScaleAdjustSoftWashSvg(params: { + boxes: Array<{ left: number; top: number; width: number; height: number }>; + height: number; + width: number; +}) { + const { boxes, height, width } = params; + const blur = Math.max(8, Math.min(42, Math.round(Math.min(width, height) * 0.025))); + const rects = boxes + .map((box) => { + const left = Math.max(0, box.left - blur); + const top = Math.max(0, box.top - blur); + const right = Math.min(width, box.left + box.width + blur); + const bottom = Math.min(height, box.top + box.height + blur); + return ``; + }) + .join(""); + + return Buffer.from( + ` + + + + + + ${rects} + ` + ); +} + +function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) { + const { height, width } = params; + const blur = Math.max(2, Math.min(18, Math.round(Math.min(width, height) * 0.055))); + const inset = Math.min( + Math.floor(Math.min(width, height) / 3), + Math.max(2, blur * 2) + ); + const rectWidth = Math.max(1, width - inset * 2); + const rectHeight = Math.max(1, height - inset * 2); + + return Buffer.from( + ` + + + + + + + ` + ); +} + async function buildScaleAdjustVisualLayoutGuideBuffer(params: { regions: NormalizedLocalEditRegion[]; scalePercent?: ScaleAdjustPercent | null; @@ -843,22 +894,45 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { }) { const { regions, scalePercent, sourceBuffer, sourceHeight, sourceWidth } = params; + const washBoxes: Array<{ left: number; top: number; width: number; height: number }> = + []; const composites: Array<{ input: Buffer; left: number; top: number }> = []; for (const region of regions) { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const workspaceBox = rectToPixels( + getScaleAdjustWorkspaceRect(region.rect, targetRect, scalePercent), + sourceWidth, + sourceHeight + ); const placement = getScaleAdjustVisualGuidePlacement({ height: sourceHeight, region, scalePercent, width: sourceWidth, }); - const cropBuffer = await sharp(sourceBuffer, { failOn: "none" }) + washBoxes.push(workspaceBox); + + const resizedCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) .extract(placement.sourceBox) .resize(placement.placedWidth, placement.placedHeight, { fit: "fill", }) .png({ compressionLevel: 6 }) .toBuffer(); + const cropBuffer = await sharp(resizedCropBuffer, { failOn: "none" }) + .ensureAlpha() + .composite([ + { + input: buildScaleAdjustPatchMaskSvg({ + height: placement.placedHeight, + width: placement.placedWidth, + }), + blend: "dest-in", + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); composites.push({ input: cropBuffer, left: placement.placedLeft, @@ -866,15 +940,25 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { }); } - return sharp({ - create: { - background: SCALE_ADJUST_LAYOUT_GUIDE_BACKGROUND, - channels: 3, - height: sourceHeight, - width: sourceWidth, - }, - }) - .composite(composites) + const guideComposites: Array<{ input: Buffer; left: number; top: number }> = [ + ...(washBoxes.length + ? [ + { + input: buildScaleAdjustSoftWashSvg({ + boxes: washBoxes, + height: sourceHeight, + width: sourceWidth, + }), + left: 0, + top: 0, + }, + ] + : []), + ...composites, + ]; + + return sharp(sourceBuffer, { failOn: "none" }) + .composite(guideComposites) .png({ compressionLevel: 6 }) .toBuffer(); } @@ -1199,11 +1283,11 @@ function buildScaleAdjustPrompt( "这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。", sourceSizeLine, "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", - "输入图片2是一张与输入图片1尺寸完全相同的中性背景目标布局参考图，不是照片，也不是输出底图。它把源框里的目标主体粗略缩放并摆放到用户希望的最终位置和大小；请只读取其中主体的最终视觉占位，不得学习、复刻或输出中性背景、矩形裁切边缘或参考图痕迹。", + "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面粗略目标布局参考图，不是输出底图。它基于原图生成：旧主体占位被柔和淡化，只表示需要自然补背景；同一目标主体被粗略缩放并摆放到用户希望的最终位置和大小。请只读取其中目标主体的最终视觉占位和原画幅关系，不得学习、复刻或输出淡化区域、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", cropLines ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` : "", - "输入图片2中的目标布局是强约束：它展示目标主体调整后的大致视觉占位和中心位置。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", + "输入图片2中的完整画面目标布局是强约束：它展示目标主体调整后的大致视觉占位、中心位置，以及整张图不应缩放/裁切/平移的稳定画幅关系。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", "如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", "系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是放大后的空间上限，更不是要整体缩放的矩形图层。", "最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度；严禁整张图整体缩放、裁切、平移、重新排版或重新生成。", @@ -1288,7 +1372,7 @@ function buildScaleAdjustCoordinatePrompt( "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具必须消失并补成自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；如果最终看起来和原图一样，就是失败。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a separate neutral-background target layout reference, not a photo and not output content. It places a rough resized crop of the selected furniture at the required final position and size. Read only the final furniture footprint from input image 2; never copy its neutral background, rectangular crop edges, rough patch quality, or guide artifacts into the output.", + "Input image 2 is a full-frame rough after-layout mockup with the same canvas as input image 1, not output content and not a photo layer. It keeps the original room frame locked, softly fades the OLD footprint to mean natural background healing, and places the same furniture at the required TARGET footprint. Read only the final furniture footprint, center, and locked frame relationship from input image 2; never copy faded areas, rectangular crop edges, rough patch quality, seams, or guide artifacts into the output.", cropLines ? `${cropLines}` : "", "", "Task:", @@ -1304,8 +1388,8 @@ function buildScaleAdjustCoordinatePrompt( "Region requirements:", regionLines, "", - "Success check: compare the final furniture footprint with input image 1 and the target layout reference in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", - "Do not paste rectangular image areas. Do not output layout-reference background, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", + "Success check: compare the final furniture footprint with input image 1 and the full-frame target layout reference in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", + "Do not paste rectangular image areas. Do not output layout-reference faded footprint, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", "Return only the clean edited photo.", ].join("\n"); } @@ -1313,13 +1397,13 @@ function buildScaleAdjustCoordinatePrompt( function buildScaleAdjustCoordinateSystemPrompt() { return [ "You are an image editing model for local furniture scale adjustment.", - "Input image 1 is the only final photo base. Input image 2 is a rough target layout reference, not a photo layer.", + "Input image 1 is the only final photo base. Input image 2 is a full-frame rough after-layout mockup, not a photo layer.", "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding unchanged furniture and full-canvas zoom.", "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is a failed edit.", "When shrinking, remove the old full-size furniture footprint outside the target footprint and heal it as natural background. Large furniture is editable and must not remain locked at original size.", "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", - "Never output guide artifacts from the layout reference: no neutral background, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", + "Never output guide artifacts from the layout reference: no faded old-footprint wash, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", ].join("\n"); } @@ -1543,7 +1627,7 @@ export async function POST(request: NextRequest) { ) : buildLocalEditPrompt(regions); const scaleAdjustCoordinateNegativePrompt = - "visible annotation, guide overlay, translucent overlay, neutral layout background, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; + "visible annotation, guide overlay, translucent overlay, faded old-footprint wash, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -1563,9 +1647,7 @@ export async function POST(request: NextRequest) { : []; const scaleAdjustVerificationConfig = isScaleAdjustOperation ? { - allowOutsideChangedWhenChangedInside: useScaleAdjustCoordinateGuide - ? true - : undefined, + allowOutsideChangedWhenChangedInside: false, protectedRegions: scaleAdjustCompositeRegions, rejectFailedCandidates: useScaleAdjustCoordinateGuide && Boolean(scaleAdjustModelFamily) @@ -1599,7 +1681,7 @@ export async function POST(request: NextRequest) { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation - ? "scale-adjust-rough-visual-layout-guide" + ? "scale-adjust-full-frame-rough-after-layout-guide" : "source-overlay", cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, @@ -1643,8 +1725,8 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-locked-source-visual-layout-guide-v32-one-shot" - : "locked-plate-visual-layout-guide-v9" + ? "same-name-locked-source-full-frame-after-layout-guide-v33-one-shot" + : "locked-plate-full-frame-after-layout-guide-v10" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, @@ -1669,7 +1751,7 @@ export async function POST(request: NextRequest) { isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? scaleAdjustCoordinateNegativePrompt - : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、整体重绘、删除文字、清理文字、改写文字、文字模糊、Logo消失、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" + : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、淡化旧占位、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、整体重绘、删除文字、清理文字、改写文字、文字模糊、Logo消失、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, From cd19fe0b8dd34c009f2a5b3f3856778a33008d6c Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 15:20:58 +0800 Subject: [PATCH 03/73] fix: reduce scale adjust guide recompose bias --- app/api/canvas/local-edit/route.ts | 82 +++++------------------------- 1 file changed, 13 insertions(+), 69 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 697eaebf1..900bbce26 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -100,7 +100,6 @@ const MAX_REFERENCE_IMAGES_PER_REGION = 3; const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; const SCALE_ADJUST_CROP_MAX_EDGE = 1536; -const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb"; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; @@ -834,35 +833,6 @@ function getScaleAdjustVisualGuidePlacement(params: { }; } -function buildScaleAdjustSoftWashSvg(params: { - boxes: Array<{ left: number; top: number; width: number; height: number }>; - height: number; - width: number; -}) { - const { boxes, height, width } = params; - const blur = Math.max(8, Math.min(42, Math.round(Math.min(width, height) * 0.025))); - const rects = boxes - .map((box) => { - const left = Math.max(0, box.left - blur); - const top = Math.max(0, box.top - blur); - const right = Math.min(width, box.left + box.width + blur); - const bottom = Math.min(height, box.top + box.height + blur); - return ``; - }) - .join(""); - - return Buffer.from( - ` - - - - - - ${rects} - ` - ); -} - function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) { const { height, width } = params; const blur = Math.max(2, Math.min(18, Math.round(Math.min(width, height) * 0.055))); @@ -894,24 +864,15 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { }) { const { regions, scalePercent, sourceBuffer, sourceHeight, sourceWidth } = params; - const washBoxes: Array<{ left: number; top: number; width: number; height: number }> = - []; const composites: Array<{ input: Buffer; left: number; top: number }> = []; for (const region of regions) { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const workspaceBox = rectToPixels( - getScaleAdjustWorkspaceRect(region.rect, targetRect, scalePercent), - sourceWidth, - sourceHeight - ); const placement = getScaleAdjustVisualGuidePlacement({ height: sourceHeight, region, scalePercent, width: sourceWidth, }); - washBoxes.push(workspaceBox); const resizedCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) .extract(placement.sourceBox) @@ -940,25 +901,8 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { }); } - const guideComposites: Array<{ input: Buffer; left: number; top: number }> = [ - ...(washBoxes.length - ? [ - { - input: buildScaleAdjustSoftWashSvg({ - boxes: washBoxes, - height: sourceHeight, - width: sourceWidth, - }), - left: 0, - top: 0, - }, - ] - : []), - ...composites, - ]; - return sharp(sourceBuffer, { failOn: "none" }) - .composite(guideComposites) + .composite(composites) .png({ compressionLevel: 6 }) .toBuffer(); } @@ -1283,11 +1227,11 @@ function buildScaleAdjustPrompt( "这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。", sourceSizeLine, "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", - "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面粗略目标布局参考图，不是输出底图。它基于原图生成：旧主体占位被柔和淡化，只表示需要自然补背景；同一目标主体被粗略缩放并摆放到用户希望的最终位置和大小。请只读取其中目标主体的最终视觉占位和原画幅关系，不得学习、复刻或输出淡化区域、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", + "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面目标尺寸提示图，不是输出底图。它保留原图完整画幅作为稳定参考，并把源框里的同一目标主体粗略缩放后叠放到用户希望的最终位置和大小。请只读取目标主体的最终视觉占位和原画幅关系；输入图片2里仍可见的原尺寸主体/背景只用于锁定原画幅，不能表示最终还要保留原尺寸主体。不得学习、复刻或输出矩形裁切边缘、拼接痕迹或参考图粗糙画质。", cropLines ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` : "", - "输入图片2中的完整画面目标布局是强约束：它展示目标主体调整后的大致视觉占位、中心位置，以及整张图不应缩放/裁切/平移的稳定画幅关系。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", + "输入图片2中的完整画面目标尺寸提示是强约束：它展示目标主体调整后的大致视觉占位、中心位置，以及整张图不应缩放/裁切/平移的稳定画幅关系。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", "如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", "系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是放大后的空间上限，更不是要整体缩放的矩形图层。", "最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度；严禁整张图整体缩放、裁切、平移、重新排版或重新生成。", @@ -1372,7 +1316,7 @@ function buildScaleAdjustCoordinatePrompt( "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具必须消失并补成自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；如果最终看起来和原图一样，就是失败。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a full-frame rough after-layout mockup with the same canvas as input image 1, not output content and not a photo layer. It keeps the original room frame locked, softly fades the OLD footprint to mean natural background healing, and places the same furniture at the required TARGET footprint. Read only the final furniture footprint, center, and locked frame relationship from input image 2; never copy faded areas, rectangular crop edges, rough patch quality, seams, or guide artifacts into the output.", + "Input image 2 is a full-frame target-size placement hint with the same canvas as input image 1, not output content and not a photo layer. It keeps the original room frame visible as the locked reference and overlays the same furniture at the required TARGET footprint. Read only the final furniture footprint, center, and locked frame relationship from input image 2. If the original-size furniture is still visible behind the rough hint, ignore that old footprint in the final; the final must contain only the TARGET-size furniture. Never copy rectangular crop edges, rough patch quality, seams, or guide artifacts into the output.", cropLines ? `${cropLines}` : "", "", "Task:", @@ -1388,8 +1332,8 @@ function buildScaleAdjustCoordinatePrompt( "Region requirements:", regionLines, "", - "Success check: compare the final furniture footprint with input image 1 and the full-frame target layout reference in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", - "Do not paste rectangular image areas. Do not output layout-reference faded footprint, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", + "Success check: compare the final furniture footprint with input image 1 and the full-frame target-size placement hint in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", + "Do not paste rectangular image areas. Do not output layout-reference overlays, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", "Return only the clean edited photo.", ].join("\n"); } @@ -1397,13 +1341,13 @@ function buildScaleAdjustCoordinatePrompt( function buildScaleAdjustCoordinateSystemPrompt() { return [ "You are an image editing model for local furniture scale adjustment.", - "Input image 1 is the only final photo base. Input image 2 is a full-frame rough after-layout mockup, not a photo layer.", + "Input image 1 is the only final photo base. Input image 2 is a full-frame target-size placement hint, not a photo layer.", "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding unchanged furniture and full-canvas zoom.", "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is a failed edit.", "When shrinking, remove the old full-size furniture footprint outside the target footprint and heal it as natural background. Large furniture is editable and must not remain locked at original size.", "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", - "Never output guide artifacts from the layout reference: no faded old-footprint wash, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", + "Never output guide artifacts from the layout reference: no overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", ].join("\n"); } @@ -1627,7 +1571,7 @@ export async function POST(request: NextRequest) { ) : buildLocalEditPrompt(regions); const scaleAdjustCoordinateNegativePrompt = - "visible annotation, guide overlay, translucent overlay, faded old-footprint wash, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; + "visible annotation, guide overlay, translucent overlay, target placement overlay, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -1681,7 +1625,7 @@ export async function POST(request: NextRequest) { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation - ? "scale-adjust-full-frame-rough-after-layout-guide" + ? "scale-adjust-full-frame-target-size-placement-hint" : "source-overlay", cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, @@ -1725,8 +1669,8 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-locked-source-full-frame-after-layout-guide-v33-one-shot" - : "locked-plate-full-frame-after-layout-guide-v10" + ? "same-name-locked-source-target-size-placement-hint-v34-one-shot" + : "locked-plate-target-size-placement-hint-v11" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, @@ -1751,7 +1695,7 @@ export async function POST(request: NextRequest) { isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? scaleAdjustCoordinateNegativePrompt - : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、淡化旧占位、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、整体重绘、删除文字、清理文字、改写文字、文字模糊、Logo消失、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" + : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、整体重绘、删除文字、清理文字、改写文字、文字模糊、Logo消失、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, From ba1bb165c318ab2409df442d8e7a84fe469bc53c Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 15:38:29 +0800 Subject: [PATCH 04/73] fix: hint old footprint for scale adjust --- app/api/canvas/local-edit/route.ts | 69 ++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 900bbce26..16beca3de 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -100,6 +100,7 @@ const MAX_REFERENCE_IMAGES_PER_REGION = 3; const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; const SCALE_ADJUST_CROP_MAX_EDGE = 1536; +const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb"; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; @@ -855,6 +856,35 @@ function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) ); } +function buildScaleAdjustSourceFootprintWashSvg(params: { + boxes: Array<{ left: number; top: number; width: number; height: number }>; + height: number; + width: number; +}) { + const { boxes, height, width } = params; + const blur = Math.max(4, Math.min(22, Math.round(Math.min(width, height) * 0.016))); + const rects = boxes + .map((box) => { + const left = Math.max(0, box.left - blur); + const top = Math.max(0, box.top - blur); + const right = Math.min(width, box.left + box.width + blur); + const bottom = Math.min(height, box.top + box.height + blur); + return ``; + }) + .join(""); + + return Buffer.from( + ` + + + + + + ${rects} + ` + ); +} + async function buildScaleAdjustVisualLayoutGuideBuffer(params: { regions: NormalizedLocalEditRegion[]; scalePercent?: ScaleAdjustPercent | null; @@ -864,6 +894,12 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { }) { const { regions, scalePercent, sourceBuffer, sourceHeight, sourceWidth } = params; + const sourceFootprintBoxes: Array<{ + left: number; + top: number; + width: number; + height: number; + }> = []; const composites: Array<{ input: Buffer; left: number; top: number }> = []; for (const region of regions) { @@ -873,6 +909,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { scalePercent, width: sourceWidth, }); + sourceFootprintBoxes.push(placement.sourceBox); const resizedCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) .extract(placement.sourceBox) @@ -901,8 +938,22 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { }); } + const washComposite = sourceFootprintBoxes.length + ? [ + { + input: buildScaleAdjustSourceFootprintWashSvg({ + boxes: sourceFootprintBoxes, + height: sourceHeight, + width: sourceWidth, + }), + left: 0, + top: 0, + }, + ] + : []; + return sharp(sourceBuffer, { failOn: "none" }) - .composite(composites) + .composite([...washComposite, ...composites]) .png({ compressionLevel: 6 }) .toBuffer(); } @@ -1227,7 +1278,7 @@ function buildScaleAdjustPrompt( "这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。", sourceSizeLine, "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", - "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面目标尺寸提示图，不是输出底图。它保留原图完整画幅作为稳定参考，并把源框里的同一目标主体粗略缩放后叠放到用户希望的最终位置和大小。请只读取目标主体的最终视觉占位和原画幅关系；输入图片2里仍可见的原尺寸主体/背景只用于锁定原画幅，不能表示最终还要保留原尺寸主体。不得学习、复刻或输出矩形裁切边缘、拼接痕迹或参考图粗糙画质。", + "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面目标尺寸提示图，不是输出底图。它保留原图完整画幅作为稳定参考，只在源框旧主体占位内做轻微淡化来提示“旧主体需要被目标尺寸替换”，并把源框里的同一目标主体粗略缩放后叠放到用户希望的最终位置和大小。请只读取目标主体的最终视觉占位和原画幅关系；输入图片2里仍可见的原尺寸主体/背景只用于锁定原画幅，不能表示最终还要保留原尺寸主体。不得学习、复刻或输出淡化痕迹、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", cropLines ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` : "", @@ -1316,7 +1367,7 @@ function buildScaleAdjustCoordinatePrompt( "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具必须消失并补成自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；如果最终看起来和原图一样，就是失败。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a full-frame target-size placement hint with the same canvas as input image 1, not output content and not a photo layer. It keeps the original room frame visible as the locked reference and overlays the same furniture at the required TARGET footprint. Read only the final furniture footprint, center, and locked frame relationship from input image 2. If the original-size furniture is still visible behind the rough hint, ignore that old footprint in the final; the final must contain only the TARGET-size furniture. Never copy rectangular crop edges, rough patch quality, seams, or guide artifacts into the output.", + "Input image 2 is a full-frame target-size placement hint with the same canvas as input image 1, not output content and not a photo layer. It keeps the original room frame visible as the locked reference, softly mutes only the OLD source footprint to indicate replacement, and overlays the same furniture at the required TARGET footprint. Read only the final furniture footprint, center, and locked frame relationship from input image 2. If the original-size furniture is still visible behind the rough hint, ignore that old footprint in the final; the final must contain only the TARGET-size furniture. Never copy muted wash, rectangular crop edges, rough patch quality, seams, or guide artifacts into the output.", cropLines ? `${cropLines}` : "", "", "Task:", @@ -1333,7 +1384,7 @@ function buildScaleAdjustCoordinatePrompt( regionLines, "", "Success check: compare the final furniture footprint with input image 1 and the full-frame target-size placement hint in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", - "Do not paste rectangular image areas. Do not output layout-reference overlays, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", + "Do not paste rectangular image areas. Do not output layout-reference overlays, muted wash, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", "Return only the clean edited photo.", ].join("\n"); } @@ -1347,7 +1398,7 @@ function buildScaleAdjustCoordinateSystemPrompt() { "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is a failed edit.", "When shrinking, remove the old full-size furniture footprint outside the target footprint and heal it as natural background. Large furniture is editable and must not remain locked at original size.", "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", - "Never output guide artifacts from the layout reference: no overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", + "Never output guide artifacts from the layout reference: no muted wash, overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", ].join("\n"); } @@ -1571,7 +1622,7 @@ export async function POST(request: NextRequest) { ) : buildLocalEditPrompt(regions); const scaleAdjustCoordinateNegativePrompt = - "visible annotation, guide overlay, translucent overlay, target placement overlay, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; + "visible annotation, guide overlay, translucent overlay, target placement overlay, muted wash, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -1669,8 +1720,8 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-locked-source-target-size-placement-hint-v34-one-shot" - : "locked-plate-target-size-placement-hint-v11" + ? "same-name-locked-source-tight-muted-target-placement-hint-v35-one-shot" + : "locked-plate-tight-muted-target-placement-hint-v12" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, @@ -1695,7 +1746,7 @@ export async function POST(request: NextRequest) { isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? scaleAdjustCoordinateNegativePrompt - : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、整体重绘、删除文字、清理文字、改写文字、文字模糊、Logo消失、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" + : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、淡化痕迹、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、整体重绘、删除文字、清理文字、改写文字、文字模糊、Logo消失、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, From 183d47fba021f63bda6df26abd434b28d503ece3 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 15:44:57 +0800 Subject: [PATCH 05/73] chore: log rejected scale adjust previews --- lib/canvas/generate-image.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 988592daf..20750cab9 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -2674,6 +2674,10 @@ export async function generateCanvasImage( console.warn("[Canvas Generate Image] Rejecting scale adjust candidate because no effective edit was detected:", { debugLabel, imageCount: scaleAdjustRejection.images.length, + rejectedImageUrls: scaleAdjustRejection.images.map((image) => ({ + originalUrl: image.originalUrl, + thumbnailUrl: image.thumbnailUrl, + })), reason: scaleAdjustRejection.reason, selectedModelRef: selectedModel.model.id, selectedModelId: selectedModel.model.modelId, From 354fa98477c524325bfe73da033919fb5c1582bd Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 15:53:42 +0800 Subject: [PATCH 06/73] fix: strengthen scale adjust target hint --- app/api/canvas/local-edit/route.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 16beca3de..a85c90dbd 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -869,7 +869,7 @@ function buildScaleAdjustSourceFootprintWashSvg(params: { const top = Math.max(0, box.top - blur); const right = Math.min(width, box.left + box.width + blur); const bottom = Math.min(height, box.top + box.height + blur); - return ``; + return ``; }) .join(""); @@ -1720,8 +1720,8 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-locked-source-tight-muted-target-placement-hint-v35-one-shot" - : "locked-plate-tight-muted-target-placement-hint-v12" + ? "same-name-locked-source-tight-muted-target-placement-hint-v36-one-shot" + : "locked-plate-tight-muted-target-placement-hint-v13" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, From fb4ef202d8e8c58b4638a5ebe15f79b773538b0b Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 16:03:29 +0800 Subject: [PATCH 07/73] fix: respect requested scale adjust provider --- app/api/canvas/local-edit/route.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index a85c90dbd..456672515 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -367,6 +367,13 @@ function resolveScaleAdjustPreferredModelId(params: { params.models, params.configuredModelId ); + if (isScaleAdjustUsableImageEditModel(requestedModel)) { + const requestedFamily = getScaleAdjustModelFamily(requestedModel); + if (requestedFamily) { + return requestedModel?.model.id || params.requestedModelId; + } + } + const autoDetailModel = params.models.find(isScaleAdjustDedicatedDetailModel); const preferredModel = (isScaleAdjustDedicatedDetailModel(requestedModel) ? requestedModel : null) || From 2ad8bc007302f3d6ae6e6ef602d245b7a278d67c Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 16:26:05 +0800 Subject: [PATCH 08/73] fix: strengthen scale adjust semantic guide --- app/api/canvas/local-edit/route.ts | 96 +++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 16 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 456672515..cd92370c2 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -843,10 +843,10 @@ function getScaleAdjustVisualGuidePlacement(params: { function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) { const { height, width } = params; - const blur = Math.max(2, Math.min(18, Math.round(Math.min(width, height) * 0.055))); + const blur = Math.max(2, Math.min(14, Math.round(Math.min(width, height) * 0.035))); const inset = Math.min( - Math.floor(Math.min(width, height) / 3), - Math.max(2, blur * 2) + Math.floor(Math.min(width, height) / 8), + Math.max(1, Math.round(blur * 0.7)) ); const rectWidth = Math.max(1, width - inset * 2); const rectHeight = Math.max(1, height - inset * 2); @@ -858,7 +858,7 @@ function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) - + ` ); } @@ -876,7 +876,7 @@ function buildScaleAdjustSourceFootprintWashSvg(params: { const top = Math.max(0, box.top - blur); const right = Math.min(width, box.left + box.width + blur); const bottom = Math.min(height, box.top + box.height + blur); - return ``; + return ``; }) .join(""); @@ -892,6 +892,54 @@ function buildScaleAdjustSourceFootprintWashSvg(params: { ); } +function buildScaleAdjustTargetFootprintShadowSvg(params: { + height: number; + placements: Array>; + width: number; +}) { + const { height, placements, width } = params; + const minSide = Math.max(1, Math.min(width, height)); + const blur = Math.max(4, Math.min(18, Math.round(minSide * 0.014))); + const rects = placements + .map((placement) => { + const pad = Math.max(2, Math.round(blur * 0.45)); + const left = clamp( + placement.placedLeft - pad, + 0, + Math.max(0, width - 1) + ); + const top = clamp( + placement.placedTop - pad, + 0, + Math.max(0, height - 1) + ); + const right = clamp( + placement.placedLeft + placement.placedWidth + pad, + left + 1, + width + ); + const bottom = clamp( + placement.placedTop + placement.placedHeight + pad, + top + 1, + height + ); + const radius = Math.max(4, Math.round(blur * 1.2)); + return ``; + }) + .join(""); + + return Buffer.from( + ` + + + + + + ${rects} + ` + ); +} + async function buildScaleAdjustVisualLayoutGuideBuffer(params: { regions: NormalizedLocalEditRegion[]; scalePercent?: ScaleAdjustPercent | null; @@ -907,6 +955,8 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { width: number; height: number; }> = []; + const placements: Array> = + []; const composites: Array<{ input: Buffer; left: number; top: number }> = []; for (const region of regions) { @@ -917,6 +967,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { width: sourceWidth, }); sourceFootprintBoxes.push(placement.sourceBox); + placements.push(placement); const resizedCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) .extract(placement.sourceBox) @@ -958,9 +1009,22 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { }, ] : []; + const targetShadowComposite = placements.length + ? [ + { + input: buildScaleAdjustTargetFootprintShadowSvg({ + height: sourceHeight, + placements, + width: sourceWidth, + }), + left: 0, + top: 0, + }, + ] + : []; return sharp(sourceBuffer, { failOn: "none" }) - .composite([...washComposite, ...composites]) + .composite([...washComposite, ...targetShadowComposite, ...composites]) .png({ compressionLevel: 6 }) .toBuffer(); } @@ -1285,11 +1349,11 @@ function buildScaleAdjustPrompt( "这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。", sourceSizeLine, "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", - "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面目标尺寸提示图，不是输出底图。它保留原图完整画幅作为稳定参考，只在源框旧主体占位内做轻微淡化来提示“旧主体需要被目标尺寸替换”，并把源框里的同一目标主体粗略缩放后叠放到用户希望的最终位置和大小。请只读取目标主体的最终视觉占位和原画幅关系；输入图片2里仍可见的原尺寸主体/背景只用于锁定原画幅，不能表示最终还要保留原尺寸主体。不得学习、复刻或输出淡化痕迹、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", + "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面编辑意图图，不是输出底图。它保留原图完整画幅作为稳定参考，并把源框旧主体占位强淡化为“这里的原尺寸主体需要消失并自然修复”的指令；同时把同一目标主体粗略缩放后放在用户希望的最终位置和大小。请只读取目标主体的最终视觉占位、中心位置和原画幅关系；输入图片2里被淡化的旧主体不能表示最终还要保留原尺寸主体。不得学习、复刻或输出淡化痕迹、阴影占位、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", cropLines ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` : "", - "输入图片2中的完整画面目标尺寸提示是强约束：它展示目标主体调整后的大致视觉占位、中心位置，以及整张图不应缩放/裁切/平移的稳定画幅关系。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", + "输入图片2中的完整画面编辑意图是强约束：淡化的 OLD 占位表示旧尺寸主体要被移除/修复，清晰的 TARGET 占位表示最终主体大小和中心位置；整张图不应缩放、裁切或平移。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", "如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", "系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是放大后的空间上限，更不是要整体缩放的矩形图层。", "最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度；严禁整张图整体缩放、裁切、平移、重新排版或重新生成。", @@ -1374,7 +1438,7 @@ function buildScaleAdjustCoordinatePrompt( "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具必须消失并补成自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；如果最终看起来和原图一样，就是失败。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a full-frame target-size placement hint with the same canvas as input image 1, not output content and not a photo layer. It keeps the original room frame visible as the locked reference, softly mutes only the OLD source footprint to indicate replacement, and overlays the same furniture at the required TARGET footprint. Read only the final furniture footprint, center, and locked frame relationship from input image 2. If the original-size furniture is still visible behind the rough hint, ignore that old footprint in the final; the final must contain only the TARGET-size furniture. Never copy muted wash, rectangular crop edges, rough patch quality, seams, or guide artifacts into the output.", + "Input image 2 is a full-frame semantic edit map with the same canvas as input image 1, not output content and not a photo layer. It keeps the original room frame visible as the locked reference, strongly fades the OLD source footprint to mean remove/heal the old full-size furniture, and shows the same furniture at the required TARGET footprint. Read only the final furniture footprint, center, and locked frame relationship from input image 2. If any original-size furniture is still faintly visible in the faded OLD area, treat it as removed in the final; the final must contain only the TARGET-size furniture. Never copy faded wash, placeholder shadow, rectangular crop edges, rough patch quality, seams, or guide artifacts into the output.", cropLines ? `${cropLines}` : "", "", "Task:", @@ -1390,7 +1454,7 @@ function buildScaleAdjustCoordinatePrompt( "Region requirements:", regionLines, "", - "Success check: compare the final furniture footprint with input image 1 and the full-frame target-size placement hint in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", + "Success check: compare the final furniture footprint with input image 1 and the full-frame semantic edit map in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", "Do not paste rectangular image areas. Do not output layout-reference overlays, muted wash, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", "Return only the clean edited photo.", ].join("\n"); @@ -1399,13 +1463,13 @@ function buildScaleAdjustCoordinatePrompt( function buildScaleAdjustCoordinateSystemPrompt() { return [ "You are an image editing model for local furniture scale adjustment.", - "Input image 1 is the only final photo base. Input image 2 is a full-frame target-size placement hint, not a photo layer.", + "Input image 1 is the only final photo base. Input image 2 is a full-frame semantic edit map, not a photo layer.", "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding unchanged furniture and full-canvas zoom.", "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is a failed edit.", "When shrinking, remove the old full-size furniture footprint outside the target footprint and heal it as natural background. Large furniture is editable and must not remain locked at original size.", "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", - "Never output guide artifacts from the layout reference: no muted wash, overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", + "Never output guide artifacts from the edit map: no faded wash, placeholder shadow, overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", ].join("\n"); } @@ -1629,7 +1693,7 @@ export async function POST(request: NextRequest) { ) : buildLocalEditPrompt(regions); const scaleAdjustCoordinateNegativePrompt = - "visible annotation, guide overlay, translucent overlay, target placement overlay, muted wash, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; + "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, old-size target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -1683,7 +1747,7 @@ export async function POST(request: NextRequest) { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation - ? "scale-adjust-full-frame-target-size-placement-hint" + ? "scale-adjust-full-frame-semantic-edit-map" : "source-overlay", cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, @@ -1727,8 +1791,8 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-locked-source-tight-muted-target-placement-hint-v36-one-shot" - : "locked-plate-tight-muted-target-placement-hint-v13" + ? "same-name-locked-source-strong-erasure-semantic-target-map-v37-one-shot" + : "locked-plate-strong-erasure-semantic-target-map-v14" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, From 22b007c2d905a27658c5e88cd2e6a4f0c85dfb46 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 16:44:09 +0800 Subject: [PATCH 09/73] fix: avoid old crop reference for scale adjust --- app/api/canvas/local-edit/route.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index cd92370c2..afe8b9042 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1665,15 +1665,7 @@ export async function POST(request: NextRequest) { annotatedFilename, toArrayBuffer(annotatedBuffer) ); - const scaleAdjustCropReferences = - isScaleAdjustOperation - ? await createScaleAdjustCropReferences({ - regions, - sourceBuffer, - sourceHeight, - sourceWidth, - }) - : []; + const scaleAdjustCropReferences: ScaleAdjustCropReference[] = []; const prompt = isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? buildScaleAdjustCoordinatePrompt( @@ -1791,8 +1783,8 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-locked-source-strong-erasure-semantic-target-map-v37-one-shot" - : "locked-plate-strong-erasure-semantic-target-map-v14" + ? "same-name-locked-source-semantic-target-map-no-old-crop-v38-one-shot" + : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, From 6f0fb9fe4bd370c83cecb95769a066054ed0ec8d Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 16:56:25 +0800 Subject: [PATCH 10/73] fix: use coordinate map for scale adjust --- app/api/canvas/local-edit/route.ts | 47 ++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index afe8b9042..237214422 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1438,11 +1438,11 @@ function buildScaleAdjustCoordinatePrompt( "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具必须消失并补成自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；如果最终看起来和原图一样，就是失败。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a full-frame semantic edit map with the same canvas as input image 1, not output content and not a photo layer. It keeps the original room frame visible as the locked reference, strongly fades the OLD source footprint to mean remove/heal the old full-size furniture, and shows the same furniture at the required TARGET footprint. Read only the final furniture footprint, center, and locked frame relationship from input image 2. If any original-size furniture is still faintly visible in the faded OLD area, treat it as removed in the final; the final must contain only the TARGET-size furniture. Never copy faded wash, placeholder shadow, rectangular crop edges, rough patch quality, seams, or guide artifacts into the output.", + "Input image 2 is a neutral coordinate instruction map, not a photo and not output content. The OLD box/label locates the current furniture in input image 1; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE = BACKGROUND, the old full-size furniture outside TARGET must disappear and be healed as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics.", cropLines ? `${cropLines}` : "", "", "Task:", - "Find the main movable furniture subject inside each OLD box in input image 1. Resize and, if needed, move that subject only so its final visible footprint matches the target layout shown in input image 2 and the numeric TARGET coordinates below.", + "Find the main movable furniture subject inside each OLD box in input image 1. Resize and, if needed, move that subject only so its final visible footprint matches the TARGET box shown in input image 2 and the numeric TARGET coordinates below.", "Do a real subject-scale edit, not a full image zoom. The camera, room layout, windows, wall decor, text, floor, rug, and all non-target furniture must remain stable.", "If an OLD box contains more than one object, choose the dominant furniture/product inside that box, usually the largest or most central movable item. Keep adjacent lamps, plants, side tables, pillows, wall decor, windows, text, and background objects at their original size unless they are clearly part of the same furniture subject.", "Keep the furniture identity, material, perspective, objects on top, and natural aspect ratio from input image 1. Do not create a second copy or extra parts. Do not preserve the original furniture size just because the scene looks realistic; the user explicitly wants a visible scale change.", @@ -1454,8 +1454,8 @@ function buildScaleAdjustCoordinatePrompt( "Region requirements:", regionLines, "", - "Success check: compare the final furniture footprint with input image 1 and the full-frame semantic edit map in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", - "Do not paste rectangular image areas. Do not output layout-reference overlays, muted wash, coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", + "Success check: compare the final furniture footprint with input image 1 and the neutral coordinate map in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", + "Do not paste rectangular image areas. Do not output coordinate-map lines, labels, boxes, arrows, scale text, muted wash, overlays, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", "Return only the clean edited photo.", ].join("\n"); } @@ -1463,7 +1463,7 @@ function buildScaleAdjustCoordinatePrompt( function buildScaleAdjustCoordinateSystemPrompt() { return [ "You are an image editing model for local furniture scale adjustment.", - "Input image 1 is the only final photo base. Input image 2 is a full-frame semantic edit map, not a photo layer.", + "Input image 1 is the only final photo base. Input image 2 is a neutral coordinate instruction map, not a photo layer.", "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding unchanged furniture and full-canvas zoom.", "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is a failed edit.", @@ -1638,13 +1638,25 @@ export async function POST(request: NextRequest) { resolvedScaleAdjustModelEntry ); const annotatedBuffer = isScaleAdjustOperation - ? await buildScaleAdjustVisualLayoutGuideBuffer({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceWidth, - }) + ? useScaleAdjustCoordinateGuide + ? await sharp( + buildScaleAdjustAnnotationSvg({ + height: sourceHeight, + regions, + scalePercent, + width: sourceWidth, + }), + { failOn: "none" } + ) + .png({ compressionLevel: 6 }) + .toBuffer() + : await buildScaleAdjustVisualLayoutGuideBuffer({ + regions, + scalePercent, + sourceBuffer, + sourceHeight, + sourceWidth, + }) : await sharp(sourceBuffer, { failOn: "none" }) .composite([ { @@ -1739,7 +1751,9 @@ export async function POST(request: NextRequest) { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation - ? "scale-adjust-full-frame-semantic-edit-map" + ? useScaleAdjustCoordinateGuide + ? "scale-adjust-neutral-coordinate-map" + : "scale-adjust-full-frame-semantic-edit-map" : "source-overlay", cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, @@ -1754,7 +1768,10 @@ export async function POST(request: NextRequest) { resolvedScaleAdjustModelEntry?.provider?.type || null, scaleAdjustModelFamily: scaleAdjustModelFamily || null, scaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide, - scaleAdjustVisualLayoutGuideApplied: isScaleAdjustOperation, + scaleAdjustNeutralCoordinateMapApplied: + isScaleAdjustOperation && useScaleAdjustCoordinateGuide, + scaleAdjustVisualLayoutGuideApplied: + isScaleAdjustOperation && !useScaleAdjustCoordinateGuide, falScaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image", @@ -1783,7 +1800,7 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-locked-source-semantic-target-map-no-old-crop-v38-one-shot" + ? "same-name-neutral-coordinate-map-no-old-crop-v39-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: false, From ffd13c0f314925148df9c7252d19527026b9fbee Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 17:01:43 +0800 Subject: [PATCH 11/73] fix: keep scale adjust single model attempt --- app/api/canvas/local-edit/route.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 237214422..0432bb999 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1743,6 +1743,7 @@ export async function POST(request: NextRequest) { } : undefined; const imageRequest: CanvasGenerateImageRequest = { + allowFailover: isScaleAdjustOperation ? false : undefined, aspectRatio: resolvedAspectRatio, billingFeature: "repaint", debugLabel: @@ -1776,17 +1777,15 @@ export async function POST(request: NextRequest) { useScaleAdjustCoordinateGuide && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image", scaleAdjustProviderFallbackPolicy: isScaleAdjustOperation - ? scaleAdjustModelFamily - ? "same-name-service-error-only-fal-last" - : "default-model-failover" + ? "disabled-one-shot-single-model" : undefined, scaleAdjustQualityFailoverPolicy: isScaleAdjustOperation ? "disabled-reject-unchanged-ai-candidate" : undefined, scaleAdjustPreferSameNameProviderFailover: - isScaleAdjustOperation && Boolean(scaleAdjustModelFamily), + false, scaleAdjustStrictSameFamilyFailover: - isScaleAdjustOperation && Boolean(scaleAdjustModelFamily), + false, manualRegionReferenceImageCount: manualRegionReferenceImageUrls.length, regionReferenceImageCount: referenceImages.length, From b3b4fa8ed28e26b4b1f166eb19d6298b5476afb2 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 17:14:06 +0800 Subject: [PATCH 12/73] fix: enlarge scale adjust coordinate map --- app/api/canvas/local-edit/route.ts | 130 ++++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 4 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 0432bb999..bd13e2a1c 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1249,6 +1249,128 @@ function buildScaleAdjustAnnotationSvg(params: { `); } +function buildScaleAdjustZoomedCoordinateSvg(params: { + width: number; + height: number; + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; +}) { + const { width, height, regions, scalePercent } = params; + const minSide = Math.max(1, Math.min(width, height)); + const strokeWidth = Math.max(3, Math.round(minSide * 0.006)); + const fontSize = Math.max(22, Math.round(minSide * 0.026)); + const smallFontSize = Math.max(18, Math.round(fontSize * 0.72)); + const panelMarginX = Math.round(width * 0.1); + const panelMarginY = Math.round(height * 0.12); + const panelLeft = panelMarginX; + const panelTop = panelMarginY; + const panelWidth = Math.max(1, width - panelMarginX * 2); + const panelHeight = Math.max(1, height - panelMarginY * 2); + const sourceColor = "#9ca3af"; + const targetColor = "#111827"; + const helperColor = "#4b5563"; + const padX = Math.round(fontSize * 0.42); + const labelHeight = Math.round(fontSize * 1.35); + const label = (text: string, x: number, y: number, color: string) => { + const labelWidth = Math.min( + Math.round(text.length * fontSize * 0.58 + padX * 2), + Math.max(160, width - 16) + ); + const left = clamp(x, 8, Math.max(8, width - labelWidth - 8)); + const top = clamp(y, 8, Math.max(8, height - labelHeight - 8)); + return ` + + ${escapeXml(text)} + `; + }; + + const regionSvg = regions + .map((region) => { + const source = region.pixelRect; + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const target = + region.targetPixelRect || rectToPixels(targetRect, width, height); + const sourceRight = source.left + source.width; + const sourceBottom = source.top + source.height; + const targetRight = target.left + target.width; + const targetBottom = target.top + target.height; + const unionLeft = Math.min(source.left, target.left); + const unionTop = Math.min(source.top, target.top); + const unionRight = Math.max(sourceRight, targetRight); + const unionBottom = Math.max(sourceBottom, targetBottom); + const marginX = Math.max(12, Math.round((unionRight - unionLeft) * 0.35)); + const marginY = Math.max(12, Math.round((unionBottom - unionTop) * 0.35)); + const viewLeft = unionLeft - marginX; + const viewTop = unionTop - marginY; + const viewWidth = Math.max(1, unionRight - unionLeft + marginX * 2); + const viewHeight = Math.max(1, unionBottom - unionTop + marginY * 2); + const scale = Math.min( + panelWidth / viewWidth, + panelHeight / viewHeight + ); + const drawnWidth = viewWidth * scale; + const drawnHeight = viewHeight * scale; + const originX = panelLeft + (panelWidth - drawnWidth) / 2 - viewLeft * scale; + const originY = panelTop + (panelHeight - drawnHeight) / 2 - viewTop * scale; + const project = (box: { left: number; top: number; width: number; height: number }) => ({ + left: Math.round(originX + box.left * scale), + top: Math.round(originY + box.top * scale), + width: Math.max(1, Math.round(box.width * scale)), + height: Math.max(1, Math.round(box.height * scale)), + }); + const sourceBox = project(source); + const targetBox = project(target); + const sourceCenterX = sourceBox.left + sourceBox.width / 2; + const sourceCenterY = sourceBox.top + sourceBox.height / 2; + const targetCenterX = targetBox.left + targetBox.width / 2; + const targetCenterY = targetBox.top + targetBox.height / 2; + const widthFactor = source.width > 0 ? target.width / source.width : 1; + const heightFactor = source.height > 0 ? target.height / source.height : 1; + const actionLabel = + widthFactor < 0.92 || heightFactor < 0.92 + ? "SHRINK" + : widthFactor > 1.08 || heightFactor > 1.08 + ? "ENLARGE" + : "MOVE"; + const scaleLabel = `${actionLabel} ${widthFactor.toFixed(2)}W ${heightFactor.toFixed(2)}H`; + const radius = Math.max(10, Math.round(strokeWidth * 2.2)); + const centerMoved = + Math.hypot(targetCenterX - sourceCenterX, targetCenterY - sourceCenterY) > + Math.max(10, minSide * 0.018); + + return ` + + + + + ${ + centerMoved + ? `` + : "" + } + ${label(`OLD ${region.index}`, sourceBox.left, sourceBox.top - labelHeight - strokeWidth, sourceColor)} + ${label(`TARGET ${region.index}`, targetBox.left, targetBox.top + strokeWidth, targetColor)} + ${label(scaleLabel, targetBox.left, targetBox.top + targetBox.height + strokeWidth * 2, helperColor)} + ${label("OLD OUTSIDE = BACKGROUND", sourceBox.left, sourceBox.top + sourceBox.height + strokeWidth * 4 + labelHeight, helperColor)} + `; + }) + .join(""); + + return Buffer.from(` + + + + + + + + + LOCAL SCALE MAP - USE PROMPT PIXELS FOR ABSOLUTE POSITION + ${regionSvg} + + `); +} + function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { const regionLines = regions .map((region) => { @@ -1438,7 +1560,7 @@ function buildScaleAdjustCoordinatePrompt( "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具必须消失并补成自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；如果最终看起来和原图一样，就是失败。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a neutral coordinate instruction map, not a photo and not output content. The OLD box/label locates the current furniture in input image 1; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE = BACKGROUND, the old full-size furniture outside TARGET must disappear and be healed as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics.", + "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE = BACKGROUND, the old full-size furniture outside TARGET must disappear and be healed as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics.", cropLines ? `${cropLines}` : "", "", "Task:", @@ -1463,7 +1585,7 @@ function buildScaleAdjustCoordinatePrompt( function buildScaleAdjustCoordinateSystemPrompt() { return [ "You are an image editing model for local furniture scale adjustment.", - "Input image 1 is the only final photo base. Input image 2 is a neutral coordinate instruction map, not a photo layer.", + "Input image 1 is the only final photo base. Input image 2 is a zoomed neutral coordinate instruction map, not a photo layer.", "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding unchanged furniture and full-canvas zoom.", "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is a failed edit.", @@ -1640,7 +1762,7 @@ export async function POST(request: NextRequest) { const annotatedBuffer = isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? await sharp( - buildScaleAdjustAnnotationSvg({ + buildScaleAdjustZoomedCoordinateSvg({ height: sourceHeight, regions, scalePercent, @@ -1799,7 +1921,7 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-neutral-coordinate-map-no-old-crop-v39-one-shot" + ? "same-name-zoomed-coordinate-map-no-old-crop-v41-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: false, From f5a318beef60c7fdba8a66ac92983db591bb3b23 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 17:25:47 +0800 Subject: [PATCH 13/73] fix: neutralize scale adjust coordinate prompt --- app/api/canvas/local-edit/route.ts | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index bd13e2a1c..3f5d5d536 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1207,7 +1207,7 @@ function buildScaleAdjustAnnotationSvg(params: { const actionLabel = isShrink ? "SHRINK" : isEnlarge ? "ENLARGE" : "MOVE"; const scaleLabel = `${actionLabel} ${widthFactor.toFixed(2)}W ${heightFactor.toFixed(2)}H`; const instructionLabel = isShrink - ? "OLD OUTSIDE = BACKGROUND" + ? "OLD OUTSIDE -> BG" : isEnlarge ? "TARGET = FINAL FOOTPRINT" : "FINAL SUBJECT CENTER"; @@ -1351,7 +1351,7 @@ function buildScaleAdjustZoomedCoordinateSvg(params: { ${label(`OLD ${region.index}`, sourceBox.left, sourceBox.top - labelHeight - strokeWidth, sourceColor)} ${label(`TARGET ${region.index}`, targetBox.left, targetBox.top + strokeWidth, targetColor)} ${label(scaleLabel, targetBox.left, targetBox.top + targetBox.height + strokeWidth * 2, helperColor)} - ${label("OLD OUTSIDE = BACKGROUND", sourceBox.left, sourceBox.top + sourceBox.height + strokeWidth * 4 + labelHeight, helperColor)} + ${label("OLD OUTSIDE -> BG", sourceBox.left, sourceBox.top + sourceBox.height + strokeWidth * 4 + labelHeight, helperColor)} `; }) .join(""); @@ -1471,7 +1471,7 @@ function buildScaleAdjustPrompt( "这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。", sourceSizeLine, "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", - "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面编辑意图图，不是输出底图。它保留原图完整画幅作为稳定参考，并把源框旧主体占位强淡化为“这里的原尺寸主体需要消失并自然修复”的指令；同时把同一目标主体粗略缩放后放在用户希望的最终位置和大小。请只读取目标主体的最终视觉占位、中心位置和原画幅关系；输入图片2里被淡化的旧主体不能表示最终还要保留原尺寸主体。不得学习、复刻或输出淡化痕迹、阴影占位、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", + "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面编辑意图图，不是输出底图。它保留原图完整画幅作为稳定参考，并把源框旧主体占位强淡化为“这里的原尺寸主体需要被目标尺寸替换并自然修复背景”的指令；同时把同一目标主体粗略缩放后放在用户希望的最终位置和大小。请只读取目标主体的最终视觉占位、中心位置和原画幅关系；输入图片2里被淡化的旧主体不能表示最终还要保留原尺寸主体。不得学习、复刻或输出淡化痕迹、阴影占位、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", cropLines ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` : "", @@ -1483,13 +1483,13 @@ function buildScaleAdjustPrompt( "请不要生成任何彩色框、矩形边框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件或半透明色块。", "请同时使用下方每个区域的源框/目标框像素坐标和输入图片2的目标布局参考来理解目标占位。坐标和参考图只用于指导主体位置和大小，不是要求把某个矩形图块缩放后贴回。", "", - "任务：先在输入图片1中识别源框范围内最主要、可独立调整的主体物体，再只对这个“主体物体”做局部比例调整。调整后的主体大小和位置必须接近目标框表示的视觉占位；如果输出看起来和原图一样，就是失败。", + "任务：先在输入图片1中识别源框范围内最主要、可独立调整的主体物体，再只对这个“主体物体”做局部比例调整。调整后的主体大小和位置必须接近目标框表示的视觉占位；不接受看起来和原图几乎相同的输出。", regionLines, "", "关键要求：", "1. 只调整目标主体物体本身的整体视觉比例，不得保留原尺寸；主体轮廓宽度和高度必须同步等比变化，宽高比必须与原主体一致，不能为了贴合目标框而拉宽、压窄、拉高或压扁。", "2. 严禁把整个矩形框选区域、输入图片1整张图或任意裁切图当成图片块整体缩放、平移、复制、裁切或贴回；最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。", - "2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图；如果画面中的标题、Logo、页面文字、边框、窗框、背景家具或非目标主体发生整体位移/缩放/消失，必须视为失败并重新生成。请把这些元素当作像素锚点，它们的位置和比例不能跟着目标主体变化。", + "2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图；画面中的标题、Logo、页面文字、边框、窗框、背景家具或非目标主体必须作为像素锚点保持原位置和原比例，不能跟着目标主体变化。", "3. 如果输入图片2中的目标框明显小于源框，目标主体应自然变小，主体缩小后露出的背景、桌面、墙面、阴影、纹理和透视必须被重新生成并无缝补全，选区内应出现明显新增留白或背景。", "4. 如果输入图片2中的目标框明显大于源框，目标主体应自然变大；允许主体自然超出用户手画的原始源框，使用周围真实背景完成边缘、阴影和遮挡衔接。宽度和高度都要同步变大，不能出现“高度变高但宽度变窄”的非等比变形；主体边缘与遮挡关系要合理，不能产生突兀裁切。", "5. 如果目标框中心与源框中心不同，允许主体同步移动到目标框中心附近；如果主体靠近画布边界导致目标空间不足，请用能完整放入画面的最大统一倍率整体调整，绝不能为了塞进画面而只压缩宽度或只拉伸高度。", @@ -1544,7 +1544,7 @@ function buildScaleAdjustCoordinatePrompt( cropIndex >= 0 ? ` Input image ${cropIndex + 3} is a clean crop reference for this same furniture identity and silhouette; use it only to recognize the subject, never paste it, and do not preserve its old size.` : ""; - return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}. The final furniture silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged furniture footprint is invalid. ${footprintRule}${cropNote}`; + return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}. The final furniture silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged furniture footprint is not acceptable. ${footprintRule}${cropNote}`; }) .join("\n"); const cropLines = cropReferences @@ -1556,11 +1556,11 @@ function buildScaleAdjustCoordinatePrompt( return [ "Local furniture scale edit. Use input image 1 as the clean original photo and the locked camera frame.", - "One-shot cost constraint: solve this in the first generation. Do not rely on later retries or alternate candidates; an unchanged result wastes the request and is invalid.", - "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具必须消失并补成自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", - "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；如果最终看起来和原图一样，就是失败。", + "One-shot cost constraint: produce the requested edit in this generation without relying on later retries or alternate candidates; an unchanged result is not acceptable.", + "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具占位必须改为自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", + "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；不接受看起来和原图几乎相同的结果。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE = BACKGROUND, the old full-size furniture outside TARGET must disappear and be healed as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics.", + "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size furniture area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics.", cropLines ? `${cropLines}` : "", "", "Task:", @@ -1569,9 +1569,9 @@ function buildScaleAdjustCoordinatePrompt( "If an OLD box contains more than one object, choose the dominant furniture/product inside that box, usually the largest or most central movable item. Keep adjacent lamps, plants, side tables, pillows, wall decor, windows, text, and background objects at their original size unless they are clearly part of the same furniture subject.", "Keep the furniture identity, material, perspective, objects on top, and natural aspect ratio from input image 1. Do not create a second copy or extra parts. Do not preserve the original furniture size just because the scene looks realistic; the user explicitly wants a visible scale change.", "For beds, sofas, tables, chairs, cabinets, benches, and other large furniture, still perform the scale edit on that furniture. Do not treat large furniture as locked architecture.", - "If the target is smaller, the furniture must visibly become smaller and be mostly contained by the TARGET footprint. The OLD area outside the TARGET footprint must no longer contain the full-size furniture; naturally heal newly exposed floor, rug, wall, bedding, shadows, and background. If the target is larger, enlarge the same furniture naturally without stretching it.", - "缩小时请优先让目标家具明显缩小，而不是为了保守而保持原样。原尺寸保留、只轻微变形、或只改变整张图构图都算失败。", - "Never keep the selected furniture at its original footprint. If the edited furniture still fills the OLD rectangle instead of the TARGET rectangle, the result is invalid.", + "If the target is smaller, the furniture must visibly become smaller and be mostly contained by the TARGET footprint. Rebuild the OLD area outside the TARGET footprint as newly exposed floor, rug, wall, bedding, shadows, and background. If the target is larger, enlarge the same furniture naturally without stretching it.", + "缩小时请优先让目标家具明显缩小，而不是为了保守而保持原样。不接受原尺寸保留、只轻微变形、或只改变整张图构图。", + "Do not keep the selected furniture at its original footprint. The edited furniture should fit the TARGET rectangle much better than the OLD rectangle.", "", "Region requirements:", regionLines, @@ -1588,8 +1588,8 @@ function buildScaleAdjustCoordinateSystemPrompt() { "Input image 1 is the only final photo base. Input image 2 is a zoomed neutral coordinate instruction map, not a photo layer.", "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding unchanged furniture and full-canvas zoom.", "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", - "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is a failed edit.", - "When shrinking, remove the old full-size furniture footprint outside the target footprint and heal it as natural background. Large furniture is editable and must not remain locked at original size.", + "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is not acceptable.", + "When shrinking, rebuild the old full-size furniture footprint outside the target footprint as natural background. Large furniture is editable and must not remain locked at original size.", "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", "Never output guide artifacts from the edit map: no faded wash, placeholder shadow, overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", @@ -1947,7 +1947,7 @@ export async function POST(request: NextRequest) { isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? scaleAdjustCoordinateNegativePrompt - : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、淡化痕迹、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、整体重绘、删除文字、清理文字、改写文字、文字模糊、Logo消失、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" + : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、淡化痕迹、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、文字位置变化、文字内容变化、文字模糊、Logo位置变化、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, From 291597a69fe3a30138dfdfc5e49b69d6c21398ea Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 17:36:43 +0800 Subject: [PATCH 14/73] fix: use fal-safe scale adjust prompt --- app/api/canvas/local-edit/route.ts | 55 ++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 3f5d5d536..f5463c937 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1507,7 +1507,8 @@ function buildScaleAdjustCoordinatePrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, sourceSize: { width: number; height: number }, - cropReferences: ScaleAdjustCropReference[] = [] + cropReferences: ScaleAdjustCropReference[] = [], + options: { falSafe?: boolean } = {} ) { const regionLines = regions .map((region) => { @@ -1542,8 +1543,25 @@ function buildScaleAdjustCoordinatePrompt( const cropNote = cropIndex >= 0 - ? ` Input image ${cropIndex + 3} is a clean crop reference for this same furniture identity and silhouette; use it only to recognize the subject, never paste it, and do not preserve its old size.` + ? options.falSafe + ? ` Input image ${cropIndex + 3} is an extra identity and silhouette reference.` + : ` Input image ${cropIndex + 3} is a clean crop reference for this same furniture identity and silhouette; use it only to recognize the subject, never paste it, and do not preserve its old size.` : ""; + if (options.falSafe) { + const actionPhrase = + isShrink + ? "resize the furniture proportionally smaller and place it inside the final footprint" + : isEnlarge + ? "resize the same furniture proportionally larger toward the final footprint" + : "place the same furniture subject on the final footprint"; + const blendPhrase = + isShrink + ? "Blend the newly visible floor, rug, wall, window area, shadow, and background around it naturally." + : isEnlarge + ? "Blend edges, contact shadows, and occlusion with the surrounding room naturally." + : "Keep the surrounding room stable."; + return `Region ${region.index}: current furniture box in source photo left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; requested final footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; scale factor width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. Edit direction: ${actionPhrase}. ${blendPhrase}${cropNote}`; + } return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}. The final furniture silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged furniture footprint is not acceptable. ${footprintRule}${cropNote}`; }) .join("\n"); @@ -1554,6 +1572,23 @@ function buildScaleAdjustCoordinatePrompt( ) .join("\n"); + if (options.falSafe) { + return [ + "Interior furniture photo edit.", + `Return one realistic photo with the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, + "Input image 1 is the source photo. Input image 2 is a coordinate guide only and should not appear in the final photo.", + "The guide labels OLD as the current furniture box and TARGET as the requested final footprint.", + "Resize only the furniture subject indicated by each current box. Keep the camera, room layout, walls, windows, floor, rug, sofa, lamp, text, and background stable.", + "Preserve the furniture identity, material, perspective, objects on top, and natural aspect ratio.", + "For a smaller final footprint, show the furniture smaller and blend newly visible floor, rug, wall, window area, shadow, and background around it naturally. For a larger final footprint, scale the same furniture proportionally with natural edges and contact shadows.", + "Return a clean photo without guide marks, labels, boxes, arrows, overlays, patch edges, color mismatch, or full-frame zoom.", + cropLines ? `${cropLines}` : "", + "", + "Region instructions:", + regionLines, + ].join("\n"); + } + return [ "Local furniture scale edit. Use input image 1 as the clean original photo and the locked camera frame.", "One-shot cost constraint: produce the requested edit in this generation without relying on later retries or alternate candidates; an unchanged result is not acceptable.", @@ -1759,6 +1794,9 @@ export async function POST(request: NextRequest) { const scaleAdjustModelFamily = getScaleAdjustModelFamily( resolvedScaleAdjustModelEntry ); + const useFalScaleAdjustCoordinatePrompt = + useScaleAdjustCoordinateGuide && + resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; const annotatedBuffer = isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? await sharp( @@ -1809,7 +1847,8 @@ export async function POST(request: NextRequest) { width: sourceWidth, height: sourceHeight, }, - scaleAdjustCropReferences + scaleAdjustCropReferences, + { falSafe: useFalScaleAdjustCoordinatePrompt } ) : buildScaleAdjustPrompt( regions, @@ -1818,8 +1857,9 @@ export async function POST(request: NextRequest) { { width: sourceWidth, height: sourceHeight } ) : buildLocalEditPrompt(regions); - const scaleAdjustCoordinateNegativePrompt = - "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, old-size target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; + const scaleAdjustCoordinateNegativePrompt = useFalScaleAdjustCoordinatePrompt + ? "guide marks, labels, boxes, arrows, overlays, patch edges, color mismatch, full-frame zoom, duplicate furniture, stretched furniture, distorted furniture" + : "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, old-size target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -1898,6 +1938,7 @@ export async function POST(request: NextRequest) { falScaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image", + falScaleAdjustSafePromptApplied: useFalScaleAdjustCoordinatePrompt, scaleAdjustProviderFallbackPolicy: isScaleAdjustOperation ? "disabled-one-shot-single-model" : undefined, @@ -1921,7 +1962,9 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "same-name-zoomed-coordinate-map-no-old-crop-v41-one-shot" + ? useFalScaleAdjustCoordinatePrompt + ? "fal-safe-zoomed-coordinate-map-no-old-crop-v42-one-shot" + : "same-name-zoomed-coordinate-map-no-old-crop-v42-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: false, From 2cccac34fff6e8a62b08145b00c4aed2696561d8 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 17:49:03 +0800 Subject: [PATCH 15/73] fix: guide fal scale adjust with marked photo --- app/api/canvas/local-edit/route.ts | 115 ++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 12 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index f5463c937..b21461da7 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1371,6 +1371,84 @@ function buildScaleAdjustZoomedCoordinateSvg(params: { `); } +function buildScaleAdjustFalPhotoGuideSvg(params: { + width: number; + height: number; + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; +}) { + const { width, height, regions, scalePercent } = params; + const minSide = Math.max(1, Math.min(width, height)); + const strokeWidth = Math.max(3, Math.round(minSide * 0.005)); + const fontSize = Math.max(16, Math.round(minSide * 0.022)); + const labelPadX = Math.round(fontSize * 0.48); + const labelHeight = Math.round(fontSize * 1.35); + const currentColor = "#00b8d4"; + const finalColor = "#2563eb"; + const label = (text: string, x: number, y: number, color: string) => { + const labelWidth = Math.round(text.length * fontSize * 0.74 + labelPadX * 2); + const left = clamp(x, 6, Math.max(6, width - labelWidth - 6)); + const top = clamp(y, 6, Math.max(6, height - labelHeight - 6)); + return ` + + ${escapeXml(text)} + `; + }; + + const regionSvg = regions + .map((region) => { + const source = region.pixelRect; + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const target = + region.targetPixelRect || rectToPixels(targetRect, width, height); + const radius = Math.max(8, Math.round(strokeWidth * 2)); + return ` + + + + + ${label(`CURRENT ${region.index}`, source.left + strokeWidth, source.top + strokeWidth, currentColor)} + ${label(`FINAL ${region.index}`, target.left + strokeWidth, target.top + strokeWidth, finalColor)} + `; + }) + .join(""); + + return Buffer.from(` + + + ${regionSvg} + + `); +} + +async function buildScaleAdjustFalPhotoGuideBuffer(params: { + sourceBuffer: Buffer; + width: number; + height: number; + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; +}) { + const { sourceBuffer, width, height, regions, scalePercent } = params; + return sharp(sourceBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "cover", position: "center" }) + .modulate({ brightness: 1.04, saturation: 0.45 }) + .composite([ + { + input: buildScaleAdjustFalPhotoGuideSvg({ + height, + regions, + scalePercent, + width, + }), + left: 0, + top: 0, + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { const regionLines = regions .map((region) => { @@ -1576,8 +1654,8 @@ function buildScaleAdjustCoordinatePrompt( return [ "Interior furniture photo edit.", `Return one realistic photo with the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 1 is the source photo. Input image 2 is a coordinate guide only and should not appear in the final photo.", - "The guide labels OLD as the current furniture box and TARGET as the requested final footprint.", + "Input image 1 is the source photo. Input image 2 is a marked copy of the source photo for location guidance only and should not appear in the final photo.", + "The guide labels CURRENT as the existing furniture box and FINAL as the requested final footprint.", "Resize only the furniture subject indicated by each current box. Keep the camera, room layout, walls, windows, floor, rug, sofa, lamp, text, and background stable.", "Preserve the furniture identity, material, perspective, objects on top, and natural aspect ratio.", "For a smaller final footprint, show the furniture smaller and blend newly visible floor, rug, wall, window area, shadow, and background around it naturally. For a larger final footprint, scale the same furniture proportionally with natural edges and contact shadows.", @@ -1799,17 +1877,25 @@ export async function POST(request: NextRequest) { resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; const annotatedBuffer = isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? await sharp( - buildScaleAdjustZoomedCoordinateSvg({ + ? useFalScaleAdjustCoordinatePrompt + ? await buildScaleAdjustFalPhotoGuideBuffer({ height: sourceHeight, regions, scalePercent, + sourceBuffer, width: sourceWidth, - }), - { failOn: "none" } - ) - .png({ compressionLevel: 6 }) - .toBuffer() + }) + : await sharp( + buildScaleAdjustZoomedCoordinateSvg({ + height: sourceHeight, + regions, + scalePercent, + width: sourceWidth, + }), + { failOn: "none" } + ) + .png({ compressionLevel: 6 }) + .toBuffer() : await buildScaleAdjustVisualLayoutGuideBuffer({ regions, scalePercent, @@ -1915,7 +2001,9 @@ export async function POST(request: NextRequest) { annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? "scale-adjust-neutral-coordinate-map" + ? useFalScaleAdjustCoordinatePrompt + ? "scale-adjust-fal-marked-photo-guide" + : "scale-adjust-neutral-coordinate-map" : "scale-adjust-full-frame-semantic-edit-map" : "source-overlay", cleanSourceReferenceIndex: 1, @@ -1932,7 +2020,10 @@ export async function POST(request: NextRequest) { scaleAdjustModelFamily: scaleAdjustModelFamily || null, scaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide, scaleAdjustNeutralCoordinateMapApplied: - isScaleAdjustOperation && useScaleAdjustCoordinateGuide, + isScaleAdjustOperation && + useScaleAdjustCoordinateGuide && + !useFalScaleAdjustCoordinatePrompt, + scaleAdjustFalMarkedPhotoGuideApplied: useFalScaleAdjustCoordinatePrompt, scaleAdjustVisualLayoutGuideApplied: isScaleAdjustOperation && !useScaleAdjustCoordinateGuide, falScaleAdjustCoordinateGuideApplied: @@ -1963,7 +2054,7 @@ export async function POST(request: NextRequest) { scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? useFalScaleAdjustCoordinatePrompt - ? "fal-safe-zoomed-coordinate-map-no-old-crop-v42-one-shot" + ? "fal-safe-marked-photo-guide-no-old-crop-v43-one-shot" : "same-name-zoomed-coordinate-map-no-old-crop-v42-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, From ee403bff11b83bc4963776ff2a7d05b97b6f453d Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 17:55:57 +0800 Subject: [PATCH 16/73] fix: simplify fal scale adjust guide marks --- app/api/canvas/local-edit/route.ts | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index b21461da7..9a0704f8a 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1380,20 +1380,8 @@ function buildScaleAdjustFalPhotoGuideSvg(params: { const { width, height, regions, scalePercent } = params; const minSide = Math.max(1, Math.min(width, height)); const strokeWidth = Math.max(3, Math.round(minSide * 0.005)); - const fontSize = Math.max(16, Math.round(minSide * 0.022)); - const labelPadX = Math.round(fontSize * 0.48); - const labelHeight = Math.round(fontSize * 1.35); const currentColor = "#00b8d4"; const finalColor = "#2563eb"; - const label = (text: string, x: number, y: number, color: string) => { - const labelWidth = Math.round(text.length * fontSize * 0.74 + labelPadX * 2); - const left = clamp(x, 6, Math.max(6, width - labelWidth - 6)); - const top = clamp(y, 6, Math.max(6, height - labelHeight - 6)); - return ` - - ${escapeXml(text)} - `; - }; const regionSvg = regions .map((region) => { @@ -1407,8 +1395,6 @@ function buildScaleAdjustFalPhotoGuideSvg(params: { - ${label(`CURRENT ${region.index}`, source.left + strokeWidth, source.top + strokeWidth, currentColor)} - ${label(`FINAL ${region.index}`, target.left + strokeWidth, target.top + strokeWidth, finalColor)} `; }) .join(""); @@ -1655,11 +1641,11 @@ function buildScaleAdjustCoordinatePrompt( "Interior furniture photo edit.", `Return one realistic photo with the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, "Input image 1 is the source photo. Input image 2 is a marked copy of the source photo for location guidance only and should not appear in the final photo.", - "The guide labels CURRENT as the existing furniture box and FINAL as the requested final footprint.", + "In the guide, the cyan dashed box marks the existing furniture area and the blue solid box marks the requested final footprint.", "Resize only the furniture subject indicated by each current box. Keep the camera, room layout, walls, windows, floor, rug, sofa, lamp, text, and background stable.", "Preserve the furniture identity, material, perspective, objects on top, and natural aspect ratio.", "For a smaller final footprint, show the furniture smaller and blend newly visible floor, rug, wall, window area, shadow, and background around it naturally. For a larger final footprint, scale the same furniture proportionally with natural edges and contact shadows.", - "Return a clean photo without guide marks, labels, boxes, arrows, overlays, patch edges, color mismatch, or full-frame zoom.", + "Return a clean photo without guide marks, boxes, arrows, overlays, patch edges, color mismatch, or full-frame zoom.", cropLines ? `${cropLines}` : "", "", "Region instructions:", @@ -2054,7 +2040,7 @@ export async function POST(request: NextRequest) { scaleAdjustPromptStrategy: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? useFalScaleAdjustCoordinatePrompt - ? "fal-safe-marked-photo-guide-no-old-crop-v43-one-shot" + ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" : "same-name-zoomed-coordinate-map-no-old-crop-v42-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, From a8c5c146566c8e8e53030f4b3e2aec541caa5d4d Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 18:04:51 +0800 Subject: [PATCH 17/73] fix: use marked photo guide for scale adjust models --- app/api/canvas/local-edit/route.ts | 33 +++++++++++++++++++----------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 9a0704f8a..5d9b4f68a 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1371,7 +1371,7 @@ function buildScaleAdjustZoomedCoordinateSvg(params: { `); } -function buildScaleAdjustFalPhotoGuideSvg(params: { +function buildScaleAdjustMarkedPhotoGuideSvg(params: { width: number; height: number; regions: NormalizedLocalEditRegion[]; @@ -1407,7 +1407,7 @@ function buildScaleAdjustFalPhotoGuideSvg(params: { `); } -async function buildScaleAdjustFalPhotoGuideBuffer(params: { +async function buildScaleAdjustMarkedPhotoGuideBuffer(params: { sourceBuffer: Buffer; width: number; height: number; @@ -1421,7 +1421,7 @@ async function buildScaleAdjustFalPhotoGuideBuffer(params: { .modulate({ brightness: 1.04, saturation: 0.45 }) .composite([ { - input: buildScaleAdjustFalPhotoGuideSvg({ + input: buildScaleAdjustMarkedPhotoGuideSvg({ height, regions, scalePercent, @@ -1572,7 +1572,7 @@ function buildScaleAdjustCoordinatePrompt( scalePercent: ScaleAdjustPercent | null | undefined, sourceSize: { width: number; height: number }, cropReferences: ScaleAdjustCropReference[] = [], - options: { falSafe?: boolean } = {} + options: { falSafe?: boolean; markedPhotoGuide?: boolean } = {} ) { const regionLines = regions .map((region) => { @@ -1653,13 +1653,17 @@ function buildScaleAdjustCoordinatePrompt( ].join("\n"); } + const coordinateGuideLine = options.markedPhotoGuide + ? "输入图片2是一张原图浅化后的完整画面位置参考图，不是输出底图。青色虚线框表示当前主体搜索框（对应下方 OLD），蓝色实线框表示目标最终占位（对应下方 TARGET）。请用输入图片2理解主体在真实室内画面中的位置关系，但最终图不能出现任何框线、色块、标注或参考图的浅化效果。" + : "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size furniture area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics."; + return [ "Local furniture scale edit. Use input image 1 as the clean original photo and the locked camera frame.", "One-shot cost constraint: produce the requested edit in this generation without relying on later retries or alternate candidates; an unchanged result is not acceptable.", "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具占位必须改为自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；不接受看起来和原图几乎相同的结果。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size furniture area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics.", + coordinateGuideLine, cropLines ? `${cropLines}` : "", "", "Task:", @@ -1861,10 +1865,11 @@ export async function POST(request: NextRequest) { const useFalScaleAdjustCoordinatePrompt = useScaleAdjustCoordinateGuide && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; + const useScaleAdjustMarkedPhotoGuide = useScaleAdjustCoordinateGuide; const annotatedBuffer = isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? useFalScaleAdjustCoordinatePrompt - ? await buildScaleAdjustFalPhotoGuideBuffer({ + ? useScaleAdjustMarkedPhotoGuide + ? await buildScaleAdjustMarkedPhotoGuideBuffer({ height: sourceHeight, regions, scalePercent, @@ -1920,7 +1925,10 @@ export async function POST(request: NextRequest) { height: sourceHeight, }, scaleAdjustCropReferences, - { falSafe: useFalScaleAdjustCoordinatePrompt } + { + falSafe: useFalScaleAdjustCoordinatePrompt, + markedPhotoGuide: useScaleAdjustMarkedPhotoGuide, + } ) : buildScaleAdjustPrompt( regions, @@ -1987,8 +1995,8 @@ export async function POST(request: NextRequest) { annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? useFalScaleAdjustCoordinatePrompt - ? "scale-adjust-fal-marked-photo-guide" + ? useScaleAdjustMarkedPhotoGuide + ? "scale-adjust-marked-photo-guide" : "scale-adjust-neutral-coordinate-map" : "scale-adjust-full-frame-semantic-edit-map" : "source-overlay", @@ -2008,7 +2016,8 @@ export async function POST(request: NextRequest) { scaleAdjustNeutralCoordinateMapApplied: isScaleAdjustOperation && useScaleAdjustCoordinateGuide && - !useFalScaleAdjustCoordinatePrompt, + !useScaleAdjustMarkedPhotoGuide, + scaleAdjustMarkedPhotoGuideApplied: useScaleAdjustMarkedPhotoGuide, scaleAdjustFalMarkedPhotoGuideApplied: useFalScaleAdjustCoordinatePrompt, scaleAdjustVisualLayoutGuideApplied: isScaleAdjustOperation && !useScaleAdjustCoordinateGuide, @@ -2041,7 +2050,7 @@ export async function POST(request: NextRequest) { ? useScaleAdjustCoordinateGuide ? useFalScaleAdjustCoordinatePrompt ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" - : "same-name-zoomed-coordinate-map-no-old-crop-v42-one-shot" + : "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: false, From 87d29607138fb7f66f93b8cf92d2f213ee094782 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 18:14:51 +0800 Subject: [PATCH 18/73] fix: include scale adjust region prompt --- app/api/canvas/local-edit/route.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 5d9b4f68a..c017dbb7c 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1591,6 +1591,10 @@ function buildScaleAdjustCoordinatePrompt( const sourceBottom = region.pixelRect.top + region.pixelRect.height; const targetRight = targetPixelRect.left + targetPixelRect.width; const targetBottom = targetPixelRect.top + targetPixelRect.height; + const regionUserInstruction = region.prompt.trim(); + const userInstructionLine = regionUserInstruction + ? ` User instruction for this region: ${regionUserInstruction}.` + : ""; const isShrink = widthFactor < 0.92 || heightFactor < 0.92; const isEnlarge = widthFactor > 1.08 || heightFactor > 1.08; const scaleDirection = @@ -1626,7 +1630,7 @@ function buildScaleAdjustCoordinatePrompt( : "Keep the surrounding room stable."; return `Region ${region.index}: current furniture box in source photo left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; requested final footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; scale factor width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. Edit direction: ${actionPhrase}. ${blendPhrase}${cropNote}`; } - return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}. The final furniture silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged furniture footprint is not acceptable. ${footprintRule}${cropNote}`; + return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}.${userInstructionLine} The final furniture silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged furniture footprint is not acceptable. ${footprintRule}${cropNote}`; }) .join("\n"); const cropLines = cropReferences From 0515fd3e462e755d817b6e3c31579a5bd133d05b Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 18:27:59 +0800 Subject: [PATCH 19/73] fix: add deterministic scale adjust fallback --- lib/canvas/generate-image.ts | 390 +++++++++++++++++++++++++++++++++++ 1 file changed, 390 insertions(+) diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 20750cab9..64247dcca 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -167,6 +167,9 @@ const SCALE_ADJUST_MARKER_CLEANUP_MARGIN = 0.035; const SCALE_ADJUST_MARKER_DILATE_RADIUS = 3; const SCALE_ADJUST_MARKER_MIN_PIXEL_COUNT = 18; const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112; +const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42; +const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18; +const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180; const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "gemini", @@ -1252,6 +1255,350 @@ async function applyScaleAdjustVerificationToImages(params: { return [selected.cleanedImage || selected.image]; } +function getRawPixelIndex(width: number, x: number, y: number) { + return (y * width + x) * 3; +} + +function getLuma(r: number, g: number, b: number) { + return r * 0.299 + g * 0.587 + b * 0.114; +} + +function dilateMask(params: { + height: number; + mask: Uint8Array; + radius: number; + width: number; +}) { + const { height, mask, radius, width } = params; + if (radius <= 0) return mask; + const output = new Uint8Array(mask.length); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + if (mask[y * width + x] === 0) continue; + const left = Math.max(0, x - radius); + const right = Math.min(width - 1, x + radius); + const top = Math.max(0, y - radius); + const bottom = Math.min(height - 1, y + radius); + for (let yy = top; yy <= bottom; yy += 1) { + for (let xx = left; xx <= right; xx += 1) { + output[yy * width + xx] = 255; + } + } + } + } + return output; +} + +function estimateScaleAdjustFallbackBackground(params: { + box: ReturnType; + raw: Buffer; + width: number; +}) { + const { box, raw, width } = params; + const samples: Array<{ b: number; g: number; luma: number; r: number }> = []; + for (let y = box.top; y < box.top + box.height; y += 1) { + for (let x = box.left; x < box.left + box.width; x += 1) { + const index = getRawPixelIndex(width, x, y); + const r = raw[index] || 0; + const g = raw[index + 1] || 0; + const b = raw[index + 2] || 0; + samples.push({ b, g, luma: getLuma(r, g, b), r }); + } + } + if (samples.length === 0) { + return { b: 245, g: 245, luma: 245, r: 245 }; + } + samples.sort((left, right) => right.luma - left.luma); + const keepCount = Math.max(16, Math.ceil(samples.length * 0.35)); + const kept = samples.slice(0, keepCount); + const totals = kept.reduce( + (acc, sample) => { + acc.r += sample.r; + acc.g += sample.g; + acc.b += sample.b; + acc.luma += sample.luma; + return acc; + }, + { b: 0, g: 0, luma: 0, r: 0 } + ); + return { + b: totals.b / kept.length, + g: totals.g / kept.length, + luma: totals.luma / kept.length, + r: totals.r / kept.length, + }; +} + +function buildScaleAdjustFallbackSubjectMask(params: { + box: ReturnType; + raw: Buffer; + width: number; +}) { + const { box, raw, width } = params; + const background = estimateScaleAdjustFallbackBackground(params); + const mask = new Uint8Array(box.width * box.height); + for (let localY = 0; localY < box.height; localY += 1) { + const y = box.top + localY; + for (let localX = 0; localX < box.width; localX += 1) { + if (localX >= box.width * 0.94) continue; + const x = box.left + localX; + const rawIndex = getRawPixelIndex(width, x, y); + const r = raw[rawIndex] || 0; + const g = raw[rawIndex + 1] || 0; + const b = raw[rawIndex + 2] || 0; + const luma = getLuma(r, g, b); + const colorDistance = Math.sqrt( + (r - background.r) ** 2 + + (g - background.g) ** 2 + + (b - background.b) ** 2 + ); + if ( + (colorDistance >= SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD && + luma <= background.luma - SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP) || + luma <= Math.min(118, background.luma - 55) + ) { + mask[localY * box.width + localX] = 255; + } + } + } + return dilateMask({ + height: box.height, + mask, + radius: Math.max(1, Math.round(Math.min(box.width, box.height) * 0.018)), + width: box.width, + }); +} + +function inpaintScaleAdjustMaskedRegion(params: { + background: { b: number; g: number; r: number }; + box: ReturnType; + mask: Uint8Array; + raw: Buffer; + width: number; +}) { + const { background, box, mask, raw, width } = params; + const working = Buffer.from(raw); + const pending = new Uint8Array(mask); + let remaining = pending.reduce((sum, value) => sum + (value ? 1 : 0), 0); + const offsets = [ + [-1, 0], + [1, 0], + [0, -1], + [0, 1], + [-1, -1], + [1, -1], + [-1, 1], + [1, 1], + ]; + + for ( + let iteration = 0; + iteration < SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS && remaining > 0; + iteration += 1 + ) { + const nextValues: Array<{ b: number; g: number; index: number; r: number }> = []; + for (let localY = 0; localY < box.height; localY += 1) { + for (let localX = 0; localX < box.width; localX += 1) { + const maskIndex = localY * box.width + localX; + if (pending[maskIndex] === 0) continue; + let count = 0; + let totalB = 0; + let totalG = 0; + let totalR = 0; + for (const [offsetX, offsetY] of offsets) { + const nx = localX + offsetX; + const ny = localY + offsetY; + if (nx < 0 || ny < 0 || nx >= box.width || ny >= box.height) continue; + if (pending[ny * box.width + nx] !== 0) continue; + const rawIndex = getRawPixelIndex(width, box.left + nx, box.top + ny); + totalR += working[rawIndex] || 0; + totalG += working[rawIndex + 1] || 0; + totalB += working[rawIndex + 2] || 0; + count += 1; + } + if (count > 0) { + nextValues.push({ + b: Math.round(totalB / count), + g: Math.round(totalG / count), + index: maskIndex, + r: Math.round(totalR / count), + }); + } + } + } + if (nextValues.length === 0) break; + for (const value of nextValues) { + const localX = value.index % box.width; + const localY = Math.floor(value.index / box.width); + const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY); + working[rawIndex] = value.r; + working[rawIndex + 1] = value.g; + working[rawIndex + 2] = value.b; + pending[value.index] = 0; + remaining -= 1; + } + } + + if (remaining > 0) { + for (let localY = 0; localY < box.height; localY += 1) { + for (let localX = 0; localX < box.width; localX += 1) { + const maskIndex = localY * box.width + localX; + if (pending[maskIndex] === 0) continue; + const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY); + working[rawIndex] = Math.round(background.r); + working[rawIndex + 1] = Math.round(background.g); + working[rawIndex + 2] = Math.round(background.b); + } + } + } + + return working; +} + +async function buildScaleAdjustFallbackObjectPng(params: { + box: ReturnType; + mask: Uint8Array; + sourceBuffer: Buffer; + targetBox: ReturnType; +}) { + const { box, mask, sourceBuffer, targetBox } = params; + const cropBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .extract(box) + .removeAlpha() + .png({ compressionLevel: 6 }) + .toBuffer(); + const alphaBuffer = await sharp(mask, { + raw: { channels: 1, height: box.height, width: box.width }, + }) + .blur(Math.max(0.7, Math.min(box.width, box.height) * 0.012)) + .png({ compressionLevel: 6 }) + .toBuffer(); + + return sharp(cropBuffer, { failOn: "none" }) + .joinChannel(alphaBuffer) + .resize(targetBox.width, targetBox.height, { fit: "fill" }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + +async function buildScaleAdjustDeterministicFallbackImages(params: { + debugLabel: string; + verification: CanvasScaleAdjustVerificationConfig; +}) { + const { debugLabel, verification } = params; + const sourceImage = await loadScaleAdjustSourceImage(verification); + if (!sourceImage) return []; + let baseRaw = await sharp(sourceImage.sourceBuffer, { failOn: "none" }) + .resize(sourceImage.sourceWidth, sourceImage.sourceHeight, { + fit: "cover", + position: "center", + }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(); + const overlays: Array<{ buffer: Buffer; left: number; top: number }> = []; + + for (const [index, region] of verification.regions.entries()) { + const targetRegion = + verification.targetRegions?.[index] || + (typeof verification.scalePercent === "number" + ? scaleRegionAroundCenter(region, verification.scalePercent) + : null); + if (!targetRegion) continue; + const sourceBox = rectToPixelBox({ + height: sourceImage.sourceHeight, + region, + width: sourceImage.sourceWidth, + }); + const targetBox = rectToPixelBox({ + height: sourceImage.sourceHeight, + region: targetRegion, + width: sourceImage.sourceWidth, + }); + const mask = buildScaleAdjustFallbackSubjectMask({ + box: sourceBox, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + const maskedPixels = mask.reduce((sum, value) => sum + (value ? 1 : 0), 0); + if (maskedPixels < Math.max(24, sourceBox.width * sourceBox.height * 0.015)) { + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback skipped sparse mask:", { + debugLabel, + maskedPixels, + regionIndex: index, + }); + continue; + } + const background = estimateScaleAdjustFallbackBackground({ + box: sourceBox, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + const objectBuffer = await buildScaleAdjustFallbackObjectPng({ + box: sourceBox, + mask, + sourceBuffer: sourceImage.sourceBuffer, + targetBox, + }); + baseRaw = inpaintScaleAdjustMaskedRegion({ + background, + box: sourceBox, + mask, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + overlays.push({ + buffer: objectBuffer, + left: targetBox.left, + top: targetBox.top, + }); + } + + if (overlays.length === 0) return []; + const baseBuffer = await sharp(baseRaw, { + raw: { + channels: 3, + height: sourceImage.sourceHeight, + width: sourceImage.sourceWidth, + }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const mergedBuffer = await sharp(baseBuffer, { failOn: "none" }) + .composite(overlays) + .png({ compressionLevel: 6 }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + .resize(512, 512, { fit: "inside", withoutEnlargement: true }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-deterministic-fallback-${suffix}.png`, + bufferToArrayBuffer(mergedBuffer) + ), + uploadFile( + `canvas-scale-adjust-deterministic-fallback-thumb-${suffix}.png`, + bufferToArrayBuffer(thumbnailBuffer) + ), + ]); + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback generated:", { + debugLabel, + originalUrl: originalUpload.url, + regionCount: verification.regions.length, + thumbnailUrl: thumbnailUpload.url, + }); + + return [ + { + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + }, + ] satisfies ParsedImage[]; +} + function buildLocalEditCompositeMaskSvg(params: { width: number; height: number; @@ -2683,6 +3030,49 @@ export async function generateCanvasImage( selectedModelId: selectedModel.model.modelId, ...debugMeta, }); + if (scaleAdjustVerification) { + try { + const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({ + debugLabel, + verification: scaleAdjustVerification, + }); + if (fallbackImages.length > 0) { + const verifiedFallbackImages = await applyScaleAdjustVerificationToImages({ + debugLabel, + images: fallbackImages, + verification: { + ...scaleAdjustVerification, + rejectFailedCandidates: true, + }, + }); + console.warn("[Canvas Generate Image] Returning deterministic scale adjust fallback:", { + debugLabel, + fallbackImageCount: verifiedFallbackImages.length, + rejectedReason: scaleAdjustRejection.reason, + selectedModelRef: selectedModel.model.id, + selectedModelId: selectedModel.model.modelId, + ...debugMeta, + }); + return { + images: verifiedFallbackImages, + model: selectedModel.model.displayName, + modelRef: selectedModel.model.id, + prompt: resultPrompt, + }; + } + } catch (fallbackError) { + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback failed:", { + debugLabel, + error: + fallbackError instanceof Error + ? fallbackError.message + : String(fallbackError), + selectedModelRef: selectedModel.model.id, + selectedModelId: selectedModel.model.modelId, + ...debugMeta, + }); + } + } throw caughtError; } From 1ae9b5a89b66fced1aaadd1dd6218277e02e9edd Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 18:40:09 +0800 Subject: [PATCH 20/73] fix: verify scale adjust footprint shrink --- lib/canvas/generate-image.ts | 150 ++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 64247dcca..8159e3389 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -170,6 +170,8 @@ const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112; const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42; const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18; const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180; +const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO = 0.72; +const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60; const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "gemini", @@ -815,6 +817,109 @@ function isScaleAdjustOutsidePixelChanged(params: { ); } +async function getScaleAdjustFootprintChangeStats(params: { + generatedBuffer: Buffer; + sourceBuffer: Buffer; + sourceWidth: number; + sourceHeight: number; + verification: CanvasScaleAdjustVerificationConfig; +}) { + const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } = + params; + const normalizeToRaw = (buffer: Buffer) => + sharp(buffer, { failOn: "none" }) + .rotate() + .resize(sourceWidth, sourceHeight, { + fit: "cover", + position: "center", + }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(); + const [sourceRaw, generatedRaw] = await Promise.all([ + normalizeToRaw(sourceBuffer), + normalizeToRaw(generatedBuffer), + ]); + const shrinkStats: Array<{ + generatedOutsidePixels: number; + outsideRatio: number; + passed: boolean; + regionIndex: number; + sourceOutsidePixels: number; + }> = []; + + for (const [index, region] of verification.regions.entries()) { + const targetRegion = + verification.targetRegions?.[index] || + (typeof verification.scalePercent === "number" + ? scaleRegionAroundCenter(region, verification.scalePercent) + : null); + if (!targetRegion) continue; + const widthFactor = region.width > 0 ? targetRegion.width / region.width : 1; + const heightFactor = region.height > 0 ? targetRegion.height / region.height : 1; + const isShrink = widthFactor < 0.92 || heightFactor < 0.92; + if (!isShrink) continue; + + const sourceBox = rectToPixelBox({ + height: sourceHeight, + region, + width: sourceWidth, + }); + const targetBox = rectToPixelBox({ + height: sourceHeight, + region: targetRegion, + width: sourceWidth, + }); + const sourceMask = buildScaleAdjustFallbackSubjectMask({ + box: sourceBox, + raw: sourceRaw, + width: sourceWidth, + }); + const generatedMask = buildScaleAdjustFallbackSubjectMask({ + box: sourceBox, + raw: generatedRaw, + width: sourceWidth, + }); + let sourceOutsidePixels = 0; + let generatedOutsidePixels = 0; + for (let localY = 0; localY < sourceBox.height; localY += 1) { + const y = sourceBox.top + localY; + for (let localX = 0; localX < sourceBox.width; localX += 1) { + const x = sourceBox.left + localX; + const insideTarget = + x >= targetBox.left && + x < targetBox.left + targetBox.width && + y >= targetBox.top && + y < targetBox.top + targetBox.height; + if (insideTarget) continue; + const maskIndex = localY * sourceBox.width + localX; + if (sourceMask[maskIndex] > 0) sourceOutsidePixels += 1; + if (generatedMask[maskIndex] > 0) generatedOutsidePixels += 1; + } + } + const outsideRatio = + sourceOutsidePixels > 0 ? generatedOutsidePixels / sourceOutsidePixels : 0; + const passed = + sourceOutsidePixels < SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS || + outsideRatio <= SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO; + shrinkStats.push({ + generatedOutsidePixels, + outsideRatio, + passed, + regionIndex: index, + sourceOutsidePixels, + }); + } + + return { + failedRegionCount: shrinkStats.filter((item) => !item.passed).length, + passed: shrinkStats.every((item) => item.passed), + shrinkRegionCount: shrinkStats.length, + shrinkStats, + }; +} + async function loadScaleAdjustSourceImage( verification: CanvasScaleAdjustVerificationConfig ) { @@ -1038,6 +1143,7 @@ async function applyScaleAdjustVerificationToImages(params: { changedInside: boolean; cleanedImage?: ParsedImage; diffScores: number[]; + footprintStats: Awaited>; image: ParsedImage; index: number; outsideChanged: boolean; @@ -1050,7 +1156,13 @@ async function applyScaleAdjustVerificationToImages(params: { const generatedUrl = image.originalUrl || image.thumbnailUrl; const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); const changeRegions = getScaleAdjustEffectiveChangeRegions(verification); - const [diffScores, changeDiffScores, outsideDiffScores, outsidePixelStats] = await Promise.all([ + const [ + diffScores, + changeDiffScores, + outsideDiffScores, + outsidePixelStats, + footprintStats, + ] = await Promise.all([ getScaleAdjustDiffScores({ generatedBuffer, regions: verification.regions, @@ -1079,10 +1191,18 @@ async function applyScaleAdjustVerificationToImages(params: { sourceWidth: sourceImage.sourceWidth, verification, }), + getScaleAdjustFootprintChangeStats({ + generatedBuffer, + sourceBuffer: sourceImage.sourceBuffer, + sourceHeight: sourceImage.sourceHeight, + sourceWidth: sourceImage.sourceWidth, + verification, + }), ]); - const changedInside = changeDiffScores.some( + const visuallyChangedInside = changeDiffScores.some( (score) => score >= SCALE_ADJUST_VISUAL_DIFF_THRESHOLD ); + const changedInside = visuallyChangedInside && footprintStats.passed; const outsideChanged = isScaleAdjustOutsideChanged(outsideDiffScores) || isScaleAdjustOutsidePixelChanged(outsidePixelStats); @@ -1090,6 +1210,18 @@ async function applyScaleAdjustVerificationToImages(params: { console.info("[Canvas Generate Image] Scale adjust result guard:", { debugLabel, changedInside, + footprintChanged: footprintStats.passed, + footprintStats: { + failedRegionCount: footprintStats.failedRegionCount, + shrinkRegionCount: footprintStats.shrinkRegionCount, + shrinkStats: footprintStats.shrinkStats.map((item) => ({ + generatedOutsidePixels: item.generatedOutsidePixels, + outsideRatio: Number(item.outsideRatio.toFixed(3)), + passed: item.passed, + regionIndex: item.regionIndex, + sourceOutsidePixels: item.sourceOutsidePixels, + })), + }, imageIndex: index, outsideChanged, outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))), @@ -1120,6 +1252,18 @@ async function applyScaleAdjustVerificationToImages(params: { imageIndex: index, reasons: guardWarnings, changedInside, + footprintChanged: footprintStats.passed, + footprintStats: { + failedRegionCount: footprintStats.failedRegionCount, + shrinkRegionCount: footprintStats.shrinkRegionCount, + shrinkStats: footprintStats.shrinkStats.map((item) => ({ + generatedOutsidePixels: item.generatedOutsidePixels, + outsideRatio: Number(item.outsideRatio.toFixed(3)), + passed: item.passed, + regionIndex: item.regionIndex, + sourceOutsidePixels: item.sourceOutsidePixels, + })), + }, outsideChanged, outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))), outsidePixelStats: { @@ -1155,6 +1299,7 @@ async function applyScaleAdjustVerificationToImages(params: { changedInside, cleanedImage, diffScores, + footprintStats, image, index, outsideChanged, @@ -1176,6 +1321,7 @@ async function applyScaleAdjustVerificationToImages(params: { changedInside, cleanedImage, diffScores, + footprintStats, image, index, outsideChanged, From 956f9e0fedaec9a96d9d90c8776457d3974edde9 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 18:59:01 +0800 Subject: [PATCH 21/73] fix: guide scale adjust with layout draft --- app/api/canvas/local-edit/route.ts | 52 ++++++++++++++++++++++-------- lib/canvas/generate-image.ts | 4 +-- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index c017dbb7c..637809b5d 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1572,7 +1572,11 @@ function buildScaleAdjustCoordinatePrompt( scalePercent: ScaleAdjustPercent | null | undefined, sourceSize: { width: number; height: number }, cropReferences: ScaleAdjustCropReference[] = [], - options: { falSafe?: boolean; markedPhotoGuide?: boolean } = {} + options: { + falSafe?: boolean; + markedPhotoGuide?: boolean; + visualLayoutGuide?: boolean; + } = {} ) { const regionLines = regions .map((region) => { @@ -1657,9 +1661,11 @@ function buildScaleAdjustCoordinatePrompt( ].join("\n"); } - const coordinateGuideLine = options.markedPhotoGuide - ? "输入图片2是一张原图浅化后的完整画面位置参考图，不是输出底图。青色虚线框表示当前主体搜索框（对应下方 OLD），蓝色实线框表示目标最终占位（对应下方 TARGET）。请用输入图片2理解主体在真实室内画面中的位置关系，但最终图不能出现任何框线、色块、标注或参考图的浅化效果。" - : "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size furniture area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics."; + const coordinateGuideLine = options.visualLayoutGuide + ? "输入图片2是一张完整画面的粗略目标布局草稿，不是输出底图。它把 OLD 源框旧主体占位洗淡，表示旧尺寸主体需要移除并自然修复背景；同时把源框中的目标主体粗略缩放/移动到 TARGET 最终占位附近，帮助你理解最终主体大小、位置和室内画面关系。请只读取 input 2 的最终几何意图，不要复制它的淡化痕迹、粗糙拼接、局部透明边、矩形补丁、亮度变化或参考图画质；最终必须以 input 1 的真实照片为唯一画面基底重新完成无缝图片编辑。" + : options.markedPhotoGuide + ? "输入图片2是一张原图浅化后的完整画面位置参考图，不是输出底图。青色虚线框表示当前主体搜索框（对应下方 OLD），蓝色实线框表示目标最终占位（对应下方 TARGET）。请用输入图片2理解主体在真实室内画面中的位置关系，但最终图不能出现任何框线、色块、标注或参考图的浅化效果。" + : "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size furniture area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics."; return [ "Local furniture scale edit. Use input image 1 as the clean original photo and the locked camera frame.", @@ -1869,18 +1875,29 @@ export async function POST(request: NextRequest) { const useFalScaleAdjustCoordinatePrompt = useScaleAdjustCoordinateGuide && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; - const useScaleAdjustMarkedPhotoGuide = useScaleAdjustCoordinateGuide; + const useScaleAdjustVisualLayoutGuide = + useScaleAdjustCoordinateGuide && !useFalScaleAdjustCoordinatePrompt; + const useScaleAdjustMarkedPhotoGuide = + useScaleAdjustCoordinateGuide && !useScaleAdjustVisualLayoutGuide; const annotatedBuffer = isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? useScaleAdjustMarkedPhotoGuide - ? await buildScaleAdjustMarkedPhotoGuideBuffer({ - height: sourceHeight, + ? useScaleAdjustVisualLayoutGuide + ? await buildScaleAdjustVisualLayoutGuideBuffer({ regions, scalePercent, sourceBuffer, - width: sourceWidth, + sourceHeight, + sourceWidth, }) - : await sharp( + : useScaleAdjustMarkedPhotoGuide + ? await buildScaleAdjustMarkedPhotoGuideBuffer({ + height: sourceHeight, + regions, + scalePercent, + sourceBuffer, + width: sourceWidth, + }) + : await sharp( buildScaleAdjustZoomedCoordinateSvg({ height: sourceHeight, regions, @@ -1932,6 +1949,7 @@ export async function POST(request: NextRequest) { { falSafe: useFalScaleAdjustCoordinatePrompt, markedPhotoGuide: useScaleAdjustMarkedPhotoGuide, + visualLayoutGuide: useScaleAdjustVisualLayoutGuide, } ) : buildScaleAdjustPrompt( @@ -1999,7 +2017,9 @@ export async function POST(request: NextRequest) { annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation ? useScaleAdjustCoordinateGuide - ? useScaleAdjustMarkedPhotoGuide + ? useScaleAdjustVisualLayoutGuide + ? "scale-adjust-visual-layout-guide" + : useScaleAdjustMarkedPhotoGuide ? "scale-adjust-marked-photo-guide" : "scale-adjust-neutral-coordinate-map" : "scale-adjust-full-frame-semantic-edit-map" @@ -2020,11 +2040,13 @@ export async function POST(request: NextRequest) { scaleAdjustNeutralCoordinateMapApplied: isScaleAdjustOperation && useScaleAdjustCoordinateGuide && - !useScaleAdjustMarkedPhotoGuide, + !useScaleAdjustMarkedPhotoGuide && + !useScaleAdjustVisualLayoutGuide, scaleAdjustMarkedPhotoGuideApplied: useScaleAdjustMarkedPhotoGuide, scaleAdjustFalMarkedPhotoGuideApplied: useFalScaleAdjustCoordinatePrompt, scaleAdjustVisualLayoutGuideApplied: - isScaleAdjustOperation && !useScaleAdjustCoordinateGuide, + isScaleAdjustOperation && + (!useScaleAdjustCoordinateGuide || useScaleAdjustVisualLayoutGuide), falScaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image", @@ -2054,7 +2076,9 @@ export async function POST(request: NextRequest) { ? useScaleAdjustCoordinateGuide ? useFalScaleAdjustCoordinatePrompt ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" - : "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" + : useScaleAdjustVisualLayoutGuide + ? "same-name-visual-layout-guide-v50-one-shot" + : "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: false, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 8159e3389..c5a6e8f40 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -1643,7 +1643,7 @@ async function buildScaleAdjustDeterministicFallbackImages(params: { .toColorspace("srgb") .raw() .toBuffer(); - const overlays: Array<{ buffer: Buffer; left: number; top: number }> = []; + const overlays: Array<{ input: Buffer; left: number; top: number }> = []; for (const [index, region] of verification.regions.entries()) { const targetRegion = @@ -1695,7 +1695,7 @@ async function buildScaleAdjustDeterministicFallbackImages(params: { width: sourceImage.sourceWidth, }); overlays.push({ - buffer: objectBuffer, + input: objectBuffer, left: targetBox.left, top: targetBox.top, }); From 98e987e75b7888a7ab1ff86269a60b2d29316eee Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 19:38:24 +0800 Subject: [PATCH 22/73] fix: focus scale adjust subject guide --- app/api/canvas/local-edit/route.ts | 69 +++++++++++++++++++++++++-- lib/canvas/generate-image.ts | 76 ++++++++++++++++++++++++++++-- 2 files changed, 135 insertions(+), 10 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 637809b5d..4903081ff 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -791,8 +791,18 @@ function getScaleAdjustVisualGuidePlacement(params: { sourceTop + 1, height ); - const sourceWidth = Math.max(1, Math.round(sourceRight - sourceLeft)); - const sourceHeight = Math.max(1, Math.round(sourceBottom - sourceTop)); + const sourceFootprintBox = { + left: Math.round(sourceLeft), + top: Math.round(sourceTop), + width: Math.max(1, Math.round(sourceRight - sourceLeft)), + height: Math.max(1, Math.round(sourceBottom - sourceTop)), + }; + const sourceBox = getScaleAdjustFocusedSourceBox({ + region, + sourceBox: sourceFootprintBox, + }); + const sourceWidth = sourceBox.width; + const sourceHeight = sourceBox.height; const widthScale = target.width / Math.max(1, sourceWidth); const heightScale = target.height / Math.max(1, sourceHeight); const uniformScale = Math.max( @@ -831,16 +841,64 @@ function getScaleAdjustVisualGuidePlacement(params: { placedTop: Math.max(0, Math.round(placedTop)), placedWidth: Math.max(1, Math.round(placedWidth)), sourceBox: { - left: Math.round(sourceLeft), - top: Math.round(sourceTop), + left: sourceBox.left, + top: sourceBox.top, width: sourceWidth, height: sourceHeight, }, + sourceFootprintBox, targetBox: target, uniformScale, }; } +function getScaleAdjustFocusDirection(region: NormalizedLocalEditRegion) { + const text = normalizeScaleAdjustModelText(region.prompt); + const compact = text.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, ""); + if (text.includes("left") || compact.includes("左")) return "left"; + if (text.includes("right") || compact.includes("右")) return "right"; + if (text.includes("top") || compact.includes("上")) return "top"; + if (text.includes("bottom") || compact.includes("下")) return "bottom"; + return ""; +} + +function getScaleAdjustFocusedSourceBox(params: { + region: NormalizedLocalEditRegion; + sourceBox: { left: number; top: number; width: number; height: number }; +}) { + const { region, sourceBox } = params; + const direction = getScaleAdjustFocusDirection(region); + if (!direction) return sourceBox; + + const minWidth = Math.max(1, Math.round(sourceBox.width * 0.42)); + const minHeight = Math.max(1, Math.round(sourceBox.height * 0.42)); + let left = sourceBox.left; + let top = sourceBox.top; + let width = sourceBox.width; + let height = sourceBox.height; + + if (direction === "left") { + width = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); + } else if (direction === "right") { + const nextWidth = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); + left = sourceBox.left + sourceBox.width - nextWidth; + width = nextWidth; + } else if (direction === "top") { + height = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); + } else if (direction === "bottom") { + const nextHeight = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); + top = sourceBox.top + sourceBox.height - nextHeight; + height = nextHeight; + } + + return { + left, + top, + width, + height, + }; +} + function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) { const { height, width } = params; const blur = Math.max(2, Math.min(14, Math.round(Math.min(width, height) * 0.035))); @@ -966,7 +1024,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { scalePercent, width: sourceWidth, }); - sourceFootprintBoxes.push(placement.sourceBox); + sourceFootprintBoxes.push(placement.sourceFootprintBox); placements.push(placement); const resizedCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) @@ -1989,6 +2047,7 @@ export async function POST(request: NextRequest) { : undefined, sourceImageUrl, regions: regions.map((region) => region.rect), + regionPrompts: regions.map((region) => region.prompt), ...(typeof scalePercent === "number" ? { scalePercent: scalePercent as ScaleAdjustPercent } : {}), diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index c5a6e8f40..bda4f43a6 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -56,6 +56,7 @@ export interface CanvasScaleAdjustVerificationConfig { allowOutsideChangedWhenChangedInside?: boolean; protectedRegions?: CanvasLocalEditCompositeRegion[]; rejectFailedCandidates?: boolean; + regionPrompts?: string[]; sourceImageUrl: string; regions: CanvasLocalEditCompositeRegion[]; targetRegions?: CanvasLocalEditCompositeRegion[]; @@ -1405,6 +1406,63 @@ function getRawPixelIndex(width: number, x: number, y: number) { return (y * width + x) * 3; } +function normalizeScaleAdjustFocusText(input: unknown) { + return String(input || "") + .trim() + .toLowerCase() + .normalize("NFKC") + .replace(/[「」『』【】[\]()（）{}]/g, " ") + .replace(/[_/|:：,，.。+]+/g, " ") + .replace(/\s+/g, " "); +} + +function getScaleAdjustFallbackFocusDirection(prompt: unknown) { + const text = normalizeScaleAdjustFocusText(prompt); + const compact = text.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, ""); + if (text.includes("left") || compact.includes("左")) return "left"; + if (text.includes("right") || compact.includes("右")) return "right"; + if (text.includes("top") || compact.includes("上")) return "top"; + if (text.includes("bottom") || compact.includes("下")) return "bottom"; + return ""; +} + +function getScaleAdjustFallbackFocusedSourceBox(params: { + prompt?: string; + sourceBox: ReturnType; +}) { + const { prompt, sourceBox } = params; + const direction = getScaleAdjustFallbackFocusDirection(prompt); + if (!direction) return sourceBox; + + const minWidth = Math.max(1, Math.round(sourceBox.width * 0.42)); + const minHeight = Math.max(1, Math.round(sourceBox.height * 0.42)); + let left = sourceBox.left; + let top = sourceBox.top; + let width = sourceBox.width; + let height = sourceBox.height; + + if (direction === "left") { + width = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); + } else if (direction === "right") { + const nextWidth = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); + left = sourceBox.left + sourceBox.width - nextWidth; + width = nextWidth; + } else if (direction === "top") { + height = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); + } else if (direction === "bottom") { + const nextHeight = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); + top = sourceBox.top + sourceBox.height - nextHeight; + height = nextHeight; + } + + return { + left, + top, + width, + height, + }; +} + function getLuma(r: number, g: number, b: number) { return r * 0.299 + g * 0.587 + b * 0.114; } @@ -1662,13 +1720,21 @@ async function buildScaleAdjustDeterministicFallbackImages(params: { region: targetRegion, width: sourceImage.sourceWidth, }); + const prompt = verification.regionPrompts?.[index] || ""; + const focusedSourceBox = getScaleAdjustFallbackFocusedSourceBox({ + prompt, + sourceBox, + }); const mask = buildScaleAdjustFallbackSubjectMask({ - box: sourceBox, + box: focusedSourceBox, raw: baseRaw, width: sourceImage.sourceWidth, }); const maskedPixels = mask.reduce((sum, value) => sum + (value ? 1 : 0), 0); - if (maskedPixels < Math.max(24, sourceBox.width * sourceBox.height * 0.015)) { + if ( + maskedPixels < + Math.max(24, focusedSourceBox.width * focusedSourceBox.height * 0.015) + ) { console.warn("[Canvas Generate Image] Scale adjust deterministic fallback skipped sparse mask:", { debugLabel, maskedPixels, @@ -1677,19 +1743,19 @@ async function buildScaleAdjustDeterministicFallbackImages(params: { continue; } const background = estimateScaleAdjustFallbackBackground({ - box: sourceBox, + box: focusedSourceBox, raw: baseRaw, width: sourceImage.sourceWidth, }); const objectBuffer = await buildScaleAdjustFallbackObjectPng({ - box: sourceBox, + box: focusedSourceBox, mask, sourceBuffer: sourceImage.sourceBuffer, targetBox, }); baseRaw = inpaintScaleAdjustMaskedRegion({ background, - box: sourceBox, + box: focusedSourceBox, mask, raw: baseRaw, width: sourceImage.sourceWidth, From 7152435a3d7790590b4621c7c0cba5b40fa59c31 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 19:54:33 +0800 Subject: [PATCH 23/73] fix: add focused crop reference for scale adjust --- app/api/canvas/local-edit/route.ts | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 4903081ff..919766b67 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -739,11 +739,17 @@ async function createScaleAdjustCropReferences(params: { const regionsForCrop = params.regions.slice(0, MAX_SCALE_ADJUST_CROP_REFERENCES); for (const region of regionsForCrop) { - const cropBox = expandPixelRectForCrop({ - height: params.sourceHeight, - pixelRect: region.pixelRect, - width: params.sourceWidth, + const focusBox = getScaleAdjustFocusedSourceBox({ + region, + sourceBox: region.pixelRect, }); + const cropBox = getScaleAdjustFocusDirection(region) + ? focusBox + : expandPixelRectForCrop({ + height: params.sourceHeight, + pixelRect: region.pixelRect, + width: params.sourceWidth, + }); const cropBuffer = await sharp(params.sourceBuffer, { failOn: "none" }) .extract(cropBox) .resize({ @@ -1720,7 +1726,7 @@ function buildScaleAdjustCoordinatePrompt( } const coordinateGuideLine = options.visualLayoutGuide - ? "输入图片2是一张完整画面的粗略目标布局草稿，不是输出底图。它把 OLD 源框旧主体占位洗淡，表示旧尺寸主体需要移除并自然修复背景；同时把源框中的目标主体粗略缩放/移动到 TARGET 最终占位附近，帮助你理解最终主体大小、位置和室内画面关系。请只读取 input 2 的最终几何意图，不要复制它的淡化痕迹、粗糙拼接、局部透明边、矩形补丁、亮度变化或参考图画质；最终必须以 input 1 的真实照片为唯一画面基底重新完成无缝图片编辑。" + ? "输入图片2是一张完整画面的粗略目标布局草稿，不是输出底图。它把 OLD 源框旧主体占位洗淡，表示旧尺寸主体必须从原尺寸位置移除并自然修复背景；同时把目标主体粗略缩放/移动到 TARGET 最终占位附近，这个缩小后的位置和尺寸是最终结果必须匹配的几何目标。请把 input 2 当成最终构图/比例的强约束：如果 input 2 中目标主体比 input 1 小，最终输出也必须明显变小，不能保留 input 1 的原尺寸主体。不要复制 input 2 的淡化痕迹、粗糙拼接、局部透明边、矩形补丁、亮度变化或参考图画质；最终必须以 input 1 的真实照片为唯一画面基底重新完成无缝图片编辑。" : options.markedPhotoGuide ? "输入图片2是一张原图浅化后的完整画面位置参考图，不是输出底图。青色虚线框表示当前主体搜索框（对应下方 OLD），蓝色实线框表示目标最终占位（对应下方 TARGET）。请用输入图片2理解主体在真实室内画面中的位置关系，但最终图不能出现任何框线、色块、标注或参考图的浅化效果。" : "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size furniture area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics."; @@ -1993,7 +1999,15 @@ export async function POST(request: NextRequest) { annotatedFilename, toArrayBuffer(annotatedBuffer) ); - const scaleAdjustCropReferences: ScaleAdjustCropReference[] = []; + const scaleAdjustCropReferences: ScaleAdjustCropReference[] = + isScaleAdjustOperation && useScaleAdjustVisualLayoutGuide + ? await createScaleAdjustCropReferences({ + regions, + sourceBuffer, + sourceHeight, + sourceWidth, + }) + : []; const prompt = isScaleAdjustOperation ? useScaleAdjustCoordinateGuide ? buildScaleAdjustCoordinatePrompt( @@ -2136,7 +2150,7 @@ export async function POST(request: NextRequest) { ? useFalScaleAdjustCoordinatePrompt ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" : useScaleAdjustVisualLayoutGuide - ? "same-name-visual-layout-guide-v50-one-shot" + ? "same-name-focused-layout-crop-guide-v54-one-shot" : "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, From 2f312a01dff96366133be7b2fd3f3e41226a54b3 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 20:20:01 +0800 Subject: [PATCH 24/73] fix: support mask guided scale adjust --- app/api/canvas/local-edit/route.ts | 135 +++++++++++++++++++++++++++-- lib/ai/providers/openai-image.ts | 9 ++ lib/canvas/generate-image.ts | 17 +++- 3 files changed, 155 insertions(+), 6 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 919766b67..944a6c451 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -17,6 +17,7 @@ import { import { modelLacksReferenceImageInputSupport, } from "@/lib/ai/reference-image-capability"; +import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image"; import { getImageToolCreditCost } from "@/lib/points/config"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; import { uploadFile } from "@/lib/storage/service"; @@ -278,6 +279,19 @@ function getScaleAdjustModelFamily( ); } +function isScaleAdjustMaskCapableModel( + entry: AiModelWithProviderForSelection | null | undefined +) { + if (!entry || getScaleAdjustModelFamily(entry) !== "gpt-image-2") { + return false; + } + if (entry.provider.type === "openai-image") return true; + if (entry.provider.type === "fal-image") { + return isFalGptImage2EditModelId(entry.model.modelId || ""); + } + return false; +} + function isScaleAdjustCoordinateGuideModel( entry: AiModelWithProviderForSelection | null | undefined ) { @@ -1004,6 +1018,89 @@ function buildScaleAdjustTargetFootprintShadowSvg(params: { ); } +function getScaleAdjustEditMaskBoxes(params: { + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; + sourceHeight: number; + sourceWidth: number; +}) { + const { regions, scalePercent, sourceHeight, sourceWidth } = params; + return regions.map((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const workspaceRect = getScaleAdjustWorkspaceRect( + region.rect, + targetRect, + scalePercent + ); + return rectToPixels(workspaceRect, sourceWidth, sourceHeight); + }); +} + +async function buildScaleAdjustEditMaskBuffer(params: { + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; + sourceHeight: number; + sourceWidth: number; +}) { + const { regions, scalePercent, sourceHeight, sourceWidth } = params; + const boxes = getScaleAdjustEditMaskBoxes({ + regions, + scalePercent, + sourceHeight, + sourceWidth, + }); + const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight)); + const blur = Math.max(5, Math.min(28, Math.round(minSide * 0.018))); + const pad = Math.max(2, Math.round(blur * 0.55)); + const rects = boxes + .map((box) => { + const left = clamp(box.left - pad, 0, Math.max(0, sourceWidth - 1)); + const top = clamp(box.top - pad, 0, Math.max(0, sourceHeight - 1)); + const right = clamp( + box.left + box.width + pad, + left + 1, + sourceWidth + ); + const bottom = clamp( + box.top + box.height + pad, + top + 1, + sourceHeight + ); + return ``; + }) + .join(""); + const editMaskSvg = Buffer.from( + ` + + + + + + + ${rects} + ` + ); + const editAlpha = await sharp(editMaskSvg, { failOn: "none" }) + .resize(sourceWidth, sourceHeight, { fit: "fill" }) + .greyscale() + .raw() + .toBuffer(); + const rgba = Buffer.alloc(sourceWidth * sourceHeight * 4); + for (let index = 0; index < sourceWidth * sourceHeight; index += 1) { + const offset = index * 4; + const editable = editAlpha[index] || 0; + rgba[offset] = 0; + rgba[offset + 1] = 0; + rgba[offset + 2] = 0; + rgba[offset + 3] = clamp(255 - editable, 0, 255); + } + return sharp(rgba, { + raw: { channels: 4, height: sourceHeight, width: sourceWidth }, + }) + .png({ compressionLevel: 9 }) + .toBuffer(); +} + async function buildScaleAdjustVisualLayoutGuideBuffer(params: { regions: NormalizedLocalEditRegion[]; scalePercent?: ScaleAdjustPercent | null; @@ -1999,8 +2096,29 @@ export async function POST(request: NextRequest) { annotatedFilename, toArrayBuffer(annotatedBuffer) ); + const useScaleAdjustEditMask = + isScaleAdjustOperation && + isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry); + const scaleAdjustEditMaskUpload = useScaleAdjustEditMask + ? await uploadFile( + `canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random() + .toString(36) + .slice(2, 8)}.png`, + toArrayBuffer( + await buildScaleAdjustEditMaskBuffer({ + regions, + scalePercent, + sourceHeight, + sourceWidth, + }) + ) + ) + : null; + const shouldCreateScaleAdjustCropReferences = + isScaleAdjustOperation && + (useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinateGuide); const scaleAdjustCropReferences: ScaleAdjustCropReference[] = - isScaleAdjustOperation && useScaleAdjustVisualLayoutGuide + shouldCreateScaleAdjustCropReferences ? await createScaleAdjustCropReferences({ regions, sourceBuffer, @@ -2054,11 +2172,9 @@ export async function POST(request: NextRequest) { const scaleAdjustVerificationConfig = isScaleAdjustOperation ? { allowOutsideChangedWhenChangedInside: false, + allowDeterministicFallback: false, protectedRegions: scaleAdjustCompositeRegions, - rejectFailedCandidates: - useScaleAdjustCoordinateGuide && Boolean(scaleAdjustModelFamily) - ? true - : undefined, + rejectFailedCandidates: true, sourceImageUrl, regions: regions.map((region) => region.rect), regionPrompts: regions.map((region) => region.prompt), @@ -2097,6 +2213,12 @@ export async function POST(request: NextRequest) { : "scale-adjust-neutral-coordinate-map" : "scale-adjust-full-frame-semantic-edit-map" : "source-overlay", + scaleAdjustEditMaskApplied: Boolean(scaleAdjustEditMaskUpload), + scaleAdjustEditMaskUrl: scaleAdjustEditMaskUpload?.url || null, + scaleAdjustEditMaskProviderType: + useScaleAdjustEditMask + ? resolvedScaleAdjustModelEntry?.provider?.type || null + : null, cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, detailFixedModelApplied: Boolean(detailFixedModel), @@ -2181,6 +2303,9 @@ export async function POST(request: NextRequest) { : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, + providerImageParams: scaleAdjustEditMaskUpload + ? { maskImageUrl: scaleAdjustEditMaskUpload.url } + : undefined, referenceImages, scaleAdjustVerification: scaleAdjustVerificationConfig, localEditComposite: isScaleAdjustOperation diff --git a/lib/ai/providers/openai-image.ts b/lib/ai/providers/openai-image.ts index 28eef4685..1dd0ee129 100644 --- a/lib/ai/providers/openai-image.ts +++ b/lib/ai/providers/openai-image.ts @@ -19,6 +19,7 @@ export interface OpenAIImageParams { quality?: string; background?: string; outputFormat?: string; + maskImageUrl?: string; } interface UploadedImage { @@ -643,6 +644,7 @@ async function requestOpenAIImageGeneration(params: { aspectRatio?: string; imageSizeLabel?: "1K" | "2K" | "4K"; imageUrls: string[]; + maskImageUrl?: string; quality?: string; background?: string; outputFormat?: string; @@ -656,6 +658,7 @@ async function requestOpenAIImageGeneration(params: { aspectRatio, imageSizeLabel, imageUrls, + maskImageUrl, quality, background, outputFormat, @@ -723,6 +726,10 @@ async function requestOpenAIImageGeneration(params: { formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename); } } + if (maskImageUrl) { + const mask = await loadInputImage(maskImageUrl); + formData.append("mask", new Blob([mask.buffer], { type: mask.mimeType }), mask.filename); + } const response = await fetch(endpoint, { method: "POST", @@ -898,6 +905,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { timeoutMs: resolveRequestTimeoutMs(baseURL, imageSize, activeModelId), useBltcyImageCompatParams: shouldUseBltcyImageCompatParams(baseURL), imageCount: imageUrls.length, + hasMask: Boolean(options.imageParams?.maskImageUrl), baseURL: normalizeBaseURL(baseURL), }); @@ -910,6 +918,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { aspectRatio: aspectRatio || undefined, imageSizeLabel: imageSize, imageUrls, + maskImageUrl: options.imageParams?.maskImageUrl, quality, background, outputFormat, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index bda4f43a6..151146cb2 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -53,6 +53,7 @@ export interface CanvasLocalEditCompositeConfig { } export interface CanvasScaleAdjustVerificationConfig { + allowDeterministicFallback?: boolean; allowOutsideChangedWhenChangedInside?: boolean; protectedRegions?: CanvasLocalEditCompositeRegion[]; rejectFailedCandidates?: boolean; @@ -75,6 +76,7 @@ export interface CanvasGenerateImageRetryVariant { negativePrompt?: string; prompt?: string; promptExtend?: boolean; + providerImageParams?: Record; referenceImage?: string; referenceImages?: string[]; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; @@ -92,6 +94,7 @@ export interface CanvasGenerateImageRequest { falSystemPrompt?: string; falThinkingLevel?: "minimal" | "high"; promptExtend?: boolean; + providerImageParams?: Record; referenceImage?: string; referenceImages?: string[]; billingFeature?: "multi-view" | "repaint"; @@ -395,15 +398,22 @@ function normalizeScaleAdjustVerificationConfig( Boolean(region) ) : []; + const regionPrompts = Array.isArray(value.regionPrompts) + ? value.regionPrompts + .map((prompt) => String(prompt || "").trim()) + .slice(0, regions.length) + : []; return regions.length > 0 && (targetRegions.length > 0 || typeof scalePercent === "number") ? { sourceImageUrl, regions, + allowDeterministicFallback: value.allowDeterministicFallback === true, allowOutsideChangedWhenChangedInside: value.allowOutsideChangedWhenChangedInside === true, rejectFailedCandidates: value.rejectFailedCandidates === true, + ...(regionPrompts.length > 0 ? { regionPrompts } : {}), ...(targetRegions.length > 0 ? { targetRegions } : {}), ...(typeof scalePercent === "number" ? { scalePercent } : {}), ...(protectedRegions.length > 0 ? { protectedRegions } : {}), @@ -2526,6 +2536,10 @@ export async function generateCanvasImage( : 1; const promptExtend = typeof request.promptExtend === "boolean" ? request.promptExtend : undefined; + const providerImageParams = + request.providerImageParams && typeof request.providerImageParams === "object" + ? request.providerImageParams + : undefined; const ownerKey = typeof request.ownerKey === "string" && request.ownerKey.trim() ? request.ownerKey.trim() @@ -2742,6 +2756,7 @@ export async function generateCanvasImage( ]; const imageParams = { + ...(providerImageParams || {}), aspectRatio, safetyTolerance: request.falSafetyTolerance, imageSize, @@ -3242,7 +3257,7 @@ export async function generateCanvasImage( selectedModelId: selectedModel.model.modelId, ...debugMeta, }); - if (scaleAdjustVerification) { + if (scaleAdjustVerification?.allowDeterministicFallback) { try { const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({ debugLabel, From 7c8ae2fe8baa2daf6cbb3d793d0ccc73098db037 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 20:31:13 +0800 Subject: [PATCH 25/73] fix: honor requested scale adjust model --- app/api/canvas/local-edit/route.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 944a6c451..d03e50bc2 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -382,10 +382,7 @@ function resolveScaleAdjustPreferredModelId(params: { params.configuredModelId ); if (isScaleAdjustUsableImageEditModel(requestedModel)) { - const requestedFamily = getScaleAdjustModelFamily(requestedModel); - if (requestedFamily) { - return requestedModel?.model.id || params.requestedModelId; - } + return requestedModel?.model.id || params.requestedModelId; } const autoDetailModel = params.models.find(isScaleAdjustDedicatedDetailModel); From 56240d4a6d6fe42861b90c506f4f1358884a2191 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 20:44:14 +0800 Subject: [PATCH 26/73] fix: use marked guide for alibaba scale adjust --- app/api/canvas/local-edit/route.ts | 48 +++++++++++++++++++----------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index d03e50bc2..1c2b9fd76 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2030,15 +2030,22 @@ export async function POST(request: NextRequest) { const scaleAdjustModelFamily = getScaleAdjustModelFamily( resolvedScaleAdjustModelEntry ); + const useProviderMarkedPhotoCoordinatePrompt = + isScaleAdjustOperation && + resolvedScaleAdjustModelEntry?.provider?.type === "alibaba-image"; + const useScaleAdjustCoordinatePrompt = + useScaleAdjustCoordinateGuide || useProviderMarkedPhotoCoordinatePrompt; const useFalScaleAdjustCoordinatePrompt = - useScaleAdjustCoordinateGuide && + useScaleAdjustCoordinatePrompt && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; const useScaleAdjustVisualLayoutGuide = - useScaleAdjustCoordinateGuide && !useFalScaleAdjustCoordinatePrompt; + useScaleAdjustCoordinatePrompt && + !useFalScaleAdjustCoordinatePrompt && + !useProviderMarkedPhotoCoordinatePrompt; const useScaleAdjustMarkedPhotoGuide = - useScaleAdjustCoordinateGuide && !useScaleAdjustVisualLayoutGuide; + useScaleAdjustCoordinatePrompt && !useScaleAdjustVisualLayoutGuide; const annotatedBuffer = isScaleAdjustOperation - ? useScaleAdjustCoordinateGuide + ? useScaleAdjustCoordinatePrompt ? useScaleAdjustVisualLayoutGuide ? await buildScaleAdjustVisualLayoutGuideBuffer({ regions, @@ -2113,7 +2120,9 @@ export async function POST(request: NextRequest) { : null; const shouldCreateScaleAdjustCropReferences = isScaleAdjustOperation && - (useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinateGuide); + (useScaleAdjustVisualLayoutGuide || + !useScaleAdjustCoordinatePrompt || + (useScaleAdjustMarkedPhotoGuide && !useFalScaleAdjustCoordinatePrompt)); const scaleAdjustCropReferences: ScaleAdjustCropReference[] = shouldCreateScaleAdjustCropReferences ? await createScaleAdjustCropReferences({ @@ -2124,7 +2133,7 @@ export async function POST(request: NextRequest) { }) : []; const prompt = isScaleAdjustOperation - ? useScaleAdjustCoordinateGuide + ? useScaleAdjustCoordinatePrompt ? buildScaleAdjustCoordinatePrompt( regions, scalePercent, @@ -2202,7 +2211,7 @@ export async function POST(request: NextRequest) { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation - ? useScaleAdjustCoordinateGuide + ? useScaleAdjustCoordinatePrompt ? useScaleAdjustVisualLayoutGuide ? "scale-adjust-visual-layout-guide" : useScaleAdjustMarkedPhotoGuide @@ -2229,16 +2238,17 @@ export async function POST(request: NextRequest) { resolvedScaleAdjustModelEntry?.provider?.type || null, scaleAdjustModelFamily: scaleAdjustModelFamily || null, scaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide, + scaleAdjustCoordinatePromptApplied: useScaleAdjustCoordinatePrompt, scaleAdjustNeutralCoordinateMapApplied: isScaleAdjustOperation && - useScaleAdjustCoordinateGuide && + useScaleAdjustCoordinatePrompt && !useScaleAdjustMarkedPhotoGuide && !useScaleAdjustVisualLayoutGuide, scaleAdjustMarkedPhotoGuideApplied: useScaleAdjustMarkedPhotoGuide, scaleAdjustFalMarkedPhotoGuideApplied: useFalScaleAdjustCoordinatePrompt, scaleAdjustVisualLayoutGuideApplied: isScaleAdjustOperation && - (!useScaleAdjustCoordinateGuide || useScaleAdjustVisualLayoutGuide), + (!useScaleAdjustCoordinatePrompt || useScaleAdjustVisualLayoutGuide), falScaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image", @@ -2265,13 +2275,15 @@ export async function POST(request: NextRequest) { scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) .length, scaleAdjustPromptStrategy: isScaleAdjustOperation - ? useScaleAdjustCoordinateGuide - ? useFalScaleAdjustCoordinatePrompt - ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" - : useScaleAdjustVisualLayoutGuide - ? "same-name-focused-layout-crop-guide-v54-one-shot" - : "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" - : "locked-plate-semantic-target-map-no-old-crop-v15" + ? useFalScaleAdjustCoordinatePrompt + ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" + : useScaleAdjustVisualLayoutGuide + ? "same-name-focused-layout-crop-guide-v54-one-shot" + : useProviderMarkedPhotoCoordinatePrompt + ? "provider-marked-photo-crop-guide-v60-one-shot" + : useScaleAdjustMarkedPhotoGuide + ? "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" + : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, @@ -2282,7 +2294,7 @@ export async function POST(request: NextRequest) { displayPrompt, falSafetyTolerance: useScaleAdjustCoordinateGuide ? "6" : undefined, falSystemPrompt: - useScaleAdjustCoordinateGuide + useScaleAdjustCoordinatePrompt ? buildScaleAdjustCoordinateSystemPrompt() : undefined, falThinkingLevel: useScaleAdjustCoordinateGuide ? "high" : undefined, @@ -2294,7 +2306,7 @@ export async function POST(request: NextRequest) { model: resolvedModel || undefined, negativePrompt: isScaleAdjustOperation - ? useScaleAdjustCoordinateGuide + ? useScaleAdjustCoordinatePrompt ? scaleAdjustCoordinateNegativePrompt : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、淡化痕迹、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、文字位置变化、文字内容变化、文字模糊、Logo位置变化、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", From 11e6aca724cd1fd825b3e4d8b3aa8867134802b5 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 20:57:00 +0800 Subject: [PATCH 27/73] fix: use text coordinate guide for alibaba scale adjust --- app/api/canvas/local-edit/route.ts | 96 ++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 12 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 1c2b9fd76..f42716b0c 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1867,6 +1867,61 @@ function buildScaleAdjustCoordinateSystemPrompt() { ].join("\n"); } +function buildScaleAdjustTextOnlyPrompt( + regions: NormalizedLocalEditRegion[], + scalePercent: ScaleAdjustPercent | null | undefined, + cropReferences: ScaleAdjustCropReference[] = [], + sourceSize: { width: number; height: number } +) { + const cropLines = cropReferences + .map( + (reference, index) => + `Input image ${index + 2}: clean crop reference for Region ${reference.regionIndex}; use it only to identify the furniture subject and its natural aspect ratio, never paste the crop into the output.` + ) + .join("\n"); + const regionLines = regions + .map((region) => { + const cropIndex = cropReferences.findIndex( + (reference) => reference.regionIndex === region.index + ); + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const targetPixelRect = + region.targetPixelRect || + rectToPixels(targetRect, sourceSize.width, sourceSize.height); + const widthFactor = + region.rect.width > 0 ? targetRect.width / region.rect.width : 1; + const heightFactor = + region.rect.height > 0 ? targetRect.height / region.rect.height : 1; + const sourceRight = region.pixelRect.left + region.pixelRect.width; + const sourceBottom = region.pixelRect.top + region.pixelRect.height; + const targetRight = targetPixelRect.left + targetPixelRect.width; + const targetBottom = targetPixelRect.top + targetPixelRect.height; + const cropNote = + cropIndex >= 0 + ? ` Input image ${cropIndex + 2} is the clean crop reference for this furniture identity and silhouette.` + : ""; + return `Region ${region.index}: source search box left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; target final footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || "scale the furniture to the target footprint"}.${cropNote}`; + }) + .join("\n"); + + return [ + "Local furniture scale edit. Input image 1 is the locked source photo and the only final canvas base.", + `Return one clean realistic photo with exactly the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, + cropLines ? cropLines : "", + "Do not output any guide marks, boxes, labels, arrows, blue lines, cyan lines, overlays, UI marks, text annotations, patch edges, full-frame zoom, crop, pan, or redesigned room.", + "Only edit the main movable furniture subject inside each source search box. Keep all other furniture, sofa, lamps, wall, windows, rug, floor, decor, text, lighting, and camera frame stable.", + "Use the numeric pixel coordinates below as the only geometry guide. Do not need a visible guide image.", + "When target footprint is smaller than source, make the same furniture visibly smaller and naturally rebuild the newly visible floor, rug, wall, shadow, or background around it. Do not leave the old full-size footprint.", + "Keep the furniture identity, materials, perspective, objects on top, contact shadows, and natural aspect ratio. Do not stretch, squash, duplicate, or paste a rectangular crop.", + "中文硬约束：输入图1是锁定底图，只能调整源框里的主要家具主体；目标框是最终家具可见占位。禁止整图重绘、整图缩放、输出标注框、输出蓝线或把裁切图贴回。", + "", + "Region instructions:", + regionLines, + "", + "Success check before returning: the selected furniture must be clearly closer to the target footprint than to the original source footprint, while the rest of the photo remains visually unchanged and clean.", + ].filter(Boolean).join("\n"); +} + export async function POST(request: NextRequest) { try { const session = await auth(); @@ -2030,20 +2085,21 @@ export async function POST(request: NextRequest) { const scaleAdjustModelFamily = getScaleAdjustModelFamily( resolvedScaleAdjustModelEntry ); - const useProviderMarkedPhotoCoordinatePrompt = + const useProviderTextOnlyCoordinatePrompt = isScaleAdjustOperation && resolvedScaleAdjustModelEntry?.provider?.type === "alibaba-image"; - const useScaleAdjustCoordinatePrompt = - useScaleAdjustCoordinateGuide || useProviderMarkedPhotoCoordinatePrompt; + const useScaleAdjustCoordinatePrompt = useScaleAdjustCoordinateGuide; const useFalScaleAdjustCoordinatePrompt = useScaleAdjustCoordinatePrompt && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; const useScaleAdjustVisualLayoutGuide = useScaleAdjustCoordinatePrompt && !useFalScaleAdjustCoordinatePrompt && - !useProviderMarkedPhotoCoordinatePrompt; + !useProviderTextOnlyCoordinatePrompt; const useScaleAdjustMarkedPhotoGuide = useScaleAdjustCoordinatePrompt && !useScaleAdjustVisualLayoutGuide; + const sendScaleAdjustAnnotatedReferenceToModel = + isScaleAdjustOperation && !useProviderTextOnlyCoordinatePrompt; const annotatedBuffer = isScaleAdjustOperation ? useScaleAdjustCoordinatePrompt ? useScaleAdjustVisualLayoutGuide @@ -2122,6 +2178,7 @@ export async function POST(request: NextRequest) { isScaleAdjustOperation && (useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinatePrompt || + useProviderTextOnlyCoordinatePrompt || (useScaleAdjustMarkedPhotoGuide && !useFalScaleAdjustCoordinatePrompt)); const scaleAdjustCropReferences: ScaleAdjustCropReference[] = shouldCreateScaleAdjustCropReferences @@ -2133,7 +2190,14 @@ export async function POST(request: NextRequest) { }) : []; const prompt = isScaleAdjustOperation - ? useScaleAdjustCoordinatePrompt + ? useProviderTextOnlyCoordinatePrompt + ? buildScaleAdjustTextOnlyPrompt( + regions, + scalePercent, + scaleAdjustCropReferences, + { width: sourceWidth, height: sourceHeight } + ) + : useScaleAdjustCoordinatePrompt ? buildScaleAdjustCoordinatePrompt( regions, scalePercent, @@ -2167,7 +2231,9 @@ export async function POST(request: NextRequest) { const referenceImages = isScaleAdjustOperation ? [ sourceImageUrl, - annotatedUpload.url, + ...(sendScaleAdjustAnnotatedReferenceToModel + ? [annotatedUpload.url] + : []), ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] @@ -2209,9 +2275,11 @@ export async function POST(request: NextRequest) { operation === SCALE_ADJUST_OPERATION ? "canvas-scale-adjust" : "canvas-local-edit", debugMeta: { annotatedReferenceUrl: annotatedUpload.url, - annotatedReferenceSentToModel: isScaleAdjustOperation, + annotatedReferenceSentToModel: sendScaleAdjustAnnotatedReferenceToModel, annotatedReferenceType: isScaleAdjustOperation - ? useScaleAdjustCoordinatePrompt + ? useProviderTextOnlyCoordinatePrompt + ? "scale-adjust-text-coordinate-only-debug-map" + : useScaleAdjustCoordinatePrompt ? useScaleAdjustVisualLayoutGuide ? "scale-adjust-visual-layout-guide" : useScaleAdjustMarkedPhotoGuide @@ -2238,7 +2306,10 @@ export async function POST(request: NextRequest) { resolvedScaleAdjustModelEntry?.provider?.type || null, scaleAdjustModelFamily: scaleAdjustModelFamily || null, scaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide, - scaleAdjustCoordinatePromptApplied: useScaleAdjustCoordinatePrompt, + scaleAdjustCoordinatePromptApplied: + useScaleAdjustCoordinatePrompt || useProviderTextOnlyCoordinatePrompt, + scaleAdjustTextOnlyCoordinatePromptApplied: + useProviderTextOnlyCoordinatePrompt, scaleAdjustNeutralCoordinateMapApplied: isScaleAdjustOperation && useScaleAdjustCoordinatePrompt && @@ -2248,6 +2319,7 @@ export async function POST(request: NextRequest) { scaleAdjustFalMarkedPhotoGuideApplied: useFalScaleAdjustCoordinatePrompt, scaleAdjustVisualLayoutGuideApplied: isScaleAdjustOperation && + !useProviderTextOnlyCoordinatePrompt && (!useScaleAdjustCoordinatePrompt || useScaleAdjustVisualLayoutGuide), falScaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide && @@ -2279,8 +2351,8 @@ export async function POST(request: NextRequest) { ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" : useScaleAdjustVisualLayoutGuide ? "same-name-focused-layout-crop-guide-v54-one-shot" - : useProviderMarkedPhotoCoordinatePrompt - ? "provider-marked-photo-crop-guide-v60-one-shot" + : useProviderTextOnlyCoordinatePrompt + ? "provider-text-coordinate-crop-v61-one-shot" : useScaleAdjustMarkedPhotoGuide ? "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" @@ -2306,7 +2378,7 @@ export async function POST(request: NextRequest) { model: resolvedModel || undefined, negativePrompt: isScaleAdjustOperation - ? useScaleAdjustCoordinatePrompt + ? useScaleAdjustCoordinatePrompt || useProviderTextOnlyCoordinatePrompt ? scaleAdjustCoordinateNegativePrompt : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、淡化痕迹、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、文字位置变化、文字内容变化、文字模糊、Logo位置变化、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", From 89fca5d654742fdf2071c58090bbc406a6316ccc Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 21:06:20 +0800 Subject: [PATCH 28/73] fix: avoid visual terms in alibaba scale prompt --- app/api/canvas/local-edit/route.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index f42716b0c..49bea562e 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1900,7 +1900,7 @@ function buildScaleAdjustTextOnlyPrompt( cropIndex >= 0 ? ` Input image ${cropIndex + 2} is the clean crop reference for this furniture identity and silhouette.` : ""; - return `Region ${region.index}: source search box left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; target final footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || "scale the furniture to the target footprint"}.${cropNote}`; + return `Region ${region.index}: hidden coordinate set A current area x0=${region.pixelRect.left}, y0=${region.pixelRect.top}, x1=${sourceRight}, y1=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; hidden coordinate set B final size/location x0=${targetPixelRect.left}, y0=${targetPixelRect.top}, x1=${targetRight}, y1=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; final visual scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || "make the same furniture match set B"}.${cropNote}`; }) .join("\n"); @@ -1908,17 +1908,17 @@ function buildScaleAdjustTextOnlyPrompt( "Local furniture scale edit. Input image 1 is the locked source photo and the only final canvas base.", `Return one clean realistic photo with exactly the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, cropLines ? cropLines : "", - "Do not output any guide marks, boxes, labels, arrows, blue lines, cyan lines, overlays, UI marks, text annotations, patch edges, full-frame zoom, crop, pan, or redesigned room.", - "Only edit the main movable furniture subject inside each source search box. Keep all other furniture, sofa, lamps, wall, windows, rug, floor, decor, text, lighting, and camera frame stable.", - "Use the numeric pixel coordinates below as the only geometry guide. Do not need a visible guide image.", - "When target footprint is smaller than source, make the same furniture visibly smaller and naturally rebuild the newly visible floor, rug, wall, shadow, or background around it. Do not leave the old full-size footprint.", + "The final photo must contain only natural scene content. Never add artificial graphic overlays, visible coordinate aids, labels, lines, UI elements, or text.", + "Only edit the main movable furniture subject inside each current pixel area. Keep all other furniture, sofa, lamps, wall, windows, rug, floor, decor, lighting, and camera frame stable.", + "Use the numeric pixel sets below as hidden instructions only; they are not visual content and must not be drawn.", + "When set B is smaller than set A, make the same furniture visibly smaller and naturally rebuild the newly visible floor, rug, wall, shadow, or background around it. Do not leave the old full-size furniture silhouette.", "Keep the furniture identity, materials, perspective, objects on top, contact shadows, and natural aspect ratio. Do not stretch, squash, duplicate, or paste a rectangular crop.", - "中文硬约束：输入图1是锁定底图，只能调整源框里的主要家具主体；目标框是最终家具可见占位。禁止整图重绘、整图缩放、输出标注框、输出蓝线或把裁切图贴回。", + "中文硬约束：输入图1是锁定底图，只能调整 A 像素范围里的主要家具主体，让它变成 B 像素范围对应的大小和位置。A/B 坐标只是隐藏指令，最终照片里绝对不能画出任何线条、框、文字、标注或 UI 元素。", "", "Region instructions:", regionLines, "", - "Success check before returning: the selected furniture must be clearly closer to the target footprint than to the original source footprint, while the rest of the photo remains visually unchanged and clean.", + "Success check before returning: the selected furniture must be clearly closer to set B than to set A, while the rest of the photo remains visually unchanged and clean.", ].filter(Boolean).join("\n"); } @@ -2222,6 +2222,8 @@ export async function POST(request: NextRequest) { const scaleAdjustCoordinateNegativePrompt = useFalScaleAdjustCoordinatePrompt ? "guide marks, labels, boxes, arrows, overlays, patch edges, color mismatch, full-frame zoom, duplicate furniture, stretched furniture, distorted furniture" : "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, old-size target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; + const scaleAdjustTextOnlyNegativePrompt = + "artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, full image redesign, full-frame zoom, crop, pan, pasted crop, hard patch edge, color mismatch, changed sofa, changed lamp, changed wall, changed rug, duplicate furniture, stretched furniture, distorted furniture"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -2378,7 +2380,9 @@ export async function POST(request: NextRequest) { model: resolvedModel || undefined, negativePrompt: isScaleAdjustOperation - ? useScaleAdjustCoordinatePrompt || useProviderTextOnlyCoordinatePrompt + ? useProviderTextOnlyCoordinatePrompt + ? scaleAdjustTextOnlyNegativePrompt + : useScaleAdjustCoordinatePrompt ? scaleAdjustCoordinateNegativePrompt : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、淡化痕迹、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、文字位置变化、文字内容变化、文字模糊、Logo位置变化、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", From 90a9edcd56f92415ef743921e6de2501d18e54be Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 21:31:32 +0800 Subject: [PATCH 29/73] fix: edit scale adjust through ai patch workspace --- app/api/canvas/local-edit/route.ts | 285 +++++++++++++++++++++++++++-- lib/canvas/generate-image.ts | 224 ++++++++++++++++++++++- 2 files changed, 491 insertions(+), 18 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 49bea562e..1b793fed0 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -101,6 +101,8 @@ const MAX_REFERENCE_IMAGES_PER_REGION = 3; const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; const SCALE_ADJUST_CROP_MAX_EDGE = 1536; +const SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN = 0.045; +const SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN = 0.18; const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb"; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; @@ -112,6 +114,17 @@ type ScaleAdjustCropReference = { regionIndex: number; url: string; }; +type ScaleAdjustPatchReference = { + sourceCropUrl: string; + targetDraftUrl: string; + workspaceRect: NormalizedLocalEditRegion["rect"]; + workspacePixelRect: { + left: number; + top: number; + width: number; + height: number; + }; +}; type AiModelWithProviderForSelection = Awaited< ReturnType >[number]; @@ -711,6 +724,39 @@ function buildScaleAdjustCompositeRegions( }); } +function getScaleAdjustPatchWorkspaceRect( + regions: NormalizedLocalEditRegion[], + scalePercent?: ScaleAdjustPercent | null +) { + const workspaceRects = regions.map((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const baseWorkspace = getScaleAdjustWorkspaceRect( + region.rect, + targetRect, + scalePercent + ); + const marginX = Math.min( + SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN, + Math.max( + SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN, + baseWorkspace.width * 0.22, + region.rect.width * 0.18 + ) + ); + const marginY = Math.min( + SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN, + Math.max( + SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN, + baseWorkspace.height * 0.22, + region.rect.height * 0.18 + ) + ); + return expandRectWithMargins(baseWorkspace, marginX, marginY); + }); + + return workspaceRects.reduce((combined, rect) => getRectUnion(combined, rect)); +} + function expandPixelRectForCrop(params: { height: number; pixelRect: NormalizedLocalEditRegion["pixelRect"]; @@ -1187,6 +1233,54 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { .toBuffer(); } +async function createScaleAdjustPatchReference(params: { + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; + sourceBuffer: Buffer; + sourceHeight: number; + sourceWidth: number; +}): Promise { + const { regions, scalePercent, sourceBuffer, sourceHeight, sourceWidth } = + params; + const workspaceRect = getScaleAdjustPatchWorkspaceRect(regions, scalePercent); + const workspacePixelRect = rectToPixels(workspaceRect, sourceWidth, sourceHeight); + const targetDraftBuffer = await buildScaleAdjustVisualLayoutGuideBuffer({ + regions, + scalePercent, + sourceBuffer, + sourceHeight, + sourceWidth, + }); + const [sourceCropBuffer, targetDraftCropBuffer] = await Promise.all([ + sharp(sourceBuffer, { failOn: "none" }) + .extract(workspacePixelRect) + .png({ compressionLevel: 6 }) + .toBuffer(), + sharp(targetDraftBuffer, { failOn: "none" }) + .extract(workspacePixelRect) + .png({ compressionLevel: 6 }) + .toBuffer(), + ]); + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const [sourceCropUpload, targetDraftUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-workspace-source-${suffix}.png`, + toArrayBuffer(sourceCropBuffer) + ), + uploadFile( + `canvas-scale-adjust-workspace-target-draft-${suffix}.png`, + toArrayBuffer(targetDraftCropBuffer) + ), + ]); + + return { + sourceCropUrl: sourceCropUpload.url, + targetDraftUrl: targetDraftUpload.url, + workspacePixelRect, + workspaceRect, + }; +} + function wrapText(text: string, maxChars: number, maxLines = 3) { const compact = text.replace(/\s+/g, " ").trim(); if (!compact) return []; @@ -1922,6 +2016,98 @@ function buildScaleAdjustTextOnlyPrompt( ].filter(Boolean).join("\n"); } +function toPatchRelativePixelBox( + box: { left: number; top: number; width: number; height: number }, + workspace: ScaleAdjustPatchReference["workspacePixelRect"] +) { + const left = clamp(box.left - workspace.left, 0, Math.max(0, workspace.width - 1)); + const top = clamp(box.top - workspace.top, 0, Math.max(0, workspace.height - 1)); + const right = clamp( + box.left + box.width - workspace.left, + left + 1, + workspace.width + ); + const bottom = clamp( + box.top + box.height - workspace.top, + top + 1, + workspace.height + ); + + return { + left, + top, + width: Math.max(1, right - left), + height: Math.max(1, bottom - top), + }; +} + +function buildScaleAdjustPatchPrompt( + regions: NormalizedLocalEditRegion[], + scalePercent: ScaleAdjustPercent | null | undefined, + patchReference: ScaleAdjustPatchReference, + sourceSize: { width: number; height: number }, + cropReferences: ScaleAdjustCropReference[] = [] +) { + const cropLines = cropReferences + .map( + (reference, index) => + `Input image ${index + 3}: clean identity crop for Region ${reference.regionIndex}; use it only to recognize the target furniture shape and materials.` + ) + .join("\n"); + const regionLines = regions + .map((region) => { + const cropIndex = cropReferences.findIndex( + (reference) => reference.regionIndex === region.index + ); + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const targetPixelRect = + region.targetPixelRect || + rectToPixels(targetRect, sourceSize.width, sourceSize.height); + const sourceInPatch = toPatchRelativePixelBox( + region.pixelRect, + patchReference.workspacePixelRect + ); + const targetInPatch = toPatchRelativePixelBox( + targetPixelRect, + patchReference.workspacePixelRect + ); + const widthFactor = + region.rect.width > 0 ? targetRect.width / region.rect.width : 1; + const heightFactor = + region.rect.height > 0 ? targetRect.height / region.rect.height : 1; + const action = + widthFactor < 0.92 || heightFactor < 0.92 + ? "make the same furniture visibly smaller" + : widthFactor > 1.08 || heightFactor > 1.08 + ? "make the same furniture visibly larger" + : "move the same furniture to the final footprint"; + const cropNote = + cropIndex >= 0 + ? ` Input image ${cropIndex + 3} is the clean identity reference for this exact furniture.` + : ""; + return `Region ${region.index}: current search area inside input image 1 crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside the crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}.${cropNote}`; + }) + .join("\n"); + + return [ + "Local furniture scale edit inside a cropped workspace.", + `Return exactly one clean realistic crop with the same framing and aspect as input image 1. The crop represents full-photo pixels left=${patchReference.workspacePixelRect.left}, top=${patchReference.workspacePixelRect.top}, width=${patchReference.workspacePixelRect.width}, height=${patchReference.workspacePixelRect.height}.`, + "Input image 1 is the exact crop from the original photo and is the locked crop base.", + "Input image 2 is a rough target-layout draft of the same crop. It is not final artwork. Use it only to understand the desired furniture size, position, and newly visible background area. Remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft.", + cropLines, + "Only edit the selected movable furniture subject. Keep the crop borders, wall, window frame, floor, rug, sofa, lamps, decor, lighting, camera perspective, and all non-target furniture stable.", + "When the final footprint is smaller, the selected furniture must clearly shrink; the old larger footprint outside the final footprint must become natural background/floor/rug/wall/shadow that matches the original crop.", + "Keep the same furniture identity, material, perspective, object details, contact shadows, and natural aspect ratio. Do not stretch, squash, duplicate, or replace it with a different item.", + "Do not output boxes, lines, labels, coordinate marks, UI marks, text, arrows, overlays, rough layout artifacts, rectangular patch edges, or a full-room image.", + "中文硬约束：只输出输入图1同一裁切范围的干净局部照片；输入图2只是目标大小/位置草稿，不是要照抄的图层。目标家具必须相对原图产生明显比例变化，非目标背景和裁切边缘要稳定自然。", + "", + "Region instructions:", + regionLines, + "", + "Success check before returning: the selected furniture footprint must be clearly closer to the final footprint than the current search area, and the crop must look like one natural photograph.", + ].filter(Boolean).join("\n"); +} + export async function POST(request: NextRequest) { try { const session = await auth(); @@ -2098,8 +2284,15 @@ export async function POST(request: NextRequest) { !useProviderTextOnlyCoordinatePrompt; const useScaleAdjustMarkedPhotoGuide = useScaleAdjustCoordinatePrompt && !useScaleAdjustVisualLayoutGuide; + const useScaleAdjustEditMask = + isScaleAdjustOperation && + isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry); + const useScaleAdjustPatchWorkflow = + isScaleAdjustOperation && !useScaleAdjustEditMask; const sendScaleAdjustAnnotatedReferenceToModel = - isScaleAdjustOperation && !useProviderTextOnlyCoordinatePrompt; + isScaleAdjustOperation && + !useScaleAdjustPatchWorkflow && + !useProviderTextOnlyCoordinatePrompt; const annotatedBuffer = isScaleAdjustOperation ? useScaleAdjustCoordinatePrompt ? useScaleAdjustVisualLayoutGuide @@ -2156,9 +2349,6 @@ export async function POST(request: NextRequest) { annotatedFilename, toArrayBuffer(annotatedBuffer) ); - const useScaleAdjustEditMask = - isScaleAdjustOperation && - isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry); const scaleAdjustEditMaskUpload = useScaleAdjustEditMask ? await uploadFile( `canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random() @@ -2174,9 +2364,19 @@ export async function POST(request: NextRequest) { ) ) : null; + const scaleAdjustPatchReference = useScaleAdjustPatchWorkflow + ? await createScaleAdjustPatchReference({ + regions, + scalePercent, + sourceBuffer, + sourceHeight, + sourceWidth, + }) + : null; const shouldCreateScaleAdjustCropReferences = isScaleAdjustOperation && - (useScaleAdjustVisualLayoutGuide || + (useScaleAdjustPatchWorkflow || + useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinatePrompt || useProviderTextOnlyCoordinatePrompt || (useScaleAdjustMarkedPhotoGuide && !useFalScaleAdjustCoordinatePrompt)); @@ -2190,7 +2390,15 @@ export async function POST(request: NextRequest) { }) : []; const prompt = isScaleAdjustOperation - ? useProviderTextOnlyCoordinatePrompt + ? scaleAdjustPatchReference + ? buildScaleAdjustPatchPrompt( + regions, + scalePercent, + scaleAdjustPatchReference, + { width: sourceWidth, height: sourceHeight }, + scaleAdjustCropReferences + ) + : useProviderTextOnlyCoordinatePrompt ? buildScaleAdjustTextOnlyPrompt( regions, scalePercent, @@ -2224,6 +2432,8 @@ export async function POST(request: NextRequest) { : "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, old-size target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; const scaleAdjustTextOnlyNegativePrompt = "artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, full image redesign, full-frame zoom, crop, pan, pasted crop, hard patch edge, color mismatch, changed sofa, changed lamp, changed wall, changed rug, duplicate furniture, stretched furniture, distorted furniture"; + const scaleAdjustPatchNegativePrompt = + "full room image, changed crop border, artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, guide mark, pasted crop, hard patch edge, color mismatch, duplicate furniture, unchanged target furniture, stretched furniture, distorted furniture"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -2231,7 +2441,14 @@ export async function POST(request: NextRequest) { (reference) => reference.url ); const referenceImages = isScaleAdjustOperation - ? [ + ? scaleAdjustPatchReference + ? [ + scaleAdjustPatchReference.sourceCropUrl, + scaleAdjustPatchReference.targetDraftUrl, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] + : [ sourceImageUrl, ...(sendScaleAdjustAnnotatedReferenceToModel ? [annotatedUpload.url] @@ -2243,11 +2460,15 @@ export async function POST(request: NextRequest) { const scaleAdjustCompositeRegions = isScaleAdjustOperation ? buildScaleAdjustCompositeRegions(regions, scalePercent) : []; + const scaleAdjustProtectedRegions = + isScaleAdjustOperation && scaleAdjustPatchReference + ? [...scaleAdjustCompositeRegions, scaleAdjustPatchReference.workspaceRect] + : scaleAdjustCompositeRegions; const scaleAdjustVerificationConfig = isScaleAdjustOperation ? { allowOutsideChangedWhenChangedInside: false, allowDeterministicFallback: false, - protectedRegions: scaleAdjustCompositeRegions, + protectedRegions: scaleAdjustProtectedRegions, rejectFailedCandidates: true, sourceImageUrl, regions: regions.map((region) => region.rect), @@ -2269,9 +2490,19 @@ export async function POST(request: NextRequest) { : {}), } : undefined; + const resolvedRequestImageSize = scaleAdjustPatchReference + ? "1K" + : typeof body.imageSize === "string" && body.imageSize.trim() + ? body.imageSize.trim() + : "2K"; const imageRequest: CanvasGenerateImageRequest = { allowFailover: isScaleAdjustOperation ? false : undefined, - aspectRatio: resolvedAspectRatio, + aspectRatio: scaleAdjustPatchReference + ? inferClosestAspectRatio( + scaleAdjustPatchReference.workspacePixelRect.width, + scaleAdjustPatchReference.workspacePixelRect.height + ) + : resolvedAspectRatio, billingFeature: "repaint", debugLabel: operation === SCALE_ADJUST_OPERATION ? "canvas-scale-adjust" : "canvas-local-edit", @@ -2303,6 +2534,15 @@ export async function POST(request: NextRequest) { operation, requestedModel: requestedModel || null, resolvedAspectRatio, + scaleAdjustPatchWorkflowApplied: Boolean(scaleAdjustPatchReference), + scaleAdjustPatchSourceCropUrl: + scaleAdjustPatchReference?.sourceCropUrl || null, + scaleAdjustPatchTargetDraftUrl: + scaleAdjustPatchReference?.targetDraftUrl || null, + scaleAdjustPatchWorkspacePixelRect: + scaleAdjustPatchReference?.workspacePixelRect || null, + scaleAdjustPatchWorkspaceRect: + scaleAdjustPatchReference?.workspaceRect || null, resolvedModel: resolvedModel || null, resolvedScaleAdjustProviderType: resolvedScaleAdjustModelEntry?.provider?.type || null, @@ -2349,7 +2589,9 @@ export async function POST(request: NextRequest) { scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) .length, scaleAdjustPromptStrategy: isScaleAdjustOperation - ? useFalScaleAdjustCoordinatePrompt + ? scaleAdjustPatchReference + ? "ai-patch-workspace-draft-v70-one-shot" + : useFalScaleAdjustCoordinatePrompt ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" : useScaleAdjustVisualLayoutGuide ? "same-name-focused-layout-crop-guide-v54-one-shot" @@ -2359,8 +2601,9 @@ export async function POST(request: NextRequest) { ? "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, - scaleAdjustPostCompositeApplied: false, - scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, + scaleAdjustPostCompositeApplied: Boolean(scaleAdjustPatchReference), + scaleAdjustUsesFullFrameOutput: + isScaleAdjustOperation && !scaleAdjustPatchReference, sourceHeight, sourceNodeId: body.sourceNodeId || null, sourceWidth, @@ -2373,14 +2616,13 @@ export async function POST(request: NextRequest) { : undefined, falThinkingLevel: useScaleAdjustCoordinateGuide ? "high" : undefined, imageCount: 1, - imageSize: - typeof body.imageSize === "string" && body.imageSize.trim() - ? body.imageSize.trim() - : "2K", + imageSize: resolvedRequestImageSize, model: resolvedModel || undefined, negativePrompt: isScaleAdjustOperation - ? useProviderTextOnlyCoordinatePrompt + ? scaleAdjustPatchReference + ? scaleAdjustPatchNegativePrompt + : useProviderTextOnlyCoordinatePrompt ? scaleAdjustTextOnlyNegativePrompt : useScaleAdjustCoordinatePrompt ? scaleAdjustCoordinateNegativePrompt @@ -2392,6 +2634,15 @@ export async function POST(request: NextRequest) { ? { maskImageUrl: scaleAdjustEditMaskUpload.url } : undefined, referenceImages, + scaleAdjustPatchComposite: scaleAdjustPatchReference + ? { + featherMaxPx: 42, + featherMinPx: 10, + featherRatio: 0.018, + sourceImageUrl, + workspaceRegion: scaleAdjustPatchReference.workspaceRect, + } + : undefined, scaleAdjustVerification: scaleAdjustVerificationConfig, localEditComposite: isScaleAdjustOperation ? undefined diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 151146cb2..6e54cc6d0 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -52,6 +52,14 @@ export interface CanvasLocalEditCompositeConfig { diffCompareBlurPx?: number; } +export interface CanvasScaleAdjustPatchCompositeConfig { + sourceImageUrl: string; + workspaceRegion: CanvasLocalEditCompositeRegion; + featherMinPx?: number; + featherMaxPx?: number; + featherRatio?: number; +} + export interface CanvasScaleAdjustVerificationConfig { allowDeterministicFallback?: boolean; allowOutsideChangedWhenChangedInside?: boolean; @@ -79,6 +87,7 @@ export interface CanvasGenerateImageRetryVariant { providerImageParams?: Record; referenceImage?: string; referenceImages?: string[]; + scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; } @@ -102,6 +111,7 @@ export interface CanvasGenerateImageRequest { debugLabel?: string; debugMeta?: Record; localEditComposite?: CanvasLocalEditCompositeConfig; + scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; retryVariants?: CanvasGenerateImageRetryVariant[]; internalRetryVariantIndex?: number; @@ -352,6 +362,33 @@ function normalizeLocalEditCompositeConfig( : null; } +function normalizeScaleAdjustPatchCompositeConfig( + value: CanvasGenerateImageRequest["scaleAdjustPatchComposite"] +): CanvasScaleAdjustPatchCompositeConfig | null { + if (!value || typeof value !== "object") return null; + const sourceImageUrl = String(value.sourceImageUrl || "").trim(); + if (!/^https?:\/\//i.test(sourceImageUrl)) return null; + const workspaceRegion = normalizeCanvasRegion(value.workspaceRegion); + if (!workspaceRegion) return null; + + return { + sourceImageUrl, + workspaceRegion, + featherMaxPx: normalizeOptionalNumber(value.featherMaxPx, { + max: 180, + min: 0, + }), + featherMinPx: normalizeOptionalNumber(value.featherMinPx, { + max: 100, + min: 0, + }), + featherRatio: normalizeOptionalNumber(value.featherRatio, { + max: 0.12, + min: 0, + }), + }; +} + function normalizeCanvasRegion(region: unknown): CanvasLocalEditCompositeRegion | null { if (!region || typeof region !== "object") return null; const raw = region as Partial; @@ -1896,6 +1933,54 @@ function buildLocalEditCompositeMaskSvg(params: { `); } +function buildScaleAdjustPatchCompositeMaskSvg(params: { + width: number; + height: number; + featherPx: number; + touchesLeft: boolean; + touchesTop: boolean; + touchesRight: boolean; + touchesBottom: boolean; +}) { + const { height, width } = params; + const featherPx = Math.max(0, Math.round(params.featherPx)); + if (featherPx <= 0) { + return Buffer.from(` + + + + `); + } + + const spill = Math.max(2, Math.round(featherPx * 2.5)); + const softLeft = params.touchesLeft ? -spill : 0; + const softTop = params.touchesTop ? -spill : 0; + const softRight = params.touchesRight ? width + spill : width; + const softBottom = params.touchesBottom ? height + spill : height; + const coreInset = Math.max(0, Math.round(featherPx * 1.25)); + const coreLeft = params.touchesLeft ? 0 : Math.min(coreInset, Math.floor(width / 3)); + const coreTop = params.touchesTop ? 0 : Math.min(coreInset, Math.floor(height / 3)); + const coreRight = params.touchesRight + ? width + : Math.max(coreLeft + 1, width - Math.min(coreInset, Math.floor(width / 3))); + const coreBottom = params.touchesBottom + ? height + : Math.max(coreTop + 1, height - Math.min(coreInset, Math.floor(height / 3))); + + return Buffer.from(` + + + + + + + + + + + `); +} + function resolveLocalEditCompositeFeatherPx(params: { composite: CanvasLocalEditCompositeConfig; width: number; @@ -2131,6 +2216,133 @@ async function applyLocalEditCompositeToImages(params: { return mergedImages; } +function resolveScaleAdjustPatchCompositeFeatherPx(params: { + composite: CanvasScaleAdjustPatchCompositeConfig; + width: number; + height: number; +}) { + const { composite, height, width } = params; + const minPx = + typeof composite.featherMinPx === "number" ? composite.featherMinPx : 10; + const maxPx = + typeof composite.featherMaxPx === "number" + ? Math.max(minPx, composite.featherMaxPx) + : 42; + const ratio = + typeof composite.featherRatio === "number" ? composite.featherRatio : 0.018; + + return Math.max( + minPx, + Math.min(maxPx, Math.round(Math.min(width, height) * ratio)) + ); +} + +async function applyScaleAdjustPatchCompositeToImages(params: { + images: ParsedImage[]; + composite: CanvasScaleAdjustPatchCompositeConfig | null; + debugLabel: string; +}) { + const { images, composite, debugLabel } = params; + if (!composite || images.length === 0) return images; + + const sourceBuffer = await sharp( + await fetchImageBufferFromUrl(composite.sourceImageUrl), + { failOn: "none" } + ) + .rotate() + .png({ compressionLevel: 6 }) + .toBuffer(); + const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata(); + const sourceWidth = Number(sourceMetadata.width || 0); + const sourceHeight = Number(sourceMetadata.height || 0); + if (sourceWidth <= 0 || sourceHeight <= 0) { + throw new Error("比例调整局部结果合成失败：无法识别原图尺寸"); + } + + const workspaceBox = rectToPixelBox({ + height: sourceHeight, + region: composite.workspaceRegion, + width: sourceWidth, + }); + const featherPx = resolveScaleAdjustPatchCompositeFeatherPx({ + composite, + height: workspaceBox.height, + width: workspaceBox.width, + }); + const touchesLeft = workspaceBox.left <= 0; + const touchesTop = workspaceBox.top <= 0; + const touchesRight = workspaceBox.left + workspaceBox.width >= sourceWidth; + const touchesBottom = workspaceBox.top + workspaceBox.height >= sourceHeight; + const patchMask = buildScaleAdjustPatchCompositeMaskSvg({ + featherPx, + height: workspaceBox.height, + touchesBottom, + touchesLeft, + touchesRight, + touchesTop, + width: workspaceBox.width, + }); + + const mergedImages: ParsedImage[] = []; + for (const [index, image] of images.entries()) { + const generatedUrl = image.originalUrl || image.thumbnailUrl; + const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); + const patchOverlayBuffer = await sharp(generatedBuffer, { failOn: "none" }) + .rotate() + .resize(workspaceBox.width, workspaceBox.height, { + fit: "cover", + position: "center", + }) + .ensureAlpha() + .composite([{ input: patchMask, blend: "dest-in" }]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .ensureAlpha() + .composite([ + { + input: patchOverlayBuffer, + left: workspaceBox.left, + top: workspaceBox.top, + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + .resize(512, 512, { fit: "inside", withoutEnlargement: true }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const suffix = `${Date.now()}-${index + 1}-${Math.random() + .toString(36) + .slice(2, 8)}`; + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-ai-patch-composited-${suffix}.png`, + bufferToArrayBuffer(mergedBuffer) + ), + uploadFile( + `canvas-scale-adjust-ai-patch-composited-thumb-${suffix}.png`, + bufferToArrayBuffer(thumbnailBuffer) + ), + ]); + mergedImages.push({ + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + }); + } + + console.info("[Canvas Generate Image] Scale adjust AI patch composite applied:", { + debugLabel, + featherPx, + imageCount: mergedImages.length, + sourceHeight, + sourceWidth, + workspaceBox, + }); + + return mergedImages; +} + function getProviderBatchLimit(providerType: string) { switch (providerType) { case "ruxa-image": @@ -2561,6 +2773,9 @@ export async function generateCanvasImage( const localEditComposite = normalizeLocalEditCompositeConfig( request.localEditComposite ); + const scaleAdjustPatchComposite = normalizeScaleAdjustPatchCompositeConfig( + request.scaleAdjustPatchComposite + ); const scaleAdjustVerification = normalizeScaleAdjustVerificationConfig( request.scaleAdjustVerification ); @@ -2731,6 +2946,7 @@ export async function generateCanvasImage( providerType: provider.type, ...debugMeta, hasLocalEditComposite: Boolean(localEditComposite), + hasScaleAdjustPatchComposite: Boolean(scaleAdjustPatchComposite), hasScaleAdjustVerification: Boolean(scaleAdjustVerification), }); @@ -3058,10 +3274,16 @@ export async function generateCanvasImage( debugLabel, images: baseImages, }); + const scaleAdjustPatchCompositedImages = + await applyScaleAdjustPatchCompositeToImages({ + composite: scaleAdjustPatchComposite, + debugLabel, + images: compositedImages, + }); const finalImages = await applyScaleAdjustVerificationToImages({ debugLabel, - images: compositedImages, + images: scaleAdjustPatchCompositedImages, verification: scaleAdjustVerification, }); console.info("[Canvas Generate Image] Completed:", { From ed63eb77b7d3a6b92a6db1d927db7b5934ed97b0 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 21:46:11 +0800 Subject: [PATCH 30/73] fix: prioritize scale adjust target draft --- app/api/canvas/local-edit/route.ts | 59 ++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 1b793fed0..b9175152a 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -997,7 +997,7 @@ function buildScaleAdjustSourceFootprintWashSvg(params: { const top = Math.max(0, box.top - blur); const right = Math.min(width, box.left + box.width + blur); const bottom = Math.min(height, box.top + box.height + blur); - return ``; + return ``; }) .join(""); @@ -2046,8 +2046,18 @@ function buildScaleAdjustPatchPrompt( scalePercent: ScaleAdjustPercent | null | undefined, patchReference: ScaleAdjustPatchReference, sourceSize: { width: number; height: number }, - cropReferences: ScaleAdjustCropReference[] = [] + cropReferences: ScaleAdjustCropReference[] = [], + options: { targetDraftPrimary?: boolean } = {} ) { + const targetDraftPrimary = options.targetDraftPrimary === true; + const sourceInputIndex = targetDraftPrimary ? 2 : 1; + const draftInputIndex = targetDraftPrimary ? 1 : 2; + const firstInputRole = targetDraftPrimary + ? "Input image 1 is the rough edited target-layout crop. It is the hard layout target: the selected furniture size and center in the final output must match this draft, but the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts must be cleaned." + : "Input image 1 is the exact crop from the original photo and is the locked crop base."; + const secondInputRole = targetDraftPrimary + ? "Input image 2 is the original clean crop. Use it only for stable crop framing, background texture, lighting, perspective, and furniture identity. Do not restore the selected furniture to its old larger size from this image." + : "Input image 2 is a rough target-layout draft of the same crop. It is the hard layout target for the selected furniture size and position. Use it to understand the final footprint and newly visible background area, then remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft."; const cropLines = cropReferences .map( (reference, index) => @@ -2085,21 +2095,24 @@ function buildScaleAdjustPatchPrompt( cropIndex >= 0 ? ` Input image ${cropIndex + 3} is the clean identity reference for this exact furniture.` : ""; - return `Region ${region.index}: current search area inside input image 1 crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside the crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}.${cropNote}`; + return `Region ${region.index}: current search area inside input image ${sourceInputIndex} crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside input image ${draftInputIndex} / output crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}. The final object must be much closer to the final footprint than to the current search area.${cropNote}`; }) .join("\n"); return [ "Local furniture scale edit inside a cropped workspace.", `Return exactly one clean realistic crop with the same framing and aspect as input image 1. The crop represents full-photo pixels left=${patchReference.workspacePixelRect.left}, top=${patchReference.workspacePixelRect.top}, width=${patchReference.workspacePixelRect.width}, height=${patchReference.workspacePixelRect.height}.`, - "Input image 1 is the exact crop from the original photo and is the locked crop base.", - "Input image 2 is a rough target-layout draft of the same crop. It is not final artwork. Use it only to understand the desired furniture size, position, and newly visible background area. Remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft.", + firstInputRole, + secondInputRole, cropLines, "Only edit the selected movable furniture subject. Keep the crop borders, wall, window frame, floor, rug, sofa, lamps, decor, lighting, camera perspective, and all non-target furniture stable.", - "When the final footprint is smaller, the selected furniture must clearly shrink; the old larger footprint outside the final footprint must become natural background/floor/rug/wall/shadow that matches the original crop.", + "When the final footprint is smaller, the selected furniture must clearly shrink; erase the old larger furniture footprint outside the final footprint and replace it with natural background/floor/rug/wall/shadow that matches the original crop.", "Keep the same furniture identity, material, perspective, object details, contact shadows, and natural aspect ratio. Do not stretch, squash, duplicate, or replace it with a different item.", + "Hard failure condition: if the selected furniture remains near its original old size, the answer is wrong. The output must visibly reduce the furniture footprint to the target draft footprint.", "Do not output boxes, lines, labels, coordinate marks, UI marks, text, arrows, overlays, rough layout artifacts, rectangular patch edges, or a full-room image.", - "中文硬约束：只输出输入图1同一裁切范围的干净局部照片；输入图2只是目标大小/位置草稿，不是要照抄的图层。目标家具必须相对原图产生明显比例变化，非目标背景和裁切边缘要稳定自然。", + targetDraftPrimary + ? "中文硬约束：输入图1是目标大小/位置草稿，必须按它的小尺寸家具布局输出；输入图2只是原始质感和背景参考，绝对不要把家具恢复成输入图2里的旧大尺寸。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。" + : "中文硬约束：只输出输入图1同一裁切范围的干净局部照片；输入图2是目标大小/位置草稿，必须按它的小尺寸家具布局输出，不是要保留旧尺寸。目标家具必须相对原图产生明显比例变化，非目标背景和裁切边缘要稳定自然。", "", "Region instructions:", regionLines, @@ -2389,6 +2402,7 @@ export async function POST(request: NextRequest) { sourceWidth, }) : []; + const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const prompt = isScaleAdjustOperation ? scaleAdjustPatchReference ? buildScaleAdjustPatchPrompt( @@ -2396,7 +2410,8 @@ export async function POST(request: NextRequest) { scalePercent, scaleAdjustPatchReference, { width: sourceWidth, height: sourceHeight }, - scaleAdjustCropReferences + scaleAdjustCropReferences, + { targetDraftPrimary: useScaleAdjustPatchTargetDraftPrimary } ) : useProviderTextOnlyCoordinatePrompt ? buildScaleAdjustTextOnlyPrompt( @@ -2442,12 +2457,19 @@ export async function POST(request: NextRequest) { ); const referenceImages = isScaleAdjustOperation ? scaleAdjustPatchReference - ? [ - scaleAdjustPatchReference.sourceCropUrl, - scaleAdjustPatchReference.targetDraftUrl, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] + ? useScaleAdjustPatchTargetDraftPrimary + ? [ + scaleAdjustPatchReference.targetDraftUrl, + scaleAdjustPatchReference.sourceCropUrl, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] + : [ + scaleAdjustPatchReference.sourceCropUrl, + scaleAdjustPatchReference.targetDraftUrl, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] : [ sourceImageUrl, ...(sendScaleAdjustAnnotatedReferenceToModel @@ -2526,7 +2548,8 @@ export async function POST(request: NextRequest) { useScaleAdjustEditMask ? resolvedScaleAdjustModelEntry?.provider?.type || null : null, - cleanSourceReferenceIndex: 1, + cleanSourceReferenceIndex: + scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary ? 2 : 1, configuredDetailModelId: configuredDetailModel || null, detailFixedModelApplied: Boolean(detailFixedModel), detailFixedModelId: detailFixedModel || null, @@ -2535,6 +2558,12 @@ export async function POST(request: NextRequest) { requestedModel: requestedModel || null, resolvedAspectRatio, scaleAdjustPatchWorkflowApplied: Boolean(scaleAdjustPatchReference), + scaleAdjustPatchReferenceOrder: + scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary + ? "target-draft-source-identity" + : scaleAdjustPatchReference + ? "source-target-draft-identity" + : null, scaleAdjustPatchSourceCropUrl: scaleAdjustPatchReference?.sourceCropUrl || null, scaleAdjustPatchTargetDraftUrl: From 71fd765ebcc38b2376787843661f3b22723b0691 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 21:59:12 +0800 Subject: [PATCH 31/73] fix: refine claw markdown composer display --- components/canvas/CanvasChatPanel.tsx | 118 +++++++++++++++++++++++++- components/elements/response.tsx | 18 ++-- 2 files changed, 128 insertions(+), 8 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 04b5154fa..a03975983 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -2,7 +2,7 @@ import { useChat } from "@ai-sdk/react"; import { DefaultChatTransport } from "ai"; -import { useEffect, useRef, useState, useCallback, useMemo, type ClipboardEvent as ReactClipboardEvent } from "react"; +import { useEffect, useRef, useState, useCallback, useMemo, type ClipboardEvent as ReactClipboardEvent, type MouseEvent as ReactMouseEvent } from "react"; import { useSession } from "next-auth/react"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; import { createPortal } from "react-dom"; @@ -1222,6 +1222,86 @@ function extractOpenClawActionsFromMarkdown(content: string) { }; } +function splitOpenClawMarkdownTableCells(line: string) { + return String(line || "") + .trim() + .replace(/^\|/, "") + .replace(/\|$/, "") + .split(/(? + cell + .replace(/\\\|/g, "|") + .replace(/\*\*(.*?)\*\*/g, "$1") + .replace(/`([^`]+)`/g, "$1") + .replace(/\s+/g, "") + .trim() + ); +} + +function isOpenClawRoutingDiagnosticTable(tableLines: string[]) { + if (tableLines.length < 3) return false; + + const headerCells = splitOpenClawMarkdownTableCells(tableLines[0] || ""); + const hasDiagnosticHeader = + headerCells.some((cell) => cell.includes("已尝试")) && + headerCells.some((cell) => cell.includes("结果")); + if (!hasDiagnosticHeader) return false; + + const bodyText = tableLines + .slice(2) + .map((line) => splitOpenClawMarkdownTableCells(line).join(" ")) + .join(" "); + + return ( + bodyText.includes("标准品类识别") || + bodyText.includes("市场分析准备") || + bodyText.includes("建议示例") + ); +} + +function stripOpenClawLowValueDiagnosticTables(content: string) { + const normalized = String(content || "").replace(/\r\n/g, "\n"); + if (!normalized.includes("|")) return normalized.trim(); + + const lines = normalized.split("\n"); + const keptLines: string[] = []; + let inCodeFence = false; + + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] || ""; + if (/^\s*```/.test(line)) { + inCodeFence = !inCodeFence; + keptLines.push(line); + continue; + } + + if ( + !inCodeFence && + index < lines.length - 1 && + isLikelyMarkdownTableRow(line) && + isMarkdownTableSeparatorRow(lines[index + 1] || "") + ) { + let endIndex = index + 2; + while ( + endIndex < lines.length && + isLikelyMarkdownTableRow(lines[endIndex] || "") + ) { + endIndex += 1; + } + + const tableLines = lines.slice(index, endIndex); + if (isOpenClawRoutingDiagnosticTable(tableLines)) { + index = endIndex - 1; + continue; + } + } + + keptLines.push(line); + } + + return keptLines.join("\n").replace(/\n{3,}/g, "\n\n").trim(); +} + type MarkdownTableBlockRange = { start: number; end: number; @@ -11568,6 +11648,35 @@ export function CanvasChatPanel({ (selectedSkillTag ? `请输入「${selectedSkillTag.name}」相关需求` : "请输入你的设计需求"); + const clawdEditablePresetPlaceholder = + isClawdMode && activeClawdSkillTag + ? activeClawdSkillTag.placeholder?.trim() || "" + : ""; + const handleClawdPresetPlaceholderDoubleClick = useCallback( + (event: ReactMouseEvent) => { + if (isSubmitting || !isClawdMode || !clawdEditablePresetPlaceholder) return; + + const editor = event.currentTarget; + if (input.trim() || getComposerPlainTextFromElement(editor)) return; + + event.preventDefault(); + event.stopPropagation(); + composerUserEditedAtRef.current = Date.now(); + setInput(clawdEditablePresetPlaceholder); + editor.textContent = clawdEditablePresetPlaceholder; + editor.focus(); + + const selection = window.getSelection(); + if (!selection) return; + + const range = document.createRange(); + range.selectNodeContents(editor); + range.collapse(false); + selection.removeAllRanges(); + selection.addRange(range); + }, + [clawdEditablePresetPlaceholder, input, isClawdMode, isSubmitting] + ); const detailProductCount = DETAIL_PRODUCT_SLOT_INDEXES.filter((index) => Boolean(detailSlots[index]?.url)).length; const detailModelReady = DETAIL_MODEL_SLOT_INDEX >= 0 ? Boolean(detailSlots[DETAIL_MODEL_SLOT_INDEX]?.url) : false; const detailBundleCount = DETAIL_BUNDLE_SLOT_INDEXES.filter((index) => Boolean(detailSlots[index]?.url)).length; @@ -12805,7 +12914,11 @@ export function CanvasChatPanel({ const openClawActionsResult = isClawAssistantMessage ? extractOpenClawActionsFromMarkdown(textWithoutImages) : { markdown: textWithoutImages, blocks: [] }; - const textWithoutActions = openClawActionsResult.markdown; + const textWithoutActions = isClawAssistantMessage + ? stripOpenClawLowValueDiagnosticTables( + openClawActionsResult.markdown + ) + : openClawActionsResult.markdown; const openClawActionBlocks = openClawActionsResult.blocks; const analysisSection = extractCanvasImageAnalysisSection(textWithoutActions); const plainText = normalizeCanvasAgentText(textWithoutActions); @@ -14457,6 +14570,7 @@ export function CanvasChatPanel({ : "text-[#1f2328] empty:before:text-[#a8abb2] dark:text-zinc-100 dark:empty:before:text-zinc-500" )} data-placeholder={composerPlaceholder} + onDoubleClick={handleClawdPresetPlaceholderDoubleClick} onInput={(e) => { // 从 contenteditable 中提取纯文本（排除附件标签文本） const target = e.target as HTMLDivElement; diff --git a/components/elements/response.tsx b/components/elements/response.tsx index a753ee2f7..c5f55bf87 100644 --- a/components/elements/response.tsx +++ b/components/elements/response.tsx @@ -571,10 +571,10 @@ function DownloadableMarkdownTable({ ); @@ -606,7 +609,10 @@ function MarkdownTableHeaderCell({ }: MarkdownTableHeaderCellProps) { return ( ); } @@ -701,23 +830,31 @@ export const Response = ({ } return ( - + + {(isInsideTable) => + isInsideTable ? ( + + ) : ( + + ) + } + ); }, video: ({ src, alt }: { src?: string; alt?: string }) => { @@ -736,22 +873,33 @@ export const Response = ({ onClick, ...linkProps }: ComponentProps<"a">) => ( - { - event.stopPropagation(); - onClick?.(event); - }} - rel="noreferrer" - target="_blank" - > - {children} - + + {(isInsideTable) => + isInsideTable && isLikelyImageUrl(href) ? ( + + ) : ( + { + event.stopPropagation(); + onClick?.(event); + }} + rel="noreferrer" + target="_blank" + > + {children} + + ) + } + ), }; @@ -778,7 +926,11 @@ export const Response = ({ const renderTextSegment = (text: string, keyPrefix: string): ReactNode => { if (!text.trim()) return null; - if (text.includes("![") && text.includes("](")) { + if ( + !MARKDOWN_TABLE_REGEX.test(text) && + text.includes("![") && + text.includes("](") + ) { const segments = splitMarkdownImages(text); const hasImageSegments = segments.some((segment) => segment.type === "image"); From 469ae7b02d48d2dd069887ae108f89d109c7e2ec Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 3 Jul 2026 00:18:52 +0800 Subject: [PATCH 39/73] feat: persist claw skill tag order --- app/api/canvas-skill-tags/route.ts | 122 +++++++++++++++- app/api/canvas/clawd-chat/route.ts | 16 +++ components/canvas/CanvasChatPanel.tsx | 194 +++++++++++++++++++++++--- lib/canvas-skill-tags.ts | 1 + 4 files changed, 311 insertions(+), 22 deletions(-) diff --git a/app/api/canvas-skill-tags/route.ts b/app/api/canvas-skill-tags/route.ts index 11f68b249..22c9ec3c0 100644 --- a/app/api/canvas-skill-tags/route.ts +++ b/app/api/canvas-skill-tags/route.ts @@ -4,26 +4,140 @@ import { auth } from "@/app/(auth)/auth"; import { db } from "@/lib/db"; import { systemSettings } from "@/lib/db/schema"; import { + CANVAS_SKILL_TAG_ORDER_SETTING_KEY_PREFIX, CANVAS_SKILL_TAGS_CONFIG_VERSION, CANVAS_SKILL_TAGS_SETTING_KEY, + type CanvasSkillTag, normalizeCanvasSkillTagSettings, } from "@/lib/canvas-skill-tags"; +function getUserSkillTagOrderKey(userId: string) { + return `${CANVAS_SKILL_TAG_ORDER_SETTING_KEY_PREFIX}${userId}`; +} + +function normalizeSkillTagOrder(value: unknown) { + const source = + value && typeof value === "object" && !Array.isArray(value) + ? (value as Record).tagIds + : value; + if (!Array.isArray(source)) return []; + + const seen = new Set(); + return source + .map((item) => String(item || "").trim()) + .filter((item) => { + if (!item || seen.has(item)) return false; + seen.add(item); + return true; + }); +} + +function applySkillTagOrder(tags: CanvasSkillTag[], order: string[]) { + if (tags.length < 2 || order.length === 0) return tags; + + const tagById = new Map(tags.map((tag) => [tag.id, tag])); + const ordered = order + .map((id) => tagById.get(id)) + .filter((tag): tag is CanvasSkillTag => Boolean(tag)); + const orderedIds = new Set(ordered.map((tag) => tag.id)); + const remaining = tags.filter((tag) => !orderedIds.has(tag.id)); + return [...ordered, ...remaining]; +} + export async function GET() { try { + const session = await auth().catch(() => null); + const userId = session?.user?.id || null; + const [setting, userOrderSetting] = await Promise.all([ + db + .select() + .from(systemSettings) + .where(eq(systemSettings.key, CANVAS_SKILL_TAGS_SETTING_KEY)) + .limit(1) + .then((rows) => rows[0]), + userId + ? db + .select() + .from(systemSettings) + .where(eq(systemSettings.key, getUserSkillTagOrderKey(userId))) + .limit(1) + .then((rows) => rows[0]) + : Promise.resolve(null), + ]); + + const settings = normalizeCanvasSkillTagSettings(setting?.value, { + requireCurrentVersion: true, + }); + const userOrder = normalizeSkillTagOrder(userOrderSetting?.value); + return NextResponse.json({ + ...settings, + tags: applySkillTagOrder(settings.tags, userOrder), + userOrder, + }); + } catch (error) { + console.error("[CanvasSkillTags API] Get error:", error); + return NextResponse.json(normalizeCanvasSkillTagSettings(null)); + } +} + +export async function PATCH(request: NextRequest) { + try { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const body = await request.json().catch(() => null); + const requestedOrder = normalizeSkillTagOrder( + body && typeof body === "object" ? (body as Record).tagIds : body + ); + const [setting] = await db .select() .from(systemSettings) .where(eq(systemSettings.key, CANVAS_SKILL_TAGS_SETTING_KEY)) .limit(1); - const settings = normalizeCanvasSkillTagSettings(setting?.value, { requireCurrentVersion: true, }); - return NextResponse.json(settings); + const knownIds = new Set(settings.tags.map((tag) => tag.id)); + const orderedKnownIds = requestedOrder.filter((id) => knownIds.has(id)); + const orderedSet = new Set(orderedKnownIds); + const tagIds = [ + ...orderedKnownIds, + ...settings.tags.map((tag) => tag.id).filter((id) => !orderedSet.has(id)), + ]; + const value = { + version: 1, + tagIds, + }; + const key = getUserSkillTagOrderKey(session.user.id); + + await db + .insert(systemSettings) + .values({ + key, + value, + updatedBy: session.user.id, + updatedAt: new Date(), + }) + .onConflictDoUpdate({ + target: systemSettings.key, + set: { + value, + updatedBy: session.user.id, + updatedAt: new Date(), + }, + }); + + return NextResponse.json({ + success: true, + userOrder: tagIds, + tags: applySkillTagOrder(settings.tags, tagIds), + }); } catch (error) { - console.error("[CanvasSkillTags API] Get error:", error); - return NextResponse.json(normalizeCanvasSkillTagSettings(null)); + console.error("[CanvasSkillTags API] Save order error:", error); + return NextResponse.json({ error: "Failed to save order" }, { status: 500 }); } } diff --git a/app/api/canvas/clawd-chat/route.ts b/app/api/canvas/clawd-chat/route.ts index 9df44252c..354dd11d7 100644 --- a/app/api/canvas/clawd-chat/route.ts +++ b/app/api/canvas/clawd-chat/route.ts @@ -133,6 +133,14 @@ function getOpenClawErrorCode(payload: unknown) { return typeof code === "string" ? code.trim() : ""; } +function getMetadataString( + metadata: Record | undefined, + key: string +) { + const value = metadata?.[key]; + return typeof value === "string" ? value.trim() : ""; +} + export async function POST(request: Request) { const session = await auth().catch(() => null); @@ -211,6 +219,8 @@ export async function POST(request: Request) { email: userProfile?.email ?? session.user.email, }); const openClawDepartmentName = getOpenClawDepartmentName(userProfile?.department); + const skillTagId = getMetadataString(parsed.metadata, "skillTagId"); + const skillTagName = getMetadataString(parsed.metadata, "skillTagName"); const endpoint = `${config.baseUrl}/api/chat`; let upstreamPayload: unknown = null; @@ -233,6 +243,8 @@ export async function POST(request: Request) { department: openClawDepartmentName, department_name: openClawDepartmentName, user_department: openClawDepartmentName, + skill_tag_id: skillTagId || undefined, + skill_tag_name: skillTagName || undefined, message: upstreamMessage, metadata: { source: "ai_chatbot_canvas", @@ -245,6 +257,8 @@ export async function POST(request: Request) { department: openClawDepartmentName, department_name: openClawDepartmentName, user_department: openClawDepartmentName, + skill_tag_id: skillTagId || undefined, + skill_tag_name: skillTagName || undefined, spreadsheet_files: spreadsheetMarkdownResult?.files || undefined, spreadsheet_markdown_truncated: spreadsheetMarkdownResult?.truncated || undefined, @@ -326,6 +340,8 @@ export async function POST(request: Request) { videoMetadata: { composerMode: "clawd", source: "openclaw", + skillTagId: skillTagId || null, + skillTagName: skillTagName || null, }, }; const assistantMessage = { diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index a03975983..4070b7f0c 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -2,7 +2,7 @@ import { useChat } from "@ai-sdk/react"; import { DefaultChatTransport } from "ai"; -import { useEffect, useRef, useState, useCallback, useMemo, type ClipboardEvent as ReactClipboardEvent, type MouseEvent as ReactMouseEvent } from "react"; +import { useEffect, useRef, useState, useCallback, useMemo, type ClipboardEvent as ReactClipboardEvent, type DragEvent as ReactDragEvent, type MouseEvent as ReactMouseEvent } from "react"; import { useSession } from "next-auth/react"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; import { createPortal } from "react-dom"; @@ -418,6 +418,24 @@ function buildClawdSkillTagPrompt(tag: Pick | ].join("\n"); } +function reorderCanvasSkillTags( + tags: CanvasSkillTag[], + draggedId: string, + targetId: string +) { + if (!draggedId || !targetId || draggedId === targetId) return tags; + const currentIndex = tags.findIndex((tag) => tag.id === draggedId); + const targetIndex = tags.findIndex((tag) => tag.id === targetId); + if (currentIndex < 0 || targetIndex < 0) return tags; + + const nextTags = [...tags]; + const [draggedTag] = nextTags.splice(currentIndex, 1); + if (!draggedTag) return tags; + const nextTargetIndex = nextTags.findIndex((tag) => tag.id === targetId); + nextTags.splice(nextTargetIndex < 0 ? targetIndex : nextTargetIndex, 0, draggedTag); + return nextTags; +} + type ClawdComposerSubmissionSnapshot = { chatId: string; canvasId?: string; @@ -2166,6 +2184,10 @@ export function CanvasChatPanel({ const [isComposerModeMenuOpen, setIsComposerModeMenuOpen] = useState(false); const [selectedSkillTagId, setSelectedSkillTagId] = useState(null); const [areCanvasSkillTagsExpanded, setAreCanvasSkillTagsExpanded] = useState(false); + const [draggingSkillTagId, setDraggingSkillTagId] = useState(null); + const pendingSelectedClawdSkillTagRef = useRef(null); + const draggingSkillTagIdRef = useRef(null); + const suppressSkillTagClickRef = useRef(false); const [messageViewportKey, setMessageViewportKey] = useState(0); const messagesEndRef = useRef(null); const fileInputRef = useRef(null); @@ -3948,6 +3970,7 @@ export function CanvasChatPanel({ enabled?: boolean; title?: string; tags?: CanvasSkillTag[]; + userOrder?: string[]; }>( "/api/canvas-skill-tags", fetcher, @@ -4136,6 +4159,42 @@ export function CanvasChatPanel({ () => (canvasSkillTagsEnabled ? normalizeCanvasSkillTags(canvasSkillTagsData?.tags) : []), [canvasSkillTagsData?.tags, canvasSkillTagsEnabled] ); + const handleReorderCanvasSkillTag = useCallback( + async (draggedId: string, targetId: string) => { + const nextTags = reorderCanvasSkillTags(canvasSkillTags, draggedId, targetId); + if (nextTags === canvasSkillTags) return; + + const userOrder = nextTags.map((tag) => tag.id); + const optimisticData = { + ...(canvasSkillTagsData || {}), + enabled: canvasSkillTagsData?.enabled, + title: canvasSkillTagsData?.title, + tags: nextTags, + userOrder, + }; + void mutate("/api/canvas-skill-tags", optimisticData, false); + + try { + const response = await fetch("/api/canvas-skill-tags", { + method: "PATCH", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ tagIds: userOrder }), + }); + if (!response.ok) { + throw new Error("Failed to save skill tag order"); + } + void mutate("/api/canvas-skill-tags"); + toast.success("快捷标签顺序已保存"); + } catch (error) { + console.error("[CanvasChat] 保存快捷标签顺序失败:", error); + toast.error("快捷标签顺序保存失败"); + void mutate("/api/canvas-skill-tags"); + } + }, + [canvasSkillTags, canvasSkillTagsData, mutate] + ); const selectedSkillTag = useMemo( () => canvasSkillTags.find((tag) => tag.id === selectedSkillTagId) || null, [canvasSkillTags, selectedSkillTagId] @@ -4177,13 +4236,19 @@ export function CanvasChatPanel({ if (!selectedSkillTagId) return; if (!canvasSkillTags.some((tag) => tag.id === selectedSkillTagId)) { setSelectedSkillTagId(null); + pendingSelectedClawdSkillTagRef.current = null; } }, [canvasSkillTags, selectedSkillTagId]); + useEffect(() => { + pendingSelectedClawdSkillTagRef.current = selectedSkillTag; + }, [selectedSkillTag]); + useEffect(() => { if (!isWorkspacePresentation) return; if (!selectedSkillTagId) return; setSelectedSkillTagId(null); + pendingSelectedClawdSkillTagRef.current = null; }, [isWorkspacePresentation, selectedSkillTagId]); const visualModels = useMemo(() => { @@ -5314,6 +5379,9 @@ export function CanvasChatPanel({ const handleSelectCanvasSkillTag = useCallback( (tag: CanvasSkillTag) => { + if (suppressSkillTagClickRef.current) { + return; + } if (!isClawdModeEnabled) { toast.error(`${CLAW_MODE_LABEL} 未开启`); return; @@ -5331,6 +5399,7 @@ export function CanvasChatPanel({ setSessionClawdSkillTag(targetChatSessionId, tag.id, { promptApplied: false, }); + pendingSelectedClawdSkillTagRef.current = tag; window.setTimeout(() => { setSelectedSkillTagId(tag.id); const editor = textareaRef.current as HTMLDivElement | null; @@ -5348,6 +5417,49 @@ export function CanvasChatPanel({ ] ); + const handleCanvasSkillTagDragStart = useCallback( + (event: ReactDragEvent, tagId: string) => { + draggingSkillTagIdRef.current = tagId; + setDraggingSkillTagId(tagId); + event.dataTransfer.effectAllowed = "move"; + event.dataTransfer.setData("text/plain", tagId); + }, + [] + ); + + const handleCanvasSkillTagDragOver = useCallback( + (event: ReactDragEvent, targetId: string) => { + const draggedId = draggingSkillTagIdRef.current; + if (!draggedId || draggedId === targetId) return; + event.preventDefault(); + event.dataTransfer.dropEffect = "move"; + }, + [] + ); + + const handleCanvasSkillTagDrop = useCallback( + (event: ReactDragEvent, targetId: string) => { + event.preventDefault(); + const draggedId = + event.dataTransfer.getData("text/plain") || draggingSkillTagIdRef.current; + draggingSkillTagIdRef.current = null; + setDraggingSkillTagId(null); + if (!draggedId || draggedId === targetId) return; + + suppressSkillTagClickRef.current = true; + window.setTimeout(() => { + suppressSkillTagClickRef.current = false; + }, 0); + void handleReorderCanvasSkillTag(draggedId, targetId); + }, + [handleReorderCanvasSkillTag] + ); + + const handleCanvasSkillTagDragEnd = useCallback(() => { + draggingSkillTagIdRef.current = null; + setDraggingSkillTagId(null); + }, []); + useEffect(() => { if (!effectiveAgentEnabled || typeof window === "undefined") return; const targetWindow = window as CanvasAgentPendingRequestWindow; @@ -10191,13 +10303,18 @@ export function CanvasChatPanel({ const textContent = normalizeClawdComposerText(composerInput); const tableFilesSnapshot = isProgrammaticClawdQuickSend ? [] : [...clawdTableFiles]; const hasTableFiles = tableFilesSnapshot.length > 0; + const pendingSelectedClawdSkillTag = pendingSelectedClawdSkillTagRef.current; const clawdSkillTagForRequest = isProgrammaticClawdQuickSend ? activeClawdSkillTag - : selectedSkillTag || activeClawdSkillTag; + : selectedSkillTag || pendingSelectedClawdSkillTag || activeClawdSkillTag; + const hasFreshClawdSkillTagSelection = Boolean( + !isProgrammaticClawdQuickSend && + (selectedSkillTag || pendingSelectedClawdSkillTag) + ); const shouldIncludeClawdSkillTagPrompt = Boolean( !isProgrammaticClawdQuickSend && clawdSkillTagForRequest && - (selectedSkillTag || !activeClawdSkillTagPromptApplied) + (hasFreshClawdSkillTagSelection || !activeClawdSkillTagPromptApplied) ); const clawdSkillTagPrompt = shouldIncludeClawdSkillTagPrompt ? buildClawdSkillTagPrompt(clawdSkillTagForRequest) @@ -10234,10 +10351,17 @@ export function CanvasChatPanel({ .filter(Boolean) .join("\n\n"); const optimisticBaseMessage = - textContent || - (clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt - ? `快捷功能：${clawdSkillTagForRequest.name}` - : baseRequestMessage); + [ + clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt + ? `快捷功能：${clawdSkillTagForRequest.name}` + : "", + textContent || + (clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt + ? "" + : baseRequestMessage), + ] + .filter(Boolean) + .join("\n") || baseRequestMessage; const optimisticMessageText = [ buildClawdTableDisplayText(optimisticBaseMessage, tableFilesSnapshot), buildClawdImageDisplayText(imageAttachmentsSnapshot), @@ -10341,6 +10465,7 @@ export function CanvasChatPanel({ setAttachments([]); setClawdTableFiles([]); setSelectedSkillTagId(null); + pendingSelectedClawdSkillTagRef.current = null; clearSelection(); } @@ -10356,6 +10481,9 @@ export function CanvasChatPanel({ composerMode: "clawd", activeChatId, imageAttachmentCount: imageAttachmentsSnapshot.length || undefined, + skillTagId: clawdSkillTagForRequest?.id || undefined, + skillTagName: clawdSkillTagForRequest?.name || undefined, + skillTagPromptIncluded: hasPendingClawdSkillTagPrompt || undefined, }) ); tableFilesSnapshot.forEach((file) => { @@ -10430,6 +10558,7 @@ export function CanvasChatPanel({ setAttachments(attachmentsSnapshot); if (clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt) { setSelectedSkillTagId(clawdSkillTagForRequest.id); + pendingSelectedClawdSkillTagRef.current = clawdSkillTagForRequest; } const editableDiv = textareaRef.current as HTMLDivElement | null; const currentEditorText = (editableDiv?.textContent || "") @@ -12305,7 +12434,11 @@ export function CanvasChatPanel({ )} {activeClawdSkillTag.name} - {activeClawdSkillTagPromptApplied ? "使用中" : "待带入"} + {activeClawdSkillTagPromptApplied + ? "使用中" + : isClawdSubmitting + ? "发送中" + : "待带入"} ) : null} @@ -12604,18 +12737,30 @@ export function CanvasChatPanel({ > {canvasSkillTags.map((tag) => { const isActive = selectedSkillTag?.id === tag.id; + const isDragging = draggingSkillTagId === tag.id; const TagIcon = getCanvasSkillIconComponent(tag.iconName); return ( + +
); From 0904fdf98aa16d4332f431ddfc99f3d630b82df0 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 22:01:03 +0800 Subject: [PATCH 32/73] fix: avoid source crop reversion in scale adjust --- app/api/canvas/local-edit/route.ts | 45 ++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index b9175152a..84c395c12 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2047,21 +2047,28 @@ function buildScaleAdjustPatchPrompt( patchReference: ScaleAdjustPatchReference, sourceSize: { width: number; height: number }, cropReferences: ScaleAdjustCropReference[] = [], - options: { targetDraftPrimary?: boolean } = {} + options: { includeSourceReference?: boolean; targetDraftPrimary?: boolean } = {} ) { const targetDraftPrimary = options.targetDraftPrimary === true; - const sourceInputIndex = targetDraftPrimary ? 2 : 1; + const includeSourceReference = options.includeSourceReference !== false; + const sourceInputIndex = targetDraftPrimary && includeSourceReference ? 2 : 1; const draftInputIndex = targetDraftPrimary ? 1 : 2; + const cropReferenceStartIndex = + targetDraftPrimary && !includeSourceReference ? 2 : 3; const firstInputRole = targetDraftPrimary - ? "Input image 1 is the rough edited target-layout crop. It is the hard layout target: the selected furniture size and center in the final output must match this draft, but the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts must be cleaned." + ? includeSourceReference + ? "Input image 1 is the rough edited target-layout crop. It is the hard layout target: the selected furniture size and center in the final output must match this draft, but the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts must be cleaned." + : "Input image 1 is the rough edited target-layout crop and the only crop base. It is the hard layout target: the selected furniture size and center in the final output must match this draft. Clean away the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts." : "Input image 1 is the exact crop from the original photo and is the locked crop base."; const secondInputRole = targetDraftPrimary - ? "Input image 2 is the original clean crop. Use it only for stable crop framing, background texture, lighting, perspective, and furniture identity. Do not restore the selected furniture to its old larger size from this image." + ? includeSourceReference + ? "Input image 2 is the original clean crop. Use it only for stable crop framing, background texture, lighting, perspective, and furniture identity. Do not restore the selected furniture to its old larger size from this image." + : "" : "Input image 2 is a rough target-layout draft of the same crop. It is the hard layout target for the selected furniture size and position. Use it to understand the final footprint and newly visible background area, then remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft."; const cropLines = cropReferences .map( (reference, index) => - `Input image ${index + 3}: clean identity crop for Region ${reference.regionIndex}; use it only to recognize the target furniture shape and materials.` + `Input image ${index + cropReferenceStartIndex}: clean identity crop for Region ${reference.regionIndex}; use it only to recognize the target furniture shape and materials, not its old size.` ) .join("\n"); const regionLines = regions @@ -2093,7 +2100,7 @@ function buildScaleAdjustPatchPrompt( : "move the same furniture to the final footprint"; const cropNote = cropIndex >= 0 - ? ` Input image ${cropIndex + 3} is the clean identity reference for this exact furniture.` + ? ` Input image ${cropIndex + cropReferenceStartIndex} is the clean identity reference for this exact furniture.` : ""; return `Region ${region.index}: current search area inside input image ${sourceInputIndex} crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside input image ${draftInputIndex} / output crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}. The final object must be much closer to the final footprint than to the current search area.${cropNote}`; }) @@ -2111,7 +2118,9 @@ function buildScaleAdjustPatchPrompt( "Hard failure condition: if the selected furniture remains near its original old size, the answer is wrong. The output must visibly reduce the furniture footprint to the target draft footprint.", "Do not output boxes, lines, labels, coordinate marks, UI marks, text, arrows, overlays, rough layout artifacts, rectangular patch edges, or a full-room image.", targetDraftPrimary - ? "中文硬约束：输入图1是目标大小/位置草稿，必须按它的小尺寸家具布局输出；输入图2只是原始质感和背景参考，绝对不要把家具恢复成输入图2里的旧大尺寸。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。" + ? includeSourceReference + ? "中文硬约束：输入图1是目标大小/位置草稿，必须按它的小尺寸家具布局输出；输入图2只是原始质感和背景参考，绝对不要把家具恢复成输入图2里的旧大尺寸。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。" + : "中文硬约束：输入图1是唯一的裁切底图和目标大小/位置草稿，必须按它的小尺寸家具布局输出；后续家具识别图只看材质形状，不看旧尺寸。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。" : "中文硬约束：只输出输入图1同一裁切范围的干净局部照片；输入图2是目标大小/位置草稿，必须按它的小尺寸家具布局输出，不是要保留旧尺寸。目标家具必须相对原图产生明显比例变化，非目标背景和裁切边缘要稳定自然。", "", "Region instructions:", @@ -2403,6 +2412,7 @@ export async function POST(request: NextRequest) { }) : []; const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); + const includeScaleAdjustPatchSourceReference = false; const prompt = isScaleAdjustOperation ? scaleAdjustPatchReference ? buildScaleAdjustPatchPrompt( @@ -2411,7 +2421,10 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference, { width: sourceWidth, height: sourceHeight }, scaleAdjustCropReferences, - { targetDraftPrimary: useScaleAdjustPatchTargetDraftPrimary } + { + includeSourceReference: includeScaleAdjustPatchSourceReference, + targetDraftPrimary: useScaleAdjustPatchTargetDraftPrimary, + } ) : useProviderTextOnlyCoordinatePrompt ? buildScaleAdjustTextOnlyPrompt( @@ -2460,7 +2473,9 @@ export async function POST(request: NextRequest) { ? useScaleAdjustPatchTargetDraftPrimary ? [ scaleAdjustPatchReference.targetDraftUrl, - scaleAdjustPatchReference.sourceCropUrl, + ...(includeScaleAdjustPatchSourceReference + ? [scaleAdjustPatchReference.sourceCropUrl] + : []), ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] @@ -2549,7 +2564,13 @@ export async function POST(request: NextRequest) { ? resolvedScaleAdjustModelEntry?.provider?.type || null : null, cleanSourceReferenceIndex: - scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary ? 2 : 1, + scaleAdjustPatchReference && + useScaleAdjustPatchTargetDraftPrimary && + !includeScaleAdjustPatchSourceReference + ? null + : scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary + ? 2 + : 1, configuredDetailModelId: configuredDetailModel || null, detailFixedModelApplied: Boolean(detailFixedModel), detailFixedModelId: detailFixedModel || null, @@ -2560,7 +2581,9 @@ export async function POST(request: NextRequest) { scaleAdjustPatchWorkflowApplied: Boolean(scaleAdjustPatchReference), scaleAdjustPatchReferenceOrder: scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary - ? "target-draft-source-identity" + ? includeScaleAdjustPatchSourceReference + ? "target-draft-source-identity" + : "target-draft-identity" : scaleAdjustPatchReference ? "source-target-draft-identity" : null, From da7fbb1284c2c64225ac1475e03e71fbfd70b8c8 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 22:12:18 +0800 Subject: [PATCH 33/73] fix: send only target draft for scale patch --- app/api/canvas/local-edit/route.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 84c395c12..2dd2b7211 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -997,7 +997,7 @@ function buildScaleAdjustSourceFootprintWashSvg(params: { const top = Math.max(0, box.top - blur); const right = Math.min(width, box.left + box.width + blur); const bottom = Math.min(height, box.top + box.height + blur); - return ``; + return ``; }) .join(""); @@ -2058,7 +2058,7 @@ function buildScaleAdjustPatchPrompt( const firstInputRole = targetDraftPrimary ? includeSourceReference ? "Input image 1 is the rough edited target-layout crop. It is the hard layout target: the selected furniture size and center in the final output must match this draft, but the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts must be cleaned." - : "Input image 1 is the rough edited target-layout crop and the only crop base. It is the hard layout target: the selected furniture size and center in the final output must match this draft. Clean away the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts." + : "Input image 1 is the rough edited target-layout crop and the only visual reference. It is the hard layout target: the selected furniture size and center in the final output must match this draft. Clean away the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts." : "Input image 1 is the exact crop from the original photo and is the locked crop base."; const secondInputRole = targetDraftPrimary ? includeSourceReference @@ -2120,7 +2120,7 @@ function buildScaleAdjustPatchPrompt( targetDraftPrimary ? includeSourceReference ? "中文硬约束：输入图1是目标大小/位置草稿，必须按它的小尺寸家具布局输出；输入图2只是原始质感和背景参考，绝对不要把家具恢复成输入图2里的旧大尺寸。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。" - : "中文硬约束：输入图1是唯一的裁切底图和目标大小/位置草稿，必须按它的小尺寸家具布局输出；后续家具识别图只看材质形状，不看旧尺寸。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。" + : "中文硬约束：输入图1是唯一视觉参考和目标大小/位置草稿，必须按它的小尺寸家具布局输出。不要找回旧尺寸家具。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。" : "中文硬约束：只输出输入图1同一裁切范围的干净局部照片；输入图2是目标大小/位置草稿，必须按它的小尺寸家具布局输出，不是要保留旧尺寸。目标家具必须相对原图产生明显比例变化，非目标背景和裁切边缘要稳定自然。", "", "Region instructions:", @@ -2397,11 +2397,11 @@ export async function POST(request: NextRequest) { : null; const shouldCreateScaleAdjustCropReferences = isScaleAdjustOperation && - (useScaleAdjustPatchWorkflow || - useScaleAdjustVisualLayoutGuide || + (!scaleAdjustPatchReference && + (useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinatePrompt || useProviderTextOnlyCoordinatePrompt || - (useScaleAdjustMarkedPhotoGuide && !useFalScaleAdjustCoordinatePrompt)); + (useScaleAdjustMarkedPhotoGuide && !useFalScaleAdjustCoordinatePrompt))); const scaleAdjustCropReferences: ScaleAdjustCropReference[] = shouldCreateScaleAdjustCropReferences ? await createScaleAdjustCropReferences({ From 0bab5d1c8e54b5c98e2cf98576df5cc370c2ba1f Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 22:30:12 +0800 Subject: [PATCH 34/73] fix: prefer gpt image model for scale adjust --- app/api/canvas/local-edit/route.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 2dd2b7211..40452991d 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -394,8 +394,27 @@ function resolveScaleAdjustPreferredModelId(params: { params.models, params.configuredModelId ); - if (isScaleAdjustUsableImageEditModel(requestedModel)) { - return requestedModel?.model.id || params.requestedModelId; + const gptImage2Candidates = params.models + .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) + .filter((entry) => getScaleAdjustModelFamily(entry) === "gpt-image-2") + .sort((left, right) => + compareScaleAdjustProviderCandidates( + left, + right, + params.requestedModelId || params.configuredModelId + ) + ); + if (gptImage2Candidates[0]) { + return gptImage2Candidates[0].model.id; + } + + if ( + isScaleAdjustUsableImageEditModel(requestedModel) && + requestedModel && + getScaleAdjustModelFamily(requestedModel) !== "nano-banana-2" && + requestedModel.provider.type !== "alibaba-image" + ) { + return requestedModel.model.id || params.requestedModelId; } const autoDetailModel = params.models.find(isScaleAdjustDedicatedDetailModel); From c38003a3824d1cd98d63cd0811fc7f24adf96a4f Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 22:40:04 +0800 Subject: [PATCH 35/73] fix: allow scale shrink footprint tolerance --- lib/canvas/generate-image.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 6e54cc6d0..2c0017cfd 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -186,6 +186,7 @@ const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18; const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180; const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO = 0.72; const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60; +const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018; const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "gemini", @@ -916,7 +917,10 @@ async function getScaleAdjustFootprintChangeStats(params: { }); const targetBox = rectToPixelBox({ height: sourceHeight, - region: targetRegion, + region: expandCanvasRegion( + targetRegion, + SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN + ), width: sourceWidth, }); const sourceMask = buildScaleAdjustFallbackSubjectMask({ From c4e4981a2491e4d479d92b634614df79be967c84 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 22:40:19 +0800 Subject: [PATCH 36/73] fix: preload canvas project on route load --- .../[id]/CanvasEditorLegacyPageClient.tsx | 143 +++++++++++++++++- app/canvas/[id]/CanvasPageClientRouter.tsx | 8 +- app/canvas/[id]/page.tsx | 35 ++++- types/canvas/index.ts | 11 ++ 4 files changed, 193 insertions(+), 4 deletions(-) diff --git a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx index 7d517de98..8e501117e 100644 --- a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx +++ b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx @@ -68,6 +68,7 @@ import { useCallback, useDeferredValue, useEffect, + useLayoutEffect, useMemo, useRef, useState, @@ -243,6 +244,7 @@ import type { import type { AgentSlotNodeData, CanvasEdge, + CanvasInitialProject, CanvasNode, CanvasState, CanvasWorkspaceType, @@ -523,6 +525,7 @@ const AgentWorkflowMaterialSelectionPanel = dynamic( interface PageProps { params: Promise<{ id: string }>; + initialProject?: CanvasInitialProject | null; } // 图片尺寸预设 @@ -1798,7 +1801,10 @@ function resolveCanvasImageDisplaySize(url: string) { }); } -export default function CanvasEditorLegacyPage({ params }: PageProps) { +export default function CanvasEditorLegacyPage({ + initialProject = null, + params, +}: PageProps) { // 使用 useParams 确保客户端导航时能正确获取更新的 id const routeParams = useParams(); const searchParams = useSearchParams(); @@ -1826,6 +1832,8 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { const createNewCanvasUrl = isVideoMode ? "/video-canvas?new=1" : "/canvas?new=1"; + const initialProjectForCurrentRoute = + initialProject?.id === id ? initialProject : null; const shouldOpenBatchProcessing = searchParams.get("batch") === "1"; const goHome = useCallback(() => { if (typeof window !== "undefined") { @@ -1874,7 +1882,9 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { ); // Local state - const [isLoading, setIsLoading] = useState(true); + const [isLoading, setIsLoading] = useState( + () => !initialProjectForCurrentRoute + ); const [isSaving, setIsSaving] = useState(false); const [isEditingTitle, setIsEditingTitle] = useState(false); const [isAddImageDialogOpen, setIsAddImageDialogOpen] = useState(false); @@ -3276,11 +3286,140 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { return () => window.clearTimeout(timeoutId); }, []); + const hydrateCanvasProject = useCallback( + (project: CanvasInitialProject) => { + useCanvasStore.getState().setProject(id, project.title); + const projectUpdatedAt = String(project.updatedAt || ""); + useCanvasStore + .getState() + .setCanvasRevision(projectUpdatedAt, projectUpdatedAt); + + if (project.canvasData) { + useCanvasStore.getState().loadCanvas(project.canvasData); + const persistedWorkflow = project.canvasData.workflowState; + if (!requestedAgentMode && persistedWorkflow) { + const nextAppearanceRecognitionEnabled = + persistedWorkflow.appearanceRecognitionEnabled === true; + const nextWorkflowMaterials = nextAppearanceRecognitionEnabled + ? normalizeAgentWorkflowMaterialSelections(persistedWorkflow.materials) + : clearAppearanceSummaryFromMaterials( + normalizeAgentWorkflowMaterialSelections( + persistedWorkflow.materials + ) + ); + setWorkflowInputs({ + primarySku: String( + persistedWorkflow.inputs?.primarySku || "" + ).trim(), + secondarySku: String( + persistedWorkflow.inputs?.secondarySku || "" + ).trim(), + brand: String(persistedWorkflow.inputs?.brand || "").trim(), + store: String(persistedWorkflow.inputs?.store || "").trim(), + officialSecondCategory: String( + persistedWorkflow.inputs?.officialSecondCategory || "" + ).trim(), + officialThirdCategory: String( + persistedWorkflow.inputs?.officialThirdCategory || "" + ).trim(), + thirdCategory: String( + persistedWorkflow.inputs?.thirdCategory || "" + ).trim(), + fourthCategory: String( + persistedWorkflow.inputs?.fourthCategory || "" + ).trim(), + fifthCategory: String( + persistedWorkflow.inputs?.fifthCategory || "" + ).trim(), + priceBand: String( + persistedWorkflow.inputs?.priceBand || "" + ).trim(), + }); + setWorkflowMaterials(nextWorkflowMaterials); + setAppearanceRecognitionEnabled(nextAppearanceRecognitionEnabled); + setWorkflowSelections(persistedWorkflow.selections || {}); + setSelectedBrandModel( + persistedWorkflow.selectedBrandModel + ? { + ...persistedWorkflow.selectedBrandModel, + brandName: String( + persistedWorkflow.selectedBrandModel.brandName || "" + ).trim(), + id: String( + persistedWorkflow.selectedBrandModel.id || "" + ).trim(), + imageUrl: String( + persistedWorkflow.selectedBrandModel.imageUrl || "" + ).trim(), + assetId: + String( + persistedWorkflow.selectedBrandModel.assetId || "" + ).trim() || null, + displayName: + String( + persistedWorkflow.selectedBrandModel.displayName || "" + ).trim() || null, + mimeType: + String( + persistedWorkflow.selectedBrandModel.mimeType || "" + ).trim() || null, + sortOrder: Number.isFinite( + Number(persistedWorkflow.selectedBrandModel.sortOrder) + ) + ? Math.max( + 0, + Math.min( + 999, + Math.round( + Number( + persistedWorkflow.selectedBrandModel.sortOrder + ) + ) + ) + ) + : 0, + } + : null + ); + setSelectedAgentPresetId( + String(persistedWorkflow.selectedAgentPresetId || "").trim() || + null + ); + setCurrentAgentStage( + persistedWorkflow.currentStage === "mode" ? "mode" : "workflow" + ); + setAgentModeState( + normalizeAgentModeWorkspaceState(persistedWorkflow.agentModeState) + ); + setWorkflowHydrationSource("project"); + return; + } + } + + setWorkflowHydrationSource("empty"); + }, + [id, requestedAgentMode] + ); + + useLayoutEffect(() => { + if (!initialProjectForCurrentRoute) return; + if (lastLoadedCanvasIdRef.current === id) return; + + hydrateCanvasProject(initialProjectForCurrentRoute); + setIsLoading(false); + lastLoadedCanvasIdRef.current = id; + }, [hydrateCanvasProject, id, initialProjectForCurrentRoute]); + // 加载画布数据 useEffect(() => { console.log("[Canvas] Loading project:", id); const loadProject = async () => { + if (lastLoadedCanvasIdRef.current === id) { + setIsLoading(false); + return; + } + setIsLoading(true); if (!requestedAgentMode) { setWorkflowHydrationSource("pending"); diff --git a/app/canvas/[id]/CanvasPageClientRouter.tsx b/app/canvas/[id]/CanvasPageClientRouter.tsx index 6c4fef3fa..0eb394c91 100644 --- a/app/canvas/[id]/CanvasPageClientRouter.tsx +++ b/app/canvas/[id]/CanvasPageClientRouter.tsx @@ -4,6 +4,7 @@ import dynamic from "next/dynamic"; import { useParams, useSearchParams } from "next/navigation"; import { createContext, type ReactNode, useContext, useMemo } from "react"; import { CanvasLogoLoadingView } from "@/components/canvas/canvas-logo-loading"; +import type { CanvasInitialProject } from "@/types/canvas"; const CanvasEditorLegacyPage = dynamic(() => import("./CanvasEditorLegacyPageClient"), { loading: () => , @@ -25,6 +26,7 @@ export function CanvasPageClientRouter({ initialSiteLogoDark = null, initialSiteName = "AI Chatbot", initialWorkspaceType = "", + initialProject = null, }: { initialId: string; initialMode?: string; @@ -32,6 +34,7 @@ export function CanvasPageClientRouter({ initialSiteLogoDark?: string | null; initialSiteName?: string | null; initialWorkspaceType?: string; + initialProject?: CanvasInitialProject | null; }) { const routeParams = useParams(); const searchParams = useSearchParams(); @@ -70,7 +73,10 @@ export function CanvasPageClientRouter({ return ( - + ); } diff --git a/app/canvas/[id]/page.tsx b/app/canvas/[id]/page.tsx index 9487759c8..c6e104bbb 100644 --- a/app/canvas/[id]/page.tsx +++ b/app/canvas/[id]/page.tsx @@ -1,6 +1,11 @@ import { Suspense } from "react"; import { CanvasLogoLoadingView } from "@/components/canvas/canvas-logo-loading"; +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { canvasProject } from "@/lib/db/schema"; import { getSiteSettings } from "@/lib/db/queries"; +import { and, eq } from "drizzle-orm"; +import type { CanvasInitialProject } from "@/types/canvas"; import { CanvasPageClientRouter } from "./CanvasPageClientRouter"; type CanvasEditorPageProps = { @@ -15,13 +20,40 @@ function getFirstParamValue(value: string | string[] | undefined) { return Array.isArray(value) ? value[0] || "" : value || ""; } +async function getInitialCanvasProject(id: string): Promise { + const session = await auth(); + if (!session?.user?.id) return null; + + const [project] = await db + .select() + .from(canvasProject) + .where( + and( + eq(canvasProject.id, id), + eq(canvasProject.userId, session.user.id) + ) + ) + .limit(1); + + if (!project) return null; + + return { + ...project, + createdAt: project.createdAt.toISOString(), + updatedAt: project.updatedAt.toISOString(), + }; +} + export default async function CanvasEditorPage({ params, searchParams, }: CanvasEditorPageProps) { const { id } = await params; const resolvedSearchParams = searchParams ? await searchParams : {}; - const siteSettings = await getSiteSettings().catch(() => null); + const [siteSettings, initialProject] = await Promise.all([ + getSiteSettings().catch(() => null), + getInitialCanvasProject(id).catch(() => null), + ]); const loadingBrand = { siteLogo: siteSettings?.siteLogo, siteLogoDark: siteSettings?.siteLogoDark, @@ -39,6 +71,7 @@ export default async function CanvasEditorPage({ initialWorkspaceType={getFirstParamValue( resolvedSearchParams.workspaceType )} + initialProject={initialProject} /> ); diff --git a/types/canvas/index.ts b/types/canvas/index.ts index bb9faabd1..2683959d2 100644 --- a/types/canvas/index.ts +++ b/types/canvas/index.ts @@ -271,6 +271,17 @@ export interface CanvasProject { updatedAt: Date; } +export type CanvasInitialProject = Omit< + CanvasProject, + "canvasData" | "createdAt" | "description" | "thumbnail" | "updatedAt" +> & { + canvasData: any; + createdAt: string; + description?: string | null; + thumbnail?: string | null; + updatedAt: string; +}; + export type CanvasProjectListItem = Omit; // 附件类型 From 2ca0c54de297a5a900cd82692cc12cc5bf82b1da Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 2 Jul 2026 23:04:43 +0800 Subject: [PATCH 37/73] fix: accept visible scale shrink footprints --- lib/canvas/generate-image.ts | 38 ++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 2c0017cfd..d3906f7ef 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -185,6 +185,8 @@ const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42; const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18; const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180; const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO = 0.72; +const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO = 0.84; +const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO = 0.14; const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60; const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018; @@ -894,7 +896,10 @@ async function getScaleAdjustFootprintChangeStats(params: { generatedOutsidePixels: number; outsideRatio: number; passed: boolean; + passReason: "low-source-mask" | "strict-ratio" | "removed-old-footprint" | "failed"; regionIndex: number; + removedOutsidePixels: number; + removedOutsideRatio: number; sourceOutsidePixels: number; }> = []; @@ -952,14 +957,37 @@ async function getScaleAdjustFootprintChangeStats(params: { } const outsideRatio = sourceOutsidePixels > 0 ? generatedOutsidePixels / sourceOutsidePixels : 0; - const passed = - sourceOutsidePixels < SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS || + const removedOutsidePixels = sourceOutsidePixels - generatedOutsidePixels; + const removedOutsideRatio = + sourceOutsidePixels > 0 ? removedOutsidePixels / sourceOutsidePixels : 0; + const minRemovedOutsidePixels = Math.max( + SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS, + Math.round( + sourceOutsidePixels * SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO + ) + ); + const lowSourceMask = + sourceOutsidePixels < SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS; + const strictRatio = outsideRatio <= SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO; + const removedOldFootprint = + outsideRatio <= SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO && + removedOutsidePixels >= minRemovedOutsidePixels; + const passed = lowSourceMask || strictRatio || removedOldFootprint; shrinkStats.push({ generatedOutsidePixels, outsideRatio, passed, + passReason: lowSourceMask + ? "low-source-mask" + : strictRatio + ? "strict-ratio" + : removedOldFootprint + ? "removed-old-footprint" + : "failed", regionIndex: index, + removedOutsidePixels, + removedOutsideRatio, sourceOutsidePixels, }); } @@ -1270,7 +1298,10 @@ async function applyScaleAdjustVerificationToImages(params: { generatedOutsidePixels: item.generatedOutsidePixels, outsideRatio: Number(item.outsideRatio.toFixed(3)), passed: item.passed, + passReason: item.passReason, regionIndex: item.regionIndex, + removedOutsidePixels: item.removedOutsidePixels, + removedOutsideRatio: Number(item.removedOutsideRatio.toFixed(3)), sourceOutsidePixels: item.sourceOutsidePixels, })), }, @@ -1312,7 +1343,10 @@ async function applyScaleAdjustVerificationToImages(params: { generatedOutsidePixels: item.generatedOutsidePixels, outsideRatio: Number(item.outsideRatio.toFixed(3)), passed: item.passed, + passReason: item.passReason, regionIndex: item.regionIndex, + removedOutsidePixels: item.removedOutsidePixels, + removedOutsideRatio: Number(item.removedOutsideRatio.toFixed(3)), sourceOutsidePixels: item.sourceOutsidePixels, })), }, From c789dd7a694133f10eaec387614ff7d2f6398533 Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 3 Jul 2026 00:01:47 +0800 Subject: [PATCH 38/73] fix: constrain claw table images --- components/elements/response.tsx | 250 +++++++++++++++++++++++++------ 1 file changed, 201 insertions(+), 49 deletions(-) diff --git a/components/elements/response.tsx b/components/elements/response.tsx index c5f55bf87..5ce92c208 100644 --- a/components/elements/response.tsx +++ b/components/elements/response.tsx @@ -6,7 +6,7 @@ import type { PointerEvent as ReactPointerEvent, ReactNode, } from "react"; -import { useRef, useState } from "react"; +import { createContext, useRef, useState } from "react"; import { Streamdown } from "streamdown"; import { ImagePreview } from "@/components/image-preview"; import { @@ -15,6 +15,7 @@ import { splitOpenClawChartBlocks, } from "@/components/elements/openclaw-chart-card"; import { ResponsePdfDownloadButton } from "@/components/elements/response-pdf-download"; +import { buildImageProxyUrl } from "@/lib/image-delivery"; import { Collapsible, CollapsibleContent, @@ -78,8 +79,11 @@ type MarkdownTableHeaderCellProps = ComponentProps<"th"> & { node?: unknown; }; +const MarkdownTableContext = createContext(false); const TABLE_DRAG_INTERACTIVE_SELECTOR = "a, button, input, textarea, select, summary, [role='button'], [contenteditable='true'], [data-no-drag='true']"; +const IMAGE_URL_REGEX = /^https?:\/\/\S+\.(?:png|jpe?g|webp|gif|avif)(?:[?#]\S*)?$/i; +const MARKDOWN_IMAGE_REGEX = /!\[([^\]]*)\]\((https?:\/\/[^\s)]+)\)/i; function isTableDragInteractiveTarget(target: EventTarget | null) { return ( @@ -393,6 +397,124 @@ function splitMarkdownImages(content: string) { return segments; } +function isLikelyImageUrl(value?: string | null) { + return IMAGE_URL_REGEX.test(String(value || "").trim()); +} + +function getImageUrlFromTableText(value: ReactNode) { + if (typeof value !== "string") return null; + + const trimmed = value.trim(); + if (!trimmed) return null; + + const markdownImageMatch = trimmed.match(MARKDOWN_IMAGE_REGEX); + if (markdownImageMatch?.[2] && isLikelyImageUrl(markdownImageMatch[2])) { + return markdownImageMatch[2]; + } + + return isLikelyImageUrl(trimmed) ? trimmed : null; +} + +function getOriginalImageUrlFromMarkdownImage(src: string, alt?: string) { + const normalizedAlt = String(alt || ""); + const parseByMarker = (marker: string) => { + if (!normalizedAlt.includes(marker)) return ""; + const content = normalizedAlt.slice(normalizedAlt.indexOf(marker) + marker.length); + return content.split(marker === "thumb|||" ? "|||" : marker === "thumb$$" ? "$$" : "|")[0] || ""; + }; + + return ( + parseByMarker("thumb|||") || + parseByMarker("thumb$$") || + parseByMarker("thumb|") || + src + ); +} + +function MarkdownTableImagePreview({ + alt, + src, +}: { + alt?: string; + src: string; +}) { + const [isOpen, setIsOpen] = useState(false); + const originalSrc = getOriginalImageUrlFromMarkdownImage(src, alt); + const thumbnailSrc = buildImageProxyUrl(src, { + delivery: "proxy", + format: "webp", + quality: 82, + width: 180, + }); + const largeSrc = buildImageProxyUrl(originalSrc, { + delivery: "proxy", + format: "webp", + quality: 90, + width: 1600, + }); + + return ( + <> + + + {isOpen ? ( +
setIsOpen(false)} + > + + {alt event.stopPropagation()} + onError={(event) => { + if (event.currentTarget.src !== originalSrc) { + event.currentTarget.src = originalSrc; + } + }} + /> +
+ ) : null} + + ); +} + function normalizeSectionTitle(line: string) { let normalized = line.trim(); if (!normalized) return ""; @@ -568,29 +690,34 @@ function DownloadableMarkdownTable({ - - {children} -
+ + + {children} +
+ ); } function MarkdownTableCell({ + children, className, node: _node, ...props }: MarkdownTableCellProps) { + const imageUrl = getImageUrlFromTableText(children); + return ( 		+ > + {imageUrl ? : children} +