diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 06184eba8..eb4cb677a 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -16,7 +16,6 @@ import { } from "@/lib/db/queries"; import { modelLacksReferenceImageInputSupport, - modelSupportsReferenceImageInput, } from "@/lib/ai/reference-image-capability"; import { getImageToolCreditCost } from "@/lib/points/config"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; @@ -239,36 +238,6 @@ function isFalImageModelEntry( return entry?.provider?.type === "fal-image"; } -function isHealthyImageModelEntry( - entry: AiModelWithProviderForSelection | null | undefined -) { - return ( - Boolean(entry?.model?.supportsImageGen) && - modelSupportsReferenceImageInput(entry?.model, entry?.provider) && - entry?.model?.healthStatus === "healthy" - ); -} - -function findScaleAdjustRetryModelEntry(params: { - currentModelId?: string | null; - models: AiModelWithProviderForSelection[]; -}) { - const excluded = String(params.currentModelId || "").trim(); - const isExcluded = (entry: AiModelWithProviderForSelection) => - Boolean(excluded) && - (entry.model.id === excluded || entry.model.modelId === excluded); - - return ( - params.models.find( - (entry) => - !isExcluded(entry) && - isHealthyImageModelEntry(entry) && - entry.provider.type === "snapgen-image" && - /gpt[-_ ]?image[-_ ]?2/i.test(entry.model.modelId || "") - ) || null - ); -} - function resolveScaleAdjustPreferredModelId(params: { configuredModelId: string; requestedModelId: string; @@ -818,56 +787,6 @@ function buildScaleAdjustAnnotationSvg(params: { `); } -function buildScaleAdjustPhotoGuideSvg(params: { - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { width, height, regions, scalePercent } = params; - const minSide = Math.max(1, Math.min(width, height)); - const strokeWidth = Math.max(4, Math.round(minSide * 0.006)); - const fontSize = Math.max(18, Math.round(minSide * 0.022)); - const labelPadX = Math.round(fontSize * 0.5); - const labelPadY = Math.round(fontSize * 0.28); - const sourceColor = "#ff3b30"; - const targetColor = "#00c8ff"; - - const label = (text: string, x: number, y: number, color: string) => { - const labelWidth = Math.round(text.length * fontSize * 0.68 + labelPadX * 2); - const labelHeight = Math.round(fontSize * 1.28 + labelPadY * 2); - const left = clamp(x, 6, Math.max(6, width - labelWidth - 6)); - const top = clamp(y, 6, Math.max(6, height - labelHeight - 6)); - return ` - - ${escapeXml(text)} - `; - }; - - const regionSvg = regions - .map((region) => { - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = - region.targetPixelRect || rectToPixels(targetRect, width, height); - const radius = Math.max(8, Math.round(strokeWidth * 1.8)); - const sourceDash = `${Math.round(strokeWidth * 2.2)} ${Math.round(strokeWidth * 1.6)}`; - return ` - - - ${label(`OLD ${region.index}`, source.left + strokeWidth, source.top + strokeWidth, sourceColor)} - ${label(`TARGET ${region.index}`, target.left + strokeWidth, target.top + strokeWidth, targetColor)} - `; - }) - .join(""); - - return Buffer.from(` - - ${regionSvg} - - `); -} - function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { const regionLines = regions .map((region) => { @@ -1003,10 +922,14 @@ function buildScaleAdjustPrompt( function buildFalScaleAdjustCoordinatePrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, - sourceSize: { width: number; height: number } + sourceSize: { width: number; height: number }, + cropReferences: ScaleAdjustCropReference[] = [] ) { const regionLines = regions .map((region) => { + const cropIndex = cropReferences.findIndex( + (reference) => reference.regionIndex === region.index + ); const targetRect = getScaleAdjustTargetRect(region, scalePercent); const targetPixelRect = region.targetPixelRect || @@ -1022,14 +945,25 @@ function buildFalScaleAdjustCoordinatePrompt( ? "make the furniture clearly larger" : "move/fit the furniture to the target box"; - return `Region ${region.index}: OLD search box in input 1 x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; TARGET final visual box x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}.`; + const cropNote = + cropIndex >= 0 + ? ` Input image ${cropIndex + 3} is a clean crop reference for this same furniture identity; use it only to recognize the subject, never paste it.` + : ""; + return `Region ${region.index}: OLD search box in input 1 x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; TARGET final visual box x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}.${cropNote}`; }) .join("\n"); + const cropLines = cropReferences + .map( + (reference, index) => + `Input image ${index + 3}: clean crop for Region ${reference.regionIndex}; identity/shape reference only, not a patch or layer.` + ) + .join("\n"); return [ "Local furniture scale edit. Use input image 1 as the clean original photo and the locked camera frame.", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 2 is an annotated copy of input image 1. It is only a visual instruction guide, never output content. The red dashed OLD outline locates the current furniture in input image 1; the cyan TARGET outline gives the final size and center.", + "Input image 2 is a separate neutral coordinate map, not a photo and not output content. The light gray OLD rectangle locates the current furniture in input image 1; the dark TARGET rectangle gives the final size and center.", + cropLines ? `${cropLines}` : "", "", "Task:", "Find the main movable furniture subject inside each OLD box in input image 1. Resize and, if needed, move that subject only so its final visible footprint matches the TARGET box.", @@ -1040,7 +974,7 @@ function buildFalScaleAdjustCoordinatePrompt( regionLines, "", "Success check: the selected furniture size must clearly change from the original, while the rest of the photo stays stable.", - "Do not paste rectangular image areas. Do not output the red/cyan guide lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", + "Do not paste rectangular image areas. Do not output coordinate-map lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", "Return only the clean edited photo.", ].join("\n"); } @@ -1048,10 +982,10 @@ function buildFalScaleAdjustCoordinatePrompt( function buildFalScaleAdjustSystemPrompt() { return [ "You are an image editing model for local furniture scale adjustment.", - "Interpret the input image as a soft layout guide, not as a pixel layer to copy.", + "Input image 1 is the only final photo base. Input image 2 is a coordinate instruction map, not a photo layer.", "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", - "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from the guide.", - "Never output guide artifacts from the annotated reference: no red or cyan lines, labels, blurred blocks, seams, rectangular borders, arrows, masks, UI marks, or color-coded annotations.", + "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", + "Never output guide artifacts from the coordinate reference: no lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", ].join("\n"); } @@ -1215,34 +1149,11 @@ export async function POST(request: NextRequest) { : null; const useFalScaleAdjustCoordinateGuide = isScaleAdjustOperation && isFalImageModelEntry(resolvedScaleAdjustModelEntry); - const scaleAdjustRetryModelEntry = - useFalScaleAdjustCoordinateGuide - ? findScaleAdjustRetryModelEntry({ - currentModelId: resolvedModel || null, - models: scaleAdjustModelCandidates, - }) - : null; - if (scaleAdjustRetryModelEntry) { - console.info("[Canvas Local Edit] Prepared scale adjust retry model:", { - primaryModel: resolvedScaleAdjustModelEntry?.model?.modelId || resolvedModel || null, - primaryProvider: resolvedScaleAdjustModelEntry?.provider?.type || null, - retryModel: scaleAdjustRetryModelEntry.model.modelId, - retryProvider: scaleAdjustRetryModelEntry.provider.type, - }); - } - const annotatedBuffer = await sharp(sourceBuffer, { failOn: "none" }) .composite([ { input: isScaleAdjustOperation - ? useFalScaleAdjustCoordinateGuide - ? buildScaleAdjustPhotoGuideSvg({ - height: sourceHeight, - regions, - scalePercent, - width: sourceWidth, - }) - : buildScaleAdjustAnnotationSvg({ + ? buildScaleAdjustAnnotationSvg({ height: sourceHeight, regions, scalePercent, @@ -1266,7 +1177,7 @@ export async function POST(request: NextRequest) { toArrayBuffer(annotatedBuffer) ); const scaleAdjustCropReferences = - isScaleAdjustOperation && !useFalScaleAdjustCoordinateGuide + isScaleAdjustOperation ? await createScaleAdjustCropReferences({ regions, sourceBuffer, @@ -1282,7 +1193,8 @@ export async function POST(request: NextRequest) { { width: sourceWidth, height: sourceHeight, - } + }, + scaleAdjustCropReferences ) : buildScaleAdjustPrompt( regions, @@ -1300,14 +1212,12 @@ export async function POST(request: NextRequest) { (reference) => reference.url ); const referenceImages = isScaleAdjustOperation - ? useFalScaleAdjustCoordinateGuide - ? [sourceImageUrl, annotatedUpload.url] - : [ - sourceImageUrl, - annotatedUpload.url, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] + ? [ + sourceImageUrl, + annotatedUpload.url, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] : [sourceImageUrl, ...manualRegionReferenceImageUrls]; const scaleAdjustCompositeRegions = isScaleAdjustOperation ? buildScaleAdjustCompositeRegions(regions, scalePercent) @@ -1318,7 +1228,6 @@ export async function POST(request: NextRequest) { ? true : undefined, protectedRegions: scaleAdjustCompositeRegions, - rejectFailedCandidates: useFalScaleAdjustCoordinateGuide ? true : undefined, sourceImageUrl, regions: regions.map((region) => region.rect), ...(typeof scalePercent === "number" @@ -1338,25 +1247,6 @@ export async function POST(request: NextRequest) { : {}), } : undefined; - const scaleAdjustRetryVariants = - isScaleAdjustOperation && scaleAdjustRetryModelEntry - ? [ - { - allowFailover: false, - debugMeta: { - scaleAdjustRetryAfterFalReject: true, - scaleAdjustRetryModelId: scaleAdjustRetryModelEntry.model.modelId, - scaleAdjustRetryProviderType: - scaleAdjustRetryModelEntry.provider.type, - scaleAdjustPromptStrategy: - "scale-adjust-annotated-photo-snapgen-retry-v1", - }, - label: "snapgen-gpt-image-2-after-fal-reject", - model: scaleAdjustRetryModelEntry.model.id, - }, - ] - : undefined; - const imageRequest: CanvasGenerateImageRequest = { aspectRatio: resolvedAspectRatio, billingFeature: "repaint", @@ -1366,9 +1256,7 @@ export async function POST(request: NextRequest) { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation - ? useFalScaleAdjustCoordinateGuide - ? "scale-adjust-fal-annotated-photo-guide-only" - : "scale-adjust-neutral-coordinate-guide-map" + ? "scale-adjust-neutral-coordinate-guide-map" : "source-overlay", cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, @@ -1395,7 +1283,7 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? useFalScaleAdjustCoordinateGuide - ? "fal-locked-source-annotated-photo-single-v25" + ? "fal-locked-source-neutral-coordinate-map-v26" : "locked-plate-neutral-coordinate-guide-map-pixel-anchors-v8" : undefined, scaleAdjustPostCompositeApplied: false, @@ -1425,7 +1313,6 @@ export async function POST(request: NextRequest) { prompt, promptExtend: false, referenceImages, - retryVariants: scaleAdjustRetryVariants, scaleAdjustVerification: scaleAdjustVerificationConfig, localEditComposite: isScaleAdjustOperation ? undefined diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index ca06d0ecc..03260bc5f 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -1134,7 +1134,7 @@ async function applyScaleAdjustVerificationToImages(params: { let cleanedImage: ParsedImage | undefined; try { - const cleanedImage = await removeScaleAdjustAnnotationArtifacts({ + cleanedImage = (await removeScaleAdjustAnnotationArtifacts({ debugLabel, generatedBuffer, imageIndex: index, @@ -1142,7 +1142,7 @@ async function applyScaleAdjustVerificationToImages(params: { sourceHeight: sourceImage.sourceHeight, sourceWidth: sourceImage.sourceWidth, verification, - }); + })) ?? undefined; if (cleanedImage) { evaluatedImages.push({ changeDiffScores,