diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 39fdf81c3..530df3a5c 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -97,6 +97,8 @@ const MAX_REFERENCE_IMAGES_PER_REGION = 3; const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; const SCALE_ADJUST_CROP_MAX_EDGE = 1536; +const SCALE_ADJUST_FAL_MOCKUP_PADDING_RATIO = 0.04; +const SCALE_ADJUST_FAL_MOCKUP_BLUR_PX = 38; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; @@ -226,6 +228,12 @@ function isScaleAdjustDedicatedDetailModel( return label.includes("详情专线"); } +function isFalImageModelEntry( + entry: AiModelWithProviderForSelection | null | undefined +) { + return entry?.provider?.type === "fal-image"; +} + function resolveScaleAdjustPreferredModelId(params: { configuredModelId: string; requestedModelId: string; @@ -536,6 +544,162 @@ function buildScaleAdjustCompositeRegions( }); } +function expandPixelBox(params: { + amount: number; + box: NormalizedLocalEditRegion["pixelRect"]; + height: number; + width: number; +}) { + const { amount, box, height, width } = params; + const left = clamp(box.left - amount, 0, Math.max(0, width - 1)); + const top = clamp(box.top - amount, 0, Math.max(0, height - 1)); + const right = clamp(box.left + box.width + amount, left + 1, width); + const bottom = clamp(box.top + box.height + amount, top + 1, height); + + return { + left, + top, + width: Math.max(1, right - left), + height: Math.max(1, bottom - top), + }; +} + +async function createFeatherAlphaMask(params: { + width: number; + height: number; + feather: number; + inset: number; +}) { + const { width, height } = params; + const feather = Math.max(1, Math.round(params.feather)); + const maxInset = Math.max(1, Math.floor(Math.min(width, height) * 0.42)); + const inset = clamp(Math.round(params.inset), 1, maxInset); + const innerWidth = Math.max(1, width - inset * 2); + const innerHeight = Math.max(1, height - inset * 2); + const svg = Buffer.from(` + + + + + + + `); + + return sharp(svg, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .greyscale() + .png({ compressionLevel: 6 }) + .toBuffer(); +} + +async function addFeatherAlpha(params: { + buffer: Buffer; + width: number; + height: number; + feather: number; + inset: number; +}) { + const mask = await createFeatherAlphaMask({ + feather: params.feather, + height: params.height, + inset: params.inset, + width: params.width, + }); + + return sharp(params.buffer, { failOn: "none" }) + .resize(params.width, params.height, { fit: "fill" }) + .removeAlpha() + .joinChannel(mask) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + +async function buildScaleAdjustFalTargetMockup(params: { + sourceBuffer: Buffer; + sourceHeight: number; + sourceWidth: number; + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; +}) { + const { sourceBuffer, sourceHeight, sourceWidth, regions, scalePercent } = params; + const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight)); + const composites: sharp.OverlayOptions[] = []; + + for (const region of regions) { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const targetPixelRect = + region.targetPixelRect || rectToPixels(targetRect, sourceWidth, sourceHeight); + const workspaceRect = getScaleAdjustWorkspaceRect( + region.rect, + targetRect, + scalePercent + ); + const workspacePixelRect = expandPixelBox({ + amount: Math.round(minSide * SCALE_ADJUST_FAL_MOCKUP_PADDING_RATIO), + box: rectToPixels(workspaceRect, sourceWidth, sourceHeight), + height: sourceHeight, + width: sourceWidth, + }); + + const blurredWorkspace = await sharp(sourceBuffer, { failOn: "none" }) + .extract(workspacePixelRect) + .blur(SCALE_ADJUST_FAL_MOCKUP_BLUR_PX) + .modulate({ brightness: 1.05, saturation: 0.86 }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const workspaceFeather = Math.max( + 18, + Math.round(Math.min(workspacePixelRect.width, workspacePixelRect.height) * 0.1) + ); + const featheredWorkspace = await addFeatherAlpha({ + buffer: blurredWorkspace, + feather: workspaceFeather, + height: workspacePixelRect.height, + inset: Math.max(workspaceFeather * 2.6, workspaceFeather + 1), + width: workspacePixelRect.width, + }); + + const subjectCrop = await sharp(sourceBuffer, { failOn: "none" }) + .extract(region.pixelRect) + .resize(targetPixelRect.width, targetPixelRect.height, { + fit: "cover", + position: "center", + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const subjectFeather = Math.max( + 8, + Math.round(Math.min(targetPixelRect.width, targetPixelRect.height) * 0.08) + ); + const featheredSubject = await addFeatherAlpha({ + buffer: subjectCrop, + feather: subjectFeather, + height: targetPixelRect.height, + inset: Math.max(subjectFeather * 1.6, subjectFeather + 1), + width: targetPixelRect.width, + }); + + composites.push( + { + input: featheredWorkspace, + left: workspacePixelRect.left, + top: workspacePixelRect.top, + }, + { + input: featheredSubject, + left: targetPixelRect.left, + top: targetPixelRect.top, + } + ); + } + + return sharp(sourceBuffer, { failOn: "none" }) + .composite(composites) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + function expandPixelRectForCrop(params: { height: number; pixelRect: NormalizedLocalEditRegion["pixelRect"]; @@ -907,6 +1071,42 @@ function buildScaleAdjustPrompt( ].join("\n"); } +function buildFalScaleAdjustMockupPrompt( + regions: NormalizedLocalEditRegion[], + scalePercent: ScaleAdjustPercent | null | undefined, + sourceSize: { width: number; height: number } +) { + const regionLines = regions + .map((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const targetPixelRect = + region.targetPixelRect || + rectToPixels(targetRect, sourceSize.width, sourceSize.height); + const widthFactor = + region.rect.width > 0 ? targetRect.width / region.rect.width : 1; + const heightFactor = + region.rect.height > 0 ? targetRect.height / region.rect.height : 1; + + return `区域 ${region.index}：原主体约在 x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}；目标主体约在 x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}；目标视觉占位约为原主体宽度 ${widthFactor.toFixed(2)} 倍、高度 ${heightFactor.toFixed(2)} 倍。必须保持主体自身宽高比和透视，不要拉伸或压扁。`; + }) + .join("\n"); + + return [ + "这是一项室内家用场景里的家具/物体局部比例调整任务。输入图片1是一张粗略目标效果草图，不是最终结果，也不是标注图。", + `最终输出必须是一张真实、干净、自然的单张照片，并保持原始画幅 ${sourceSize.width} x ${sourceSize.height} px。`, + "输入图片1已经把目标主体粗略放到了用户希望的最终大小和位置；请把它作为最终布局意图来理解，但不要直接复制其中的模糊、拼接边缘、软边、矩形块或局部贴片痕迹。", + "核心目标：只调整被选中的家具/物体主体比例和位置；不要恢复成原来的大尺寸，也不要返回几乎未修改的原图。", + "请用 AI 自然重建原主体移除后露出的背景，让地毯、地板、窗户、墙面、桌面、阴影、遮挡和透视连续自然。", + "请保留目标主体的真实材质、结构、轮廓、透视和光照；主体不能透明、幽灵化、变形、被裁切，不能变成说明图或标注元素。", + "非目标区域必须保持原室内照片逻辑：画幅边界、窗框、沙发、植物、墙面、地毯整体纹理、光线、色温和镜头透视不要整体移动、裁切、放大、缩小或重排。", + "", + "区域要求：", + regionLines, + "", + "输出前自检：如果目标主体仍接近原尺寸、输出像未修改原图、整张图被放大/裁切/重排，或者出现矩形框、箭头、文字、遮罩、拼图、硬边贴片、色差边缘，就是失败；必须重新生成一张无缝真实照片。", + ].join("\n"); +} + export async function POST(request: NextRequest) { try { const session = await auth(); @@ -1029,61 +1229,6 @@ export async function POST(request: NextRequest) { operation === SCALE_ADJUST_OPERATION && (hasScaleAdjustTargetRects || scalePercent !== null); const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); - - const annotationSvg = isScaleAdjustOperation - ? buildScaleAdjustAnnotationSvg({ - height: sourceHeight, - regions, - scalePercent, - width: sourceWidth, - }) - : buildAnnotationSvg({ - height: sourceHeight, - hidePromptText: false, - regions, - width: sourceWidth, - }); - const annotatedBase = sharp(sourceBuffer, { failOn: "none" }); - const annotatedBuffer = await annotatedBase - .composite([{ input: annotationSvg, left: 0, top: 0 }]) - .png({ compressionLevel: 6 }) - .toBuffer(); - const annotatedFilename = `${isScaleAdjustOperation ? "canvas-scale-adjust-guide" : "canvas-local-edit"}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; - const annotatedUpload = await uploadFile( - annotatedFilename, - toArrayBuffer(annotatedBuffer) - ); - const scaleAdjustCropReferences = isScaleAdjustOperation - ? await createScaleAdjustCropReferences({ - regions, - sourceBuffer, - sourceHeight, - sourceWidth, - }) - : []; - const prompt = - isScaleAdjustOperation - ? buildScaleAdjustPrompt( - regions, - scalePercent, - scaleAdjustCropReferences, - { width: sourceWidth, height: sourceHeight } - ) - : buildLocalEditPrompt(regions); - const manualRegionReferenceImageUrls = regions.flatMap((region) => - region.referenceImages.map((reference) => reference.url) - ); - const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map( - (reference) => reference.url - ); - const referenceImages = isScaleAdjustOperation - ? [ - sourceImageUrl, - annotatedUpload.url, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [sourceImageUrl, ...manualRegionReferenceImageUrls]; const resolvedAspectRatio = resolveLocalEditAspectRatio( body.aspectRatio, sourceWidth, @@ -1116,6 +1261,87 @@ export async function POST(request: NextRequest) { }); } const resolvedModel = detailFixedModel || requestedModel; + const resolvedScaleAdjustModelEntry = isScaleAdjustOperation + ? findModelSelectionEntry(scaleAdjustModelCandidates, resolvedModel || "") + : null; + const useFalScaleAdjustMockup = + isScaleAdjustOperation && isFalImageModelEntry(resolvedScaleAdjustModelEntry); + + const annotatedBuffer = + useFalScaleAdjustMockup + ? await buildScaleAdjustFalTargetMockup({ + regions, + scalePercent, + sourceBuffer, + sourceHeight, + sourceWidth, + }) + : await sharp(sourceBuffer, { failOn: "none" }) + .composite([ + { + input: isScaleAdjustOperation + ? buildScaleAdjustAnnotationSvg({ + height: sourceHeight, + regions, + scalePercent, + width: sourceWidth, + }) + : buildAnnotationSvg({ + height: sourceHeight, + hidePromptText: false, + regions, + width: sourceWidth, + }), + left: 0, + top: 0, + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const annotatedFilename = `${useFalScaleAdjustMockup ? "canvas-scale-adjust-fal-target-mockup" : isScaleAdjustOperation ? "canvas-scale-adjust-guide" : "canvas-local-edit"}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; + const annotatedUpload = await uploadFile( + annotatedFilename, + toArrayBuffer(annotatedBuffer) + ); + const scaleAdjustCropReferences = isScaleAdjustOperation + ? useFalScaleAdjustMockup + ? [] + : await createScaleAdjustCropReferences({ + regions, + sourceBuffer, + sourceHeight, + sourceWidth, + }) + : []; + const prompt = isScaleAdjustOperation + ? useFalScaleAdjustMockup + ? buildFalScaleAdjustMockupPrompt(regions, scalePercent, { + width: sourceWidth, + height: sourceHeight, + }) + : buildScaleAdjustPrompt( + regions, + scalePercent, + scaleAdjustCropReferences, + { width: sourceWidth, height: sourceHeight } + ) + : buildLocalEditPrompt(regions); + const manualRegionReferenceImageUrls = regions.flatMap((region) => + region.referenceImages.map((reference) => reference.url) + ); + const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map( + (reference) => reference.url + ); + const referenceImages = isScaleAdjustOperation + ? useFalScaleAdjustMockup + ? [annotatedUpload.url] + : [ + sourceImageUrl, + annotatedUpload.url, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] + : [sourceImageUrl, ...manualRegionReferenceImageUrls]; const scaleAdjustCompositeRegions = isScaleAdjustOperation ? buildScaleAdjustCompositeRegions(regions, scalePercent) : []; @@ -1129,21 +1355,26 @@ export async function POST(request: NextRequest) { annotatedReferenceUrl: annotatedUpload.url, annotatedReferenceSentToModel: isScaleAdjustOperation, annotatedReferenceType: isScaleAdjustOperation - ? "scale-adjust-neutral-coordinate-guide-map" + ? useFalScaleAdjustMockup + ? "scale-adjust-fal-rough-target-mockup" + : "scale-adjust-neutral-coordinate-guide-map" : "source-overlay", - cleanSourceReferenceIndex: 1, + cleanSourceReferenceIndex: + isScaleAdjustOperation && useFalScaleAdjustMockup ? null : 1, configuredDetailModelId: configuredDetailModel || null, detailFixedModelApplied: Boolean(detailFixedModel), detailFixedModelId: detailFixedModel || null, displayPrompt, + falScaleAdjustMockupApplied: useFalScaleAdjustMockup, operation, requestedModel: requestedModel || null, resolvedAspectRatio, resolvedModel: resolvedModel || null, + resolvedScaleAdjustProviderType: + resolvedScaleAdjustModelEntry?.provider?.type || null, manualRegionReferenceImageCount: manualRegionReferenceImageUrls.length, regionReferenceImageCount: - manualRegionReferenceImageUrls.length + - scaleAdjustCropReferenceImageUrls.length, + referenceImages.length, regionCount: regions.length, scalePercent, scaleAdjustCropReferenceCount: scaleAdjustCropReferenceImageUrls.length, @@ -1153,7 +1384,9 @@ export async function POST(request: NextRequest) { scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) .length, scaleAdjustPromptStrategy: isScaleAdjustOperation - ? "locked-plate-neutral-coordinate-guide-map-pixel-anchors-v8" + ? useFalScaleAdjustMockup + ? "fal-rough-target-mockup-only-v17" + : "locked-plate-neutral-coordinate-guide-map-pixel-anchors-v8" : undefined, scaleAdjustPostCompositeApplied: false, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation,