diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 919766b67..944a6c451 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -17,6 +17,7 @@ import { import { modelLacksReferenceImageInputSupport, } from "@/lib/ai/reference-image-capability"; +import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image"; import { getImageToolCreditCost } from "@/lib/points/config"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; import { uploadFile } from "@/lib/storage/service"; @@ -278,6 +279,19 @@ function getScaleAdjustModelFamily( ); } +function isScaleAdjustMaskCapableModel( + entry: AiModelWithProviderForSelection | null | undefined +) { + if (!entry || getScaleAdjustModelFamily(entry) !== "gpt-image-2") { + return false; + } + if (entry.provider.type === "openai-image") return true; + if (entry.provider.type === "fal-image") { + return isFalGptImage2EditModelId(entry.model.modelId || ""); + } + return false; +} + function isScaleAdjustCoordinateGuideModel( entry: AiModelWithProviderForSelection | null | undefined ) { @@ -1004,6 +1018,89 @@ function buildScaleAdjustTargetFootprintShadowSvg(params: { ); } +function getScaleAdjustEditMaskBoxes(params: { + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; + sourceHeight: number; + sourceWidth: number; +}) { + const { regions, scalePercent, sourceHeight, sourceWidth } = params; + return regions.map((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const workspaceRect = getScaleAdjustWorkspaceRect( + region.rect, + targetRect, + scalePercent + ); + return rectToPixels(workspaceRect, sourceWidth, sourceHeight); + }); +} + +async function buildScaleAdjustEditMaskBuffer(params: { + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; + sourceHeight: number; + sourceWidth: number; +}) { + const { regions, scalePercent, sourceHeight, sourceWidth } = params; + const boxes = getScaleAdjustEditMaskBoxes({ + regions, + scalePercent, + sourceHeight, + sourceWidth, + }); + const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight)); + const blur = Math.max(5, Math.min(28, Math.round(minSide * 0.018))); + const pad = Math.max(2, Math.round(blur * 0.55)); + const rects = boxes + .map((box) => { + const left = clamp(box.left - pad, 0, Math.max(0, sourceWidth - 1)); + const top = clamp(box.top - pad, 0, Math.max(0, sourceHeight - 1)); + const right = clamp( + box.left + box.width + pad, + left + 1, + sourceWidth + ); + const bottom = clamp( + box.top + box.height + pad, + top + 1, + sourceHeight + ); + return ``; + }) + .join(""); + const editMaskSvg = Buffer.from( + ` + + + + + + + ${rects} + ` + ); + const editAlpha = await sharp(editMaskSvg, { failOn: "none" }) + .resize(sourceWidth, sourceHeight, { fit: "fill" }) + .greyscale() + .raw() + .toBuffer(); + const rgba = Buffer.alloc(sourceWidth * sourceHeight * 4); + for (let index = 0; index < sourceWidth * sourceHeight; index += 1) { + const offset = index * 4; + const editable = editAlpha[index] || 0; + rgba[offset] = 0; + rgba[offset + 1] = 0; + rgba[offset + 2] = 0; + rgba[offset + 3] = clamp(255 - editable, 0, 255); + } + return sharp(rgba, { + raw: { channels: 4, height: sourceHeight, width: sourceWidth }, + }) + .png({ compressionLevel: 9 }) + .toBuffer(); +} + async function buildScaleAdjustVisualLayoutGuideBuffer(params: { regions: NormalizedLocalEditRegion[]; scalePercent?: ScaleAdjustPercent | null; @@ -1999,8 +2096,29 @@ export async function POST(request: NextRequest) { annotatedFilename, toArrayBuffer(annotatedBuffer) ); + const useScaleAdjustEditMask = + isScaleAdjustOperation && + isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry); + const scaleAdjustEditMaskUpload = useScaleAdjustEditMask + ? await uploadFile( + `canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random() + .toString(36) + .slice(2, 8)}.png`, + toArrayBuffer( + await buildScaleAdjustEditMaskBuffer({ + regions, + scalePercent, + sourceHeight, + sourceWidth, + }) + ) + ) + : null; + const shouldCreateScaleAdjustCropReferences = + isScaleAdjustOperation && + (useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinateGuide); const scaleAdjustCropReferences: ScaleAdjustCropReference[] = - isScaleAdjustOperation && useScaleAdjustVisualLayoutGuide + shouldCreateScaleAdjustCropReferences ? await createScaleAdjustCropReferences({ regions, sourceBuffer, @@ -2054,11 +2172,9 @@ export async function POST(request: NextRequest) { const scaleAdjustVerificationConfig = isScaleAdjustOperation ? { allowOutsideChangedWhenChangedInside: false, + allowDeterministicFallback: false, protectedRegions: scaleAdjustCompositeRegions, - rejectFailedCandidates: - useScaleAdjustCoordinateGuide && Boolean(scaleAdjustModelFamily) - ? true - : undefined, + rejectFailedCandidates: true, sourceImageUrl, regions: regions.map((region) => region.rect), regionPrompts: regions.map((region) => region.prompt), @@ -2097,6 +2213,12 @@ export async function POST(request: NextRequest) { : "scale-adjust-neutral-coordinate-map" : "scale-adjust-full-frame-semantic-edit-map" : "source-overlay", + scaleAdjustEditMaskApplied: Boolean(scaleAdjustEditMaskUpload), + scaleAdjustEditMaskUrl: scaleAdjustEditMaskUpload?.url || null, + scaleAdjustEditMaskProviderType: + useScaleAdjustEditMask + ? resolvedScaleAdjustModelEntry?.provider?.type || null + : null, cleanSourceReferenceIndex: 1, configuredDetailModelId: configuredDetailModel || null, detailFixedModelApplied: Boolean(detailFixedModel), @@ -2181,6 +2303,9 @@ export async function POST(request: NextRequest) { : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, + providerImageParams: scaleAdjustEditMaskUpload + ? { maskImageUrl: scaleAdjustEditMaskUpload.url } + : undefined, referenceImages, scaleAdjustVerification: scaleAdjustVerificationConfig, localEditComposite: isScaleAdjustOperation diff --git a/lib/ai/providers/openai-image.ts b/lib/ai/providers/openai-image.ts index 28eef4685..1dd0ee129 100644 --- a/lib/ai/providers/openai-image.ts +++ b/lib/ai/providers/openai-image.ts @@ -19,6 +19,7 @@ export interface OpenAIImageParams { quality?: string; background?: string; outputFormat?: string; + maskImageUrl?: string; } interface UploadedImage { @@ -643,6 +644,7 @@ async function requestOpenAIImageGeneration(params: { aspectRatio?: string; imageSizeLabel?: "1K" | "2K" | "4K"; imageUrls: string[]; + maskImageUrl?: string; quality?: string; background?: string; outputFormat?: string; @@ -656,6 +658,7 @@ async function requestOpenAIImageGeneration(params: { aspectRatio, imageSizeLabel, imageUrls, + maskImageUrl, quality, background, outputFormat, @@ -723,6 +726,10 @@ async function requestOpenAIImageGeneration(params: { formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename); } } + if (maskImageUrl) { + const mask = await loadInputImage(maskImageUrl); + formData.append("mask", new Blob([mask.buffer], { type: mask.mimeType }), mask.filename); + } const response = await fetch(endpoint, { method: "POST", @@ -898,6 +905,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { timeoutMs: resolveRequestTimeoutMs(baseURL, imageSize, activeModelId), useBltcyImageCompatParams: shouldUseBltcyImageCompatParams(baseURL), imageCount: imageUrls.length, + hasMask: Boolean(options.imageParams?.maskImageUrl), baseURL: normalizeBaseURL(baseURL), }); @@ -910,6 +918,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { aspectRatio: aspectRatio || undefined, imageSizeLabel: imageSize, imageUrls, + maskImageUrl: options.imageParams?.maskImageUrl, quality, background, outputFormat, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index bda4f43a6..151146cb2 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -53,6 +53,7 @@ export interface CanvasLocalEditCompositeConfig { } export interface CanvasScaleAdjustVerificationConfig { + allowDeterministicFallback?: boolean; allowOutsideChangedWhenChangedInside?: boolean; protectedRegions?: CanvasLocalEditCompositeRegion[]; rejectFailedCandidates?: boolean; @@ -75,6 +76,7 @@ export interface CanvasGenerateImageRetryVariant { negativePrompt?: string; prompt?: string; promptExtend?: boolean; + providerImageParams?: Record; referenceImage?: string; referenceImages?: string[]; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; @@ -92,6 +94,7 @@ export interface CanvasGenerateImageRequest { falSystemPrompt?: string; falThinkingLevel?: "minimal" | "high"; promptExtend?: boolean; + providerImageParams?: Record; referenceImage?: string; referenceImages?: string[]; billingFeature?: "multi-view" | "repaint"; @@ -395,15 +398,22 @@ function normalizeScaleAdjustVerificationConfig( Boolean(region) ) : []; + const regionPrompts = Array.isArray(value.regionPrompts) + ? value.regionPrompts + .map((prompt) => String(prompt || "").trim()) + .slice(0, regions.length) + : []; return regions.length > 0 && (targetRegions.length > 0 || typeof scalePercent === "number") ? { sourceImageUrl, regions, + allowDeterministicFallback: value.allowDeterministicFallback === true, allowOutsideChangedWhenChangedInside: value.allowOutsideChangedWhenChangedInside === true, rejectFailedCandidates: value.rejectFailedCandidates === true, + ...(regionPrompts.length > 0 ? { regionPrompts } : {}), ...(targetRegions.length > 0 ? { targetRegions } : {}), ...(typeof scalePercent === "number" ? { scalePercent } : {}), ...(protectedRegions.length > 0 ? { protectedRegions } : {}), @@ -2526,6 +2536,10 @@ export async function generateCanvasImage( : 1; const promptExtend = typeof request.promptExtend === "boolean" ? request.promptExtend : undefined; + const providerImageParams = + request.providerImageParams && typeof request.providerImageParams === "object" + ? request.providerImageParams + : undefined; const ownerKey = typeof request.ownerKey === "string" && request.ownerKey.trim() ? request.ownerKey.trim() @@ -2742,6 +2756,7 @@ export async function generateCanvasImage( ]; const imageParams = { + ...(providerImageParams || {}), aspectRatio, safetyTolerance: request.falSafetyTolerance, imageSize, @@ -3242,7 +3257,7 @@ export async function generateCanvasImage( selectedModelId: selectedModel.model.modelId, ...debugMeta, }); - if (scaleAdjustVerification) { + if (scaleAdjustVerification?.allowDeterministicFallback) { try { const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({ debugLabel,