fix: support mask guided scale adjust

app/api/canvas/local-edit/route.ts

@@ -17,6 +17,7 @@ import {
 import {
   modelLacksReferenceImageInputSupport,
 } from "@/lib/ai/reference-image-capability";
+import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image";
 import { getImageToolCreditCost } from "@/lib/points/config";
 import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
 import { uploadFile } from "@/lib/storage/service";
@@ -278,6 +279,19 @@ function getScaleAdjustModelFamily(
   );
 }
 
+function isScaleAdjustMaskCapableModel(
+  entry: AiModelWithProviderForSelection | null | undefined
+) {
+  if (!entry || getScaleAdjustModelFamily(entry) !== "gpt-image-2") {
+    return false;
+  }
+  if (entry.provider.type === "openai-image") return true;
+  if (entry.provider.type === "fal-image") {
+    return isFalGptImage2EditModelId(entry.model.modelId || "");
+  }
+  return false;
+}
+
 function isScaleAdjustCoordinateGuideModel(
   entry: AiModelWithProviderForSelection | null | undefined
 ) {
@@ -1004,6 +1018,89 @@ function buildScaleAdjustTargetFootprintShadowSvg(params: {
   );
 }
 
+function getScaleAdjustEditMaskBoxes(params: {
+  regions: NormalizedLocalEditRegion[];
+  scalePercent?: ScaleAdjustPercent | null;
+  sourceHeight: number;
+  sourceWidth: number;
+}) {
+  const { regions, scalePercent, sourceHeight, sourceWidth } = params;
+  return regions.map((region) => {
+    const targetRect = getScaleAdjustTargetRect(region, scalePercent);
+    const workspaceRect = getScaleAdjustWorkspaceRect(
+      region.rect,
+      targetRect,
+      scalePercent
+    );
+    return rectToPixels(workspaceRect, sourceWidth, sourceHeight);
+  });
+}
+
+async function buildScaleAdjustEditMaskBuffer(params: {
+  regions: NormalizedLocalEditRegion[];
+  scalePercent?: ScaleAdjustPercent | null;
+  sourceHeight: number;
+  sourceWidth: number;
+}) {
+  const { regions, scalePercent, sourceHeight, sourceWidth } = params;
+  const boxes = getScaleAdjustEditMaskBoxes({
+    regions,
+    scalePercent,
+    sourceHeight,
+    sourceWidth,
+  });
+  const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight));
+  const blur = Math.max(5, Math.min(28, Math.round(minSide * 0.018)));
+  const pad = Math.max(2, Math.round(blur * 0.55));
+  const rects = boxes
+    .map((box) => {
+      const left = clamp(box.left - pad, 0, Math.max(0, sourceWidth - 1));
+      const top = clamp(box.top - pad, 0, Math.max(0, sourceHeight - 1));
+      const right = clamp(
+        box.left + box.width + pad,
+        left + 1,
+        sourceWidth
+      );
+      const bottom = clamp(
+        box.top + box.height + pad,
+        top + 1,
+        sourceHeight
+      );
+      return `<rect x="${left}" y="${top}" width="${Math.max(1, right - left)}" height="${Math.max(1, bottom - top)}" rx="${Math.max(4, Math.round(blur * 0.8))}" ry="${Math.max(4, Math.round(blur * 0.8))}" fill="#ffffff" filter="url(#soft-edit-mask)" />`;
+    })
+    .join("");
+  const editMaskSvg = Buffer.from(
+    `<svg xmlns="http://www.w3.org/2000/svg" width="${sourceWidth}" height="${sourceHeight}" viewBox="0 0 ${sourceWidth} ${sourceHeight}">
+      <defs>
+        <filter id="soft-edit-mask" x="-20%" y="-20%" width="140%" height="140%">
+          <feGaussianBlur stdDeviation="${blur}" />
+        </filter>
+      </defs>
+      <rect x="0" y="0" width="${sourceWidth}" height="${sourceHeight}" fill="#000000" />
+      ${rects}
+    </svg>`
+  );
+  const editAlpha = await sharp(editMaskSvg, { failOn: "none" })
+    .resize(sourceWidth, sourceHeight, { fit: "fill" })
+    .greyscale()
+    .raw()
+    .toBuffer();
+  const rgba = Buffer.alloc(sourceWidth * sourceHeight * 4);
+  for (let index = 0; index < sourceWidth * sourceHeight; index += 1) {
+    const offset = index * 4;
+    const editable = editAlpha[index] || 0;
+    rgba[offset] = 0;
+    rgba[offset + 1] = 0;
+    rgba[offset + 2] = 0;
+    rgba[offset + 3] = clamp(255 - editable, 0, 255);
+  }
+  return sharp(rgba, {
+    raw: { channels: 4, height: sourceHeight, width: sourceWidth },
+  })
+    .png({ compressionLevel: 9 })
+    .toBuffer();
+}
+
 async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
   regions: NormalizedLocalEditRegion[];
   scalePercent?: ScaleAdjustPercent | null;
@@ -1999,8 +2096,29 @@ export async function POST(request: NextRequest) {
       annotatedFilename,
       toArrayBuffer(annotatedBuffer)
     );
+    const useScaleAdjustEditMask =
+      isScaleAdjustOperation &&
+      isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry);
+    const scaleAdjustEditMaskUpload = useScaleAdjustEditMask
+      ? await uploadFile(
+          `canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random()
+            .toString(36)
+            .slice(2, 8)}.png`,
+          toArrayBuffer(
+            await buildScaleAdjustEditMaskBuffer({
+              regions,
+              scalePercent,
+              sourceHeight,
+              sourceWidth,
+            })
+          )
+        )
+      : null;
+    const shouldCreateScaleAdjustCropReferences =
+      isScaleAdjustOperation &&
+      (useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinateGuide);
     const scaleAdjustCropReferences: ScaleAdjustCropReference[] =
-      isScaleAdjustOperation && useScaleAdjustVisualLayoutGuide
+      shouldCreateScaleAdjustCropReferences
         ? await createScaleAdjustCropReferences({
             regions,
             sourceBuffer,
@@ -2054,11 +2172,9 @@ export async function POST(request: NextRequest) {
     const scaleAdjustVerificationConfig = isScaleAdjustOperation
       ? {
           allowOutsideChangedWhenChangedInside: false,
+          allowDeterministicFallback: false,
           protectedRegions: scaleAdjustCompositeRegions,
-          rejectFailedCandidates:
+          rejectFailedCandidates: true,
-            useScaleAdjustCoordinateGuide && Boolean(scaleAdjustModelFamily)
-              ? true
-              : undefined,
           sourceImageUrl,
           regions: regions.map((region) => region.rect),
           regionPrompts: regions.map((region) => region.prompt),
@@ -2097,6 +2213,12 @@ export async function POST(request: NextRequest) {
               : "scale-adjust-neutral-coordinate-map"
             : "scale-adjust-full-frame-semantic-edit-map"
           : "source-overlay",
+        scaleAdjustEditMaskApplied: Boolean(scaleAdjustEditMaskUpload),
+        scaleAdjustEditMaskUrl: scaleAdjustEditMaskUpload?.url || null,
+        scaleAdjustEditMaskProviderType:
+          useScaleAdjustEditMask
+            ? resolvedScaleAdjustModelEntry?.provider?.type || null
+            : null,
         cleanSourceReferenceIndex: 1,
         configuredDetailModelId: configuredDetailModel || null,
         detailFixedModelApplied: Boolean(detailFixedModel),
@@ -2181,6 +2303,9 @@ export async function POST(request: NextRequest) {
           : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位",
       prompt,
       promptExtend: false,
+      providerImageParams: scaleAdjustEditMaskUpload
+        ? { maskImageUrl: scaleAdjustEditMaskUpload.url }
+        : undefined,
       referenceImages,
       scaleAdjustVerification: scaleAdjustVerificationConfig,
       localEditComposite: isScaleAdjustOperation

lib/ai/providers/openai-image.ts

@@ -19,6 +19,7 @@ export interface OpenAIImageParams {
   quality?: string;
   background?: string;
   outputFormat?: string;
+  maskImageUrl?: string;
 }
 
 interface UploadedImage {
@@ -643,6 +644,7 @@ async function requestOpenAIImageGeneration(params: {
   aspectRatio?: string;
   imageSizeLabel?: "1K" | "2K" | "4K";
   imageUrls: string[];
+  maskImageUrl?: string;
   quality?: string;
   background?: string;
   outputFormat?: string;
@@ -656,6 +658,7 @@ async function requestOpenAIImageGeneration(params: {
     aspectRatio,
     imageSizeLabel,
     imageUrls,
+    maskImageUrl,
     quality,
     background,
     outputFormat,
@@ -723,6 +726,10 @@ async function requestOpenAIImageGeneration(params: {
           formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename);
         }
       }
+      if (maskImageUrl) {
+        const mask = await loadInputImage(maskImageUrl);
+        formData.append("mask", new Blob([mask.buffer], { type: mask.mimeType }), mask.filename);
+      }
 
       const response = await fetch(endpoint, {
         method: "POST",
@@ -898,6 +905,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) {
           timeoutMs: resolveRequestTimeoutMs(baseURL, imageSize, activeModelId),
           useBltcyImageCompatParams: shouldUseBltcyImageCompatParams(baseURL),
           imageCount: imageUrls.length,
+          hasMask: Boolean(options.imageParams?.maskImageUrl),
           baseURL: normalizeBaseURL(baseURL),
         });
 
@@ -910,6 +918,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) {
           aspectRatio: aspectRatio || undefined,
           imageSizeLabel: imageSize,
           imageUrls,
+          maskImageUrl: options.imageParams?.maskImageUrl,
           quality,
           background,
           outputFormat,

lib/canvas/generate-image.ts

@@ -53,6 +53,7 @@ export interface CanvasLocalEditCompositeConfig {
 }
 
 export interface CanvasScaleAdjustVerificationConfig {
+  allowDeterministicFallback?: boolean;
   allowOutsideChangedWhenChangedInside?: boolean;
   protectedRegions?: CanvasLocalEditCompositeRegion[];
   rejectFailedCandidates?: boolean;
@@ -75,6 +76,7 @@ export interface CanvasGenerateImageRetryVariant {
   negativePrompt?: string;
   prompt?: string;
   promptExtend?: boolean;
+  providerImageParams?: Record<string, unknown>;
   referenceImage?: string;
   referenceImages?: string[];
   scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
@@ -92,6 +94,7 @@ export interface CanvasGenerateImageRequest {
   falSystemPrompt?: string;
   falThinkingLevel?: "minimal" | "high";
   promptExtend?: boolean;
+  providerImageParams?: Record<string, unknown>;
   referenceImage?: string;
   referenceImages?: string[];
   billingFeature?: "multi-view" | "repaint";
@@ -395,15 +398,22 @@ function normalizeScaleAdjustVerificationConfig(
           Boolean(region)
         )
     : [];
+  const regionPrompts = Array.isArray(value.regionPrompts)
+    ? value.regionPrompts
+        .map((prompt) => String(prompt || "").trim())
+        .slice(0, regions.length)
+    : [];
 
   return regions.length > 0 &&
     (targetRegions.length > 0 || typeof scalePercent === "number")
     ? {
         sourceImageUrl,
         regions,
+        allowDeterministicFallback: value.allowDeterministicFallback === true,
         allowOutsideChangedWhenChangedInside:
           value.allowOutsideChangedWhenChangedInside === true,
         rejectFailedCandidates: value.rejectFailedCandidates === true,
+        ...(regionPrompts.length > 0 ? { regionPrompts } : {}),
         ...(targetRegions.length > 0 ? { targetRegions } : {}),
         ...(typeof scalePercent === "number" ? { scalePercent } : {}),
         ...(protectedRegions.length > 0 ? { protectedRegions } : {}),
@@ -2526,6 +2536,10 @@ export async function generateCanvasImage(
       : 1;
   const promptExtend =
     typeof request.promptExtend === "boolean" ? request.promptExtend : undefined;
+  const providerImageParams =
+    request.providerImageParams && typeof request.providerImageParams === "object"
+      ? request.providerImageParams
+      : undefined;
   const ownerKey =
     typeof request.ownerKey === "string" && request.ownerKey.trim()
       ? request.ownerKey.trim()
@@ -2742,6 +2756,7 @@ export async function generateCanvasImage(
   ];
 
   const imageParams = {
+    ...(providerImageParams || {}),
     aspectRatio,
     safetyTolerance: request.falSafetyTolerance,
     imageSize,
@@ -3242,7 +3257,7 @@ export async function generateCanvasImage(
       selectedModelId: selectedModel.model.modelId,
       ...debugMeta,
     });
-    if (scaleAdjustVerification) {
+    if (scaleAdjustVerification?.allowDeterministicFallback) {
       try {
         const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({
           debugLabel,