fix: reduce scale adjust false failures

app/(chat)/api/chat/_lib/image-failover.ts

@@ -111,6 +111,34 @@ export type ImageFallbackCandidate = {
   sameProvider: boolean;
 };
 
+function messagePartHasImageInput(part: any) {
+  if (!part || typeof part !== "object") return false;
+  if (part.type === "file" || part.type === "image" || part.type === "image_url") {
+    return true;
+  }
+  if (typeof part.url === "string" && /^https?:\/\//i.test(part.url)) {
+    return true;
+  }
+  if (typeof part.image === "string" && part.image.trim()) {
+    return true;
+  }
+  if (typeof part.image_url?.url === "string" && part.image_url.url.trim()) {
+    return true;
+  }
+  return false;
+}
+
+export function chatMessagesHaveImageInput(messages: ChatMessage[]) {
+  return messages.some((message: any) => {
+    const content = message?.content;
+    if (Array.isArray(content) && content.some(messagePartHasImageInput)) {
+      return true;
+    }
+    const parts = message?.parts;
+    return Array.isArray(parts) && parts.some(messagePartHasImageInput);
+  });
+}
+
 export function getImageFailoverConvertProvider(providerType?: string | null) {
   switch (providerType) {
     case "bfl-image":
@@ -219,6 +247,11 @@ export async function generateImageWithFailoverCandidate(params: {
   const { candidate, messages, imageParams, fallbackImageParams } = params;
   const providerType = String(candidate.provider?.type || "");
   const candidateRuntimeModelId = resolveImageModelRuntimeId(candidate.model);
+  if (chatMessagesHaveImageInput(messages) && candidate.model?.supportsVision === false) {
+    throw new Error(
+      `备用模型不支持参考图/图生图输入：${candidate.model.displayName || candidate.model.id}`
+    );
+  }
   if (
     providerType === "fal-image" &&
     isFalGptImage2EditModelId(candidateRuntimeModelId)
@@ -237,9 +270,18 @@ export async function generateImageWithFailoverCandidate(params: {
     messages,
     getImageFailoverConvertProvider(providerType)
   );
+  const effectiveImageParams = imageParams || fallbackImageParams;
+  chatApiLogger.info("[Chat API] Image failover candidate params:", {
+    model: candidate.model?.modelId || candidate.model?.id,
+    providerType,
+    aspectRatio: effectiveImageParams?.aspectRatio,
+    imageSize: effectiveImageParams?.imageSize,
+    imageCount: effectiveImageParams?.imageCount,
+    hasImageInput: chatMessagesHaveImageInput(messages),
+  });
   const candidateResult = await (candidateLanguageModel as any).doGenerate({
     messages: candidateMessages,
-    imageParams: imageParams || fallbackImageParams,
+    imageParams: effectiveImageParams,
     imageGenConfig: candidate.model.imageGenConfig || null,
     modelDisplayName: candidate.model.displayName,
     overrideModelId: candidateRuntimeModelId || candidate.model.modelId,

app/api/canvas/local-edit/route.ts

@@ -892,6 +892,12 @@ export async function POST(request: NextRequest) {
         ? siteSettingsForModel.canvasDetailDefaultModelId.trim()
         : "";
     const resolvedModel = detailFixedModel || requestedModel;
+    const scaleAdjustCompositeRegions = isScaleAdjustOperation
+      ? buildScaleAdjustCompositeRegions(
+          regions,
+          scalePercent as ScaleAdjustPercent
+        )
+      : [];
 
     const imageRequest: CanvasGenerateImageRequest = {
       aspectRatio: resolvedAspectRatio,
@@ -937,6 +943,7 @@ export async function POST(request: NextRequest) {
       referenceImages,
       scaleAdjustVerification: isScaleAdjustOperation
         ? {
+            protectedRegions: scaleAdjustCompositeRegions,
             sourceImageUrl,
             regions: regions.map((region) => region.rect),
             scalePercent: scalePercent as ScaleAdjustPercent,
@@ -945,10 +952,7 @@ export async function POST(request: NextRequest) {
       localEditComposite: isScaleAdjustOperation
         ? {
             sourceImageUrl,
-            regions: buildScaleAdjustCompositeRegions(
-              regions,
-              scalePercent as ScaleAdjustPercent
-            ),
+            regions: scaleAdjustCompositeRegions,
             expandRegions: false,
             maskMode: "diff",
             featherMinPx: 4,

lib/canvas/generate-image.ts

@@ -51,6 +51,7 @@ export interface CanvasLocalEditCompositeConfig {
 }
 
 export interface CanvasScaleAdjustVerificationConfig {
+  protectedRegions?: CanvasLocalEditCompositeRegion[];
   sourceImageUrl: string;
   regions: CanvasLocalEditCompositeRegion[];
   scalePercent: number;
@@ -334,8 +335,22 @@ function normalizeScaleAdjustVerificationConfig(
   const regions = value.regions
     .map((region) => normalizeCanvasRegion(region))
     .filter((region): region is CanvasLocalEditCompositeRegion => Boolean(region));
+  const protectedRegions = Array.isArray(value.protectedRegions)
+    ? value.protectedRegions
+        .map((region) => normalizeCanvasRegion(region))
+        .filter((region): region is CanvasLocalEditCompositeRegion =>
+          Boolean(region)
+        )
+    : [];
 
-  return regions.length > 0 ? { sourceImageUrl, regions, scalePercent } : null;
+  return regions.length > 0
+    ? {
+        sourceImageUrl,
+        regions,
+        scalePercent,
+        ...(protectedRegions.length > 0 ? { protectedRegions } : {}),
+      }
+    : null;
 }
 
 function scaleRegionAroundCenter(
@@ -402,6 +417,11 @@ function canvasRegionsIntersect(
 function getScaleAdjustProtectedRegions(
   verification: CanvasScaleAdjustVerificationConfig
 ) {
+  if (verification.protectedRegions?.length) {
+    return verification.protectedRegions.map((region) =>
+      expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN)
+    );
+  }
   return verification.regions.flatMap((region) => [
     expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN),
     expandCanvasRegion(
@@ -682,6 +702,7 @@ async function applyScaleAdjustVerificationToImages(params: {
           outsideDiffScores: outsideDiffScores.map(
             (score) => Math.round(score * 10) / 10
           ),
+          protectedRegionCount: verification.protectedRegions?.length || 0,
           scalePercent: verification.scalePercent,
         });
         checkedImages.push(image);
@@ -700,6 +721,7 @@ async function applyScaleAdjustVerificationToImages(params: {
         outsideDiffScores: outsideDiffScores.map(
           (score) => Math.round(score * 10) / 10
         ),
+        protectedRegionCount: verification.protectedRegions?.length || 0,
         reason: rejection.reason,
         scalePercent: verification.scalePercent,
       });
@@ -1202,6 +1224,7 @@ function findCanvasFailoverModel(params: {
   selectedModel: CanvasModelWithProvider;
   imageModels: CanvasModelWithProvider[];
   failedModelRefs: string[];
+  requiresImageInput?: boolean;
 }) {
   const failedRefs = new Set(
     params.failedModelRefs.map((item) => String(item || "").trim()).filter(Boolean)
@@ -1218,6 +1241,9 @@ function findCanvasFailoverModel(params: {
       return false;
     }
     if (!item.model.supportsImageGen) return false;
+    if (params.requiresImageInput && item.model.supportsVision === false) {
+      return false;
+    }
     if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false;
     return getCanvasModelRefKeys(item).every((key) => !failedRefs.has(key));
   };
@@ -1319,8 +1345,49 @@ export async function generateCanvasImage(
     selectedModel = imageModels[0];
   }
 
+  const normalizedReferenceImages = Array.from(
+    new Set(
+      [
+        ...referenceImages,
+        ...(referenceImage ? [referenceImage] : []),
+      ]
+        .map((item) => String(item || "").trim())
+        .filter((url) => /^https?:\/\//i.test(url))
+    )
+  ).slice(0, 12);
+  const requiresImageInput = normalizedReferenceImages.length > 0;
+
   let provider = selectedModel.provider;
 
+  if (requiresImageInput && selectedModel.model.supportsVision === false) {
+    if (request.allowFailover === false) {
+      throw new Error("当前图片模型不支持参考图/图生图，请选择支持图像输入的生图模型。");
+    }
+
+    const imageInputFallback = findCanvasFailoverModel({
+      selectedModel,
+      imageModels,
+      failedModelRefs: getCanvasModelRefKeys(selectedModel),
+      requiresImageInput,
+    });
+
+    if (!imageInputFallback) {
+      throw new Error("当前任务需要参考图输入，但暂无可用的图生图模型。");
+    }
+
+    console.warn("[Canvas Generate Image] Fallback to reference-image capable model:", {
+      debugLabel,
+      requestedModel: selectedModel.model.modelId || selectedModel.model.id,
+      fallbackModel: imageInputFallback.model.modelId || imageInputFallback.model.id,
+      requestedProviderType: provider.type,
+      fallbackProviderType: imageInputFallback.provider.type,
+      reason: "selected-model-lacks-reference-image-support",
+    });
+
+    selectedModel = imageInputFallback;
+    provider = selectedModel.provider;
+  }
+
   if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(provider.type)) {
     if (request.allowFailover === false) {
       throw new Error(
@@ -1333,6 +1400,7 @@ export async function generateCanvasImage(
     for (const candidate of imageModels) {
       const candidateProvider = candidate.provider;
       if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(candidateProvider.type)) continue;
+      if (requiresImageInput && candidate.model.supportsVision === false) continue;
       fallbackModel = candidate;
       fallbackProvider = candidateProvider;
       break;
@@ -1374,16 +1442,6 @@ export async function generateCanvasImage(
     });
   }
 
-  const normalizedReferenceImages = Array.from(
-    new Set(
-      [
-        ...referenceImages,
-        ...(referenceImage ? [referenceImage] : []),
-      ]
-        .map((item) => String(item || "").trim())
-        .filter((url) => /^https?:\/\//i.test(url))
-    )
-  ).slice(0, 12);
   const { kept: reachableReferenceImages, dropped: unreachableReferenceImages } =
     await filterReachableRemoteUrls(normalizedReferenceImages, { timeoutMs: 3000 });
   const effectiveReferenceImages =
@@ -1850,6 +1908,7 @@ export async function generateCanvasImage(
         selectedModel,
         imageModels,
         failedModelRefs,
+        requiresImageInput: normalizedReferenceImages.length > 0,
       })
     : null;