diff --git a/app/(chat)/api/chat/_lib/image-failover.ts b/app/(chat)/api/chat/_lib/image-failover.ts index 2ae73f80d..e31efea5f 100644 --- a/app/(chat)/api/chat/_lib/image-failover.ts +++ b/app/(chat)/api/chat/_lib/image-failover.ts @@ -111,6 +111,34 @@ export type ImageFallbackCandidate = { sameProvider: boolean; }; +function messagePartHasImageInput(part: any) { + if (!part || typeof part !== "object") return false; + if (part.type === "file" || part.type === "image" || part.type === "image_url") { + return true; + } + if (typeof part.url === "string" && /^https?:\/\//i.test(part.url)) { + return true; + } + if (typeof part.image === "string" && part.image.trim()) { + return true; + } + if (typeof part.image_url?.url === "string" && part.image_url.url.trim()) { + return true; + } + return false; +} + +export function chatMessagesHaveImageInput(messages: ChatMessage[]) { + return messages.some((message: any) => { + const content = message?.content; + if (Array.isArray(content) && content.some(messagePartHasImageInput)) { + return true; + } + const parts = message?.parts; + return Array.isArray(parts) && parts.some(messagePartHasImageInput); + }); +} + export function getImageFailoverConvertProvider(providerType?: string | null) { switch (providerType) { case "bfl-image": @@ -219,6 +247,11 @@ export async function generateImageWithFailoverCandidate(params: { const { candidate, messages, imageParams, fallbackImageParams } = params; const providerType = String(candidate.provider?.type || ""); const candidateRuntimeModelId = resolveImageModelRuntimeId(candidate.model); + if (chatMessagesHaveImageInput(messages) && candidate.model?.supportsVision === false) { + throw new Error( + `备用模型不支持参考图/图生图输入：${candidate.model.displayName || candidate.model.id}` + ); + } if ( providerType === "fal-image" && isFalGptImage2EditModelId(candidateRuntimeModelId) @@ -237,9 +270,18 @@ export async function generateImageWithFailoverCandidate(params: { messages, getImageFailoverConvertProvider(providerType) ); + const effectiveImageParams = imageParams || fallbackImageParams; + chatApiLogger.info("[Chat API] Image failover candidate params:", { + model: candidate.model?.modelId || candidate.model?.id, + providerType, + aspectRatio: effectiveImageParams?.aspectRatio, + imageSize: effectiveImageParams?.imageSize, + imageCount: effectiveImageParams?.imageCount, + hasImageInput: chatMessagesHaveImageInput(messages), + }); const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, - imageParams: imageParams || fallbackImageParams, + imageParams: effectiveImageParams, imageGenConfig: candidate.model.imageGenConfig || null, modelDisplayName: candidate.model.displayName, overrideModelId: candidateRuntimeModelId || candidate.model.modelId, diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index ffa842b27..5135ce328 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -892,6 +892,12 @@ export async function POST(request: NextRequest) { ? siteSettingsForModel.canvasDetailDefaultModelId.trim() : ""; const resolvedModel = detailFixedModel || requestedModel; + const scaleAdjustCompositeRegions = isScaleAdjustOperation + ? buildScaleAdjustCompositeRegions( + regions, + scalePercent as ScaleAdjustPercent + ) + : []; const imageRequest: CanvasGenerateImageRequest = { aspectRatio: resolvedAspectRatio, @@ -937,6 +943,7 @@ export async function POST(request: NextRequest) { referenceImages, scaleAdjustVerification: isScaleAdjustOperation ? { + protectedRegions: scaleAdjustCompositeRegions, sourceImageUrl, regions: regions.map((region) => region.rect), scalePercent: scalePercent as ScaleAdjustPercent, @@ -945,10 +952,7 @@ export async function POST(request: NextRequest) { localEditComposite: isScaleAdjustOperation ? { sourceImageUrl, - regions: buildScaleAdjustCompositeRegions( - regions, - scalePercent as ScaleAdjustPercent - ), + regions: scaleAdjustCompositeRegions, expandRegions: false, maskMode: "diff", featherMinPx: 4, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 90a4fa4e5..935b4485d 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -51,6 +51,7 @@ export interface CanvasLocalEditCompositeConfig { } export interface CanvasScaleAdjustVerificationConfig { + protectedRegions?: CanvasLocalEditCompositeRegion[]; sourceImageUrl: string; regions: CanvasLocalEditCompositeRegion[]; scalePercent: number; @@ -334,8 +335,22 @@ function normalizeScaleAdjustVerificationConfig( const regions = value.regions .map((region) => normalizeCanvasRegion(region)) .filter((region): region is CanvasLocalEditCompositeRegion => Boolean(region)); + const protectedRegions = Array.isArray(value.protectedRegions) + ? value.protectedRegions + .map((region) => normalizeCanvasRegion(region)) + .filter((region): region is CanvasLocalEditCompositeRegion => + Boolean(region) + ) + : []; - return regions.length > 0 ? { sourceImageUrl, regions, scalePercent } : null; + return regions.length > 0 + ? { + sourceImageUrl, + regions, + scalePercent, + ...(protectedRegions.length > 0 ? { protectedRegions } : {}), + } + : null; } function scaleRegionAroundCenter( @@ -402,6 +417,11 @@ function canvasRegionsIntersect( function getScaleAdjustProtectedRegions( verification: CanvasScaleAdjustVerificationConfig ) { + if (verification.protectedRegions?.length) { + return verification.protectedRegions.map((region) => + expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN) + ); + } return verification.regions.flatMap((region) => [ expandCanvasRegion(region, SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN), expandCanvasRegion( @@ -682,6 +702,7 @@ async function applyScaleAdjustVerificationToImages(params: { outsideDiffScores: outsideDiffScores.map( (score) => Math.round(score * 10) / 10 ), + protectedRegionCount: verification.protectedRegions?.length || 0, scalePercent: verification.scalePercent, }); checkedImages.push(image); @@ -700,6 +721,7 @@ async function applyScaleAdjustVerificationToImages(params: { outsideDiffScores: outsideDiffScores.map( (score) => Math.round(score * 10) / 10 ), + protectedRegionCount: verification.protectedRegions?.length || 0, reason: rejection.reason, scalePercent: verification.scalePercent, }); @@ -1202,6 +1224,7 @@ function findCanvasFailoverModel(params: { selectedModel: CanvasModelWithProvider; imageModels: CanvasModelWithProvider[]; failedModelRefs: string[]; + requiresImageInput?: boolean; }) { const failedRefs = new Set( params.failedModelRefs.map((item) => String(item || "").trim()).filter(Boolean) @@ -1218,6 +1241,9 @@ function findCanvasFailoverModel(params: { return false; } if (!item.model.supportsImageGen) return false; + if (params.requiresImageInput && item.model.supportsVision === false) { + return false; + } if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false; return getCanvasModelRefKeys(item).every((key) => !failedRefs.has(key)); }; @@ -1319,8 +1345,49 @@ export async function generateCanvasImage( selectedModel = imageModels[0]; } + const normalizedReferenceImages = Array.from( + new Set( + [ + ...referenceImages, + ...(referenceImage ? [referenceImage] : []), + ] + .map((item) => String(item || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)) + ) + ).slice(0, 12); + const requiresImageInput = normalizedReferenceImages.length > 0; + let provider = selectedModel.provider; + if (requiresImageInput && selectedModel.model.supportsVision === false) { + if (request.allowFailover === false) { + throw new Error("当前图片模型不支持参考图/图生图，请选择支持图像输入的生图模型。"); + } + + const imageInputFallback = findCanvasFailoverModel({ + selectedModel, + imageModels, + failedModelRefs: getCanvasModelRefKeys(selectedModel), + requiresImageInput, + }); + + if (!imageInputFallback) { + throw new Error("当前任务需要参考图输入，但暂无可用的图生图模型。"); + } + + console.warn("[Canvas Generate Image] Fallback to reference-image capable model:", { + debugLabel, + requestedModel: selectedModel.model.modelId || selectedModel.model.id, + fallbackModel: imageInputFallback.model.modelId || imageInputFallback.model.id, + requestedProviderType: provider.type, + fallbackProviderType: imageInputFallback.provider.type, + reason: "selected-model-lacks-reference-image-support", + }); + + selectedModel = imageInputFallback; + provider = selectedModel.provider; + } + if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(provider.type)) { if (request.allowFailover === false) { throw new Error( @@ -1333,6 +1400,7 @@ export async function generateCanvasImage( for (const candidate of imageModels) { const candidateProvider = candidate.provider; if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(candidateProvider.type)) continue; + if (requiresImageInput && candidate.model.supportsVision === false) continue; fallbackModel = candidate; fallbackProvider = candidateProvider; break; @@ -1374,16 +1442,6 @@ export async function generateCanvasImage( }); } - const normalizedReferenceImages = Array.from( - new Set( - [ - ...referenceImages, - ...(referenceImage ? [referenceImage] : []), - ] - .map((item) => String(item || "").trim()) - .filter((url) => /^https?:\/\//i.test(url)) - ) - ).slice(0, 12); const { kept: reachableReferenceImages, dropped: unreachableReferenceImages } = await filterReachableRemoteUrls(normalizedReferenceImages, { timeoutMs: 3000 }); const effectiveReferenceImages = @@ -1850,6 +1908,7 @@ export async function generateCanvasImage( selectedModel, imageModels, failedModelRefs, + requiresImageInput: normalizedReferenceImages.length > 0, }) : null;