From 038730a24ba6d8adbbb0836bb969341af0d51308 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 22 Jun 2026 16:29:13 +0800 Subject: [PATCH] fix: preserve auto image ratio for agnes edits --- app/(chat)/api/chat/schema.ts | 2 + components/canvas/CanvasChatPanel.tsx | 96 ++++++++++++++++--- .../canvas/chat-panel/canvas-chat-session.ts | 6 ++ components/canvas/nodes/ImageNode.tsx | 20 ++++ lib/ai/providers/openai-image.ts | 78 +++++++++++---- lib/types.ts | 2 + store/canvas/index.ts | 2 + 7 files changed, 176 insertions(+), 30 deletions(-) diff --git a/app/(chat)/api/chat/schema.ts b/app/(chat)/api/chat/schema.ts index c634f63e5..9775e07dd 100644 --- a/app/(chat)/api/chat/schema.ts +++ b/app/(chat)/api/chat/schema.ts @@ -11,6 +11,8 @@ const filePartSchema = z.object({ name: z.string().min(1).max(100), url: z.string().url(), assetId: z.string().uuid().optional(), + width: z.number().positive().optional(), + height: z.number().positive().optional(), }); const partSchema = z.union([textPartSchema, filePartSchema]); diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 82d18ea60..97813d63d 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -1810,13 +1810,23 @@ export function CanvasChatPanel({ const getCanvasNodeAttachment = useCallback( (node: any, fallbackName: string): PickedObjectAttachment | null => { if (isCanvasImageUploadPending(node)) return null; - const src = (node?.data as any)?.src; + const nodeData = (node?.data as any) || {}; + const src = nodeData.src; if (!src || typeof src !== "string" || !isValidChatHttpUrl(src)) return null; const nodeTitle = - String((node?.data as any)?.title || "").trim() || - String((node?.data as any)?.prompt || "").trim() || + String(nodeData.title || "").trim() || + String(nodeData.prompt || "").trim() || fallbackName; + const metadata = (nodeData.metadata || {}) as Record; + const width = + Number(metadata.originalWidth || 0) || + Number(node?.size?.width || 0) || + 0; + const height = + Number(metadata.originalHeight || 0) || + Number(node?.size?.height || 0) || + 0; return { url: src, @@ -1826,6 +1836,8 @@ export function CanvasChatPanel({ typeof (node?.data as any)?.assetId === "string" && (node?.data as any)?.assetId.trim() ? String((node?.data as any)?.assetId).trim() : undefined, + width: Number.isFinite(width) && width > 0 ? Math.round(width) : undefined, + height: Number.isFinite(height) && height > 0 ? Math.round(height) : undefined, }; }, [] @@ -2079,11 +2091,31 @@ export function CanvasChatPanel({ currentAttachments.length + i ); + const pickedNode = nodes.find((node) => node.id === picked.imageNodeId); + const pickedNodeMetadata = + ((pickedNode?.data as any)?.metadata || {}) as Record; + const pickedWidth = + Number(pickedNodeMetadata.originalWidth || 0) || + Number(pickedNode?.size?.width || 0) || + 0; + const pickedHeight = + Number(pickedNodeMetadata.originalHeight || 0) || + Number(pickedNode?.size?.height || 0) || + 0; + // 添加到附件列表（带物品信息，原图片 URL 会发送给 AI 用于多图融合编辑） currentAttachments.push({ url: picked.imageUrl, // 原图片 URL name: `${picked.objectName}-${picked.objectPosition}`, // 物品名称-位置 contentType: 'image/jpeg', + width: + Number.isFinite(pickedWidth) && pickedWidth > 0 + ? Math.round(pickedWidth) + : undefined, + height: + Number.isFinite(pickedHeight) && pickedHeight > 0 + ? Math.round(pickedHeight) + : undefined, pickedObject: { objectName: picked.objectName, objectPosition: picked.objectPosition, @@ -2105,7 +2137,7 @@ export function CanvasChatPanel({ // 清空待添加列表 clearPendingPickedObjects(); } - }, [pendingPickedObjects, attachments, insertPickedObjectTagInEditor, clearPendingPickedObjects]); + }, [pendingPickedObjects, attachments, insertPickedObjectTagInEditor, clearPendingPickedObjects, nodes]); // 🆕 标签悬停预览 - 使用全局事件监听 useEffect(() => { @@ -4594,13 +4626,19 @@ export function CanvasChatPanel({ return; } - const parts: any[] = launchAttachments.map((attachment) => ({ - type: "file" as const, - mediaType: normalizeChatMediaType(attachment.contentType), - name: normalizeChatAttachmentName(attachment.name), - url: attachment.url, - assetId: isValidChatUuid(attachment.assetId) ? attachment.assetId : undefined, - })); + const parts: any[] = launchAttachments.map((attachment) => { + const width = Number(attachment.width); + const height = Number(attachment.height); + return { + type: "file" as const, + mediaType: normalizeChatMediaType(attachment.contentType), + name: normalizeChatAttachmentName(attachment.name), + url: attachment.url, + assetId: isValidChatUuid(attachment.assetId) ? attachment.assetId : undefined, + width: Number.isFinite(width) && width > 0 ? Math.round(width) : undefined, + height: Number.isFinite(height) && height > 0 ? Math.round(height) : undefined, + }; + }); if (textContent) { parts.push({ type: "text" as const, text: textContent }); @@ -7694,12 +7732,22 @@ export function CanvasChatPanel({ effectiveAttachments.forEach((a) => { const att = a as PickedObjectAttachment; + const attachmentWidth = Number(att.width); + const attachmentHeight = Number(att.height); parts.push({ type: "file" as const, mediaType: normalizeChatMediaType(a.contentType), name: normalizeChatAttachmentName(a.name), url: a.url, assetId: isValidChatUuid(a.assetId) ? a.assetId : undefined, + width: + Number.isFinite(attachmentWidth) && attachmentWidth > 0 + ? Math.round(attachmentWidth) + : undefined, + height: + Number.isFinite(attachmentHeight) && attachmentHeight > 0 + ? Math.round(attachmentHeight) + : undefined, hasAnnotations: hasAnnotatedImages, // 🆕 标记图片是否有标注 pickedObject: att.pickedObject, // 🆕 包含物品信息 }); @@ -7989,7 +8037,14 @@ export function CanvasChatPanel({ const uploadImageFileAndAddToCanvas = useCallback(async ( file: File, nodeIndex: number - ): Promise<{ imageUrl: string; name: string; contentType: string; assetId?: string } | null> => { + ): Promise<{ + imageUrl: string; + name: string; + contentType: string; + assetId?: string; + width?: number; + height?: number; + } | null> => { if (!file.type.startsWith('image/')) { toast.error('请选择图片文件'); return null; @@ -8046,6 +8101,8 @@ export function CanvasChatPanel({ name: fileName, contentType: file.type || 'image/jpeg', assetId, + width: dimensions.originalWidth || dimensions.width, + height: dimensions.originalHeight || dimensions.height, }; } catch (_error) { toast.error('图片上传失败'); @@ -8077,6 +8134,8 @@ export function CanvasChatPanel({ name: uploaded.name, contentType: uploaded.contentType, assetId: uploaded.assetId, + width: uploaded.width, + height: uploaded.height, }; updateAttachmentAtAgentSlot(targetSlotIndex, attachment); currentSlots[targetSlotIndex] = attachment; @@ -8117,6 +8176,8 @@ export function CanvasChatPanel({ url: uploaded.imageUrl, name: uploaded.name, contentType: uploaded.contentType, + width: uploaded.width, + height: uploaded.height, }; updateAttachmentAtVideoSlot(targetSlotIndex, attachment); currentSlots[targetSlotIndex] = attachment; @@ -8161,6 +8222,9 @@ export function CanvasChatPanel({ url: uploaded.imageUrl, name: uploaded.name, contentType: uploaded.contentType, + assetId: uploaded.assetId, + width: uploaded.width, + height: uploaded.height, }, ]; }); @@ -8212,6 +8276,8 @@ export function CanvasChatPanel({ name: uploaded.name, contentType: uploaded.contentType, assetId: uploaded.assetId, + width: uploaded.width, + height: uploaded.height, }); currentSlots[targetSlotIndex] = { @@ -8219,6 +8285,8 @@ export function CanvasChatPanel({ name: uploaded.name, contentType: uploaded.contentType, assetId: uploaded.assetId, + width: uploaded.width, + height: uploaded.height, }; handledCount += 1; uploadIndex += 1; @@ -8270,6 +8338,7 @@ export function CanvasChatPanel({ const imageUrl = data.url; const assetId = typeof data.assetId === "string" && data.assetId.trim() ? data.assetId.trim() : undefined; + const dimensions = await getImageDimensions(imageUrl); // 记录第一张图片的 URL（用于自动命名和缩略图） if (!firstImageUrl) { @@ -8289,11 +8358,12 @@ export function CanvasChatPanel({ name: file.name, contentType: file.type, assetId, + width: dimensions.originalWidth || dimensions.width, + height: dimensions.originalHeight || dimensions.height, }]; }); // 获取图片原始尺寸后添加到画布 - const dimensions = await getImageDimensions(imageUrl); // 使用 baseNodeCount + uploadIndex 计算位置，确保多张图片不重叠 const nodeIndex = baseNodeCount + uploadIndex; const spacing = 20; diff --git a/components/canvas/chat-panel/canvas-chat-session.ts b/components/canvas/chat-panel/canvas-chat-session.ts index 1c5580e83..63c60b654 100644 --- a/components/canvas/chat-panel/canvas-chat-session.ts +++ b/components/canvas/chat-panel/canvas-chat-session.ts @@ -83,6 +83,10 @@ export function sanitizeChatAttachment( if (!attachment || !isValidChatHttpUrl(attachment.url)) { return null; } + const width = Number(attachment.width); + const height = Number(attachment.height); + const safeWidth = Number.isFinite(width) && width > 0 ? Math.round(width) : undefined; + const safeHeight = Number.isFinite(height) && height > 0 ? Math.round(height) : undefined; return { ...attachment, @@ -90,6 +94,8 @@ export function sanitizeChatAttachment( name: normalizeChatAttachmentName(attachment.name), contentType: normalizeChatMediaType(attachment.contentType), assetId: isValidChatUuid(attachment.assetId) ? attachment.assetId : undefined, + width: safeWidth, + height: safeHeight, }; } diff --git a/components/canvas/nodes/ImageNode.tsx b/components/canvas/nodes/ImageNode.tsx index 8193a7e4a..df8671ca9 100644 --- a/components/canvas/nodes/ImageNode.tsx +++ b/components/canvas/nodes/ImageNode.tsx @@ -4505,10 +4505,27 @@ export function ImageNode({ displayTitle ); await animateAddToChat(attachment.url); + const metadata = (data.metadata || {}) as Record; + const sourceWidth = + Number(metadata.originalWidth || 0) || + Number(node.size?.width || 0) || + 0; + const sourceHeight = + Number(metadata.originalHeight || 0) || + Number(node.size?.height || 0) || + 0; addPendingChatAttachment({ url: attachment.url, name: displayTitle, contentType: attachment.contentType, + width: + Number.isFinite(sourceWidth) && sourceWidth > 0 + ? Math.round(sourceWidth) + : undefined, + height: + Number.isFinite(sourceHeight) && sourceHeight > 0 + ? Math.round(sourceHeight) + : undefined, }); toast.success("已添加到对话框"); } catch (error) { @@ -4520,10 +4537,13 @@ export function ImageNode({ }, [ addPendingChatAttachment, animateAddToChat, + data.metadata, data.src, displayTitle, isAddingToChat, isProcessing, + node.size?.height, + node.size?.width, ]); const getFloatingViewportBounds = (toolbarHeight: number) => { diff --git a/lib/ai/providers/openai-image.ts b/lib/ai/providers/openai-image.ts index fdf610ebc..28eef4685 100644 --- a/lib/ai/providers/openai-image.ts +++ b/lib/ai/providers/openai-image.ts @@ -26,6 +26,12 @@ interface UploadedImage { original: string; } +type InputImageReference = { + url: string; + width?: number; + height?: number; +}; + const DEFAULT_IMAGE_SIZE = "2K"; const DEFAULT_TIMEOUT_MS = 600000; const LONG_RUNNING_COMFLY_TIMEOUT_MS = 600000; @@ -173,6 +179,20 @@ function inferClosestAspectRatio(aspectValue: number) { return closest; } +function inferAspectRatioFromDimensions(width?: number, height?: number) { + const safeWidth = Number(width); + const safeHeight = Number(height); + if ( + !Number.isFinite(safeWidth) || + !Number.isFinite(safeHeight) || + safeWidth <= 0 || + safeHeight <= 0 + ) { + return null; + } + return inferClosestAspectRatio(safeWidth / safeHeight); +} + async function inferAspectRatioFromImageUrl(imageUrl: string): Promise { const normalized = String(imageUrl || "").trim(); if (!normalized) return null; @@ -419,14 +439,20 @@ function extractPromptAndImages(messages: Array<{ role: string; content: any }>) } let prompt = ""; - const imageUrls: string[] = []; + const imageRefs: InputImageReference[] = []; - const pushImageUrl = (value?: string) => { + const pushImageUrl = (value?: string, geometry?: { width?: unknown; height?: unknown }) => { if (!value) return; const normalized = String(value).trim(); if (!normalized) return; - if (!imageUrls.includes(normalized)) { - imageUrls.push(normalized); + if (!imageRefs.some((item) => item.url === normalized)) { + const width = Number(geometry?.width); + const height = Number(geometry?.height); + imageRefs.push({ + url: normalized, + width: Number.isFinite(width) && width > 0 ? Math.round(width) : undefined, + height: Number.isFinite(height) && height > 0 ? Math.round(height) : undefined, + }); } }; @@ -437,11 +463,11 @@ function extractPromptAndImages(messages: Array<{ role: string; content: any }>) if (part?.type === "text" && typeof part.text === "string") { prompt = `${prompt} ${part.text}`.trim(); } else if (part?.type === "image_url" && part.image_url?.url) { - pushImageUrl(part.image_url.url); + pushImageUrl(part.image_url.url, part); } else if (part?.type === "file") { - pushImageUrl(part.url || part.data); + pushImageUrl(part.url || part.data, part); } else if (part?.type === "image") { - pushImageUrl(part.image || part.url); + pushImageUrl(part.image || part.url, part); } } } @@ -450,7 +476,7 @@ function extractPromptAndImages(messages: Array<{ role: string; content: any }>) throw new Error("No prompt found in user message"); } - return { prompt, imageUrls }; + return { prompt, imageUrls: imageRefs.map((item) => item.url), imageRefs }; } async function loadInputImage(source: string) { @@ -805,14 +831,19 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { imageParams?: OpenAIImageParams; overrideModelId?: string; }) => { - const { prompt, imageUrls } = extractPromptAndImages(options.messages); + const { prompt, imageUrls, imageRefs } = extractPromptAndImages(options.messages); const activeModelId = options.overrideModelId || modelId; const normalizedAspectRatio = normalizeAspectRatio(options.imageParams?.aspectRatio); const promptAspectRatio = inferAspectRatioFromPrompt(prompt); - const referenceAspectRatio = - imageUrls.length > 0 - ? await inferAspectRatioFromImageUrl(imageUrls[0] || "") + const referenceGeometryAspectRatio = + imageRefs.length > 0 + ? inferAspectRatioFromDimensions(imageRefs[0]?.width, imageRefs[0]?.height) : null; + const referenceAspectRatio = + referenceGeometryAspectRatio || + (imageUrls.length > 0 + ? await inferAspectRatioFromImageUrl(imageUrls[0] || "") + : null); const shouldPreferReferenceAspectRatio = imageUrls.length > 0 && isAgnesImageModel(baseURL, activeModelId); const aspectRatio = @@ -826,11 +857,22 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { ? inferImageSizeFromModelId(activeModelId) : undefined; const imageSize = modelImageSize || requestedImageSize; - const size = - modelImageSize && aspectRatio - ? resolveProviderImageSize(baseURL, aspectRatio, imageSize) - : options.imageParams?.size || - (aspectRatio ? resolveProviderImageSize(baseURL, aspectRatio, imageSize) : undefined); + const shouldOmitAgnesAutoSize = + isAgnesImageModel(baseURL, activeModelId) && + imageUrls.length > 0 && + normalizedAspectRatio === "auto" && + (!aspectRatio || aspectRatio === "auto") && + !options.imageParams?.size; + let size: string | undefined; + if (shouldOmitAgnesAutoSize) { + size = undefined; + } else if (modelImageSize && aspectRatio) { + size = resolveProviderImageSize(baseURL, aspectRatio, imageSize); + } else { + size = + options.imageParams?.size || + (aspectRatio ? resolveProviderImageSize(baseURL, aspectRatio, imageSize) : undefined); + } const quality = normalizeQuality(options.imageParams?.quality); const background = normalizeBackground(options.imageParams?.background); const outputFormat = normalizeOutputFormat(options.imageParams?.outputFormat); @@ -841,8 +883,10 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { size, aspectRatio: aspectRatio || "auto", referenceAspectRatio: referenceAspectRatio || undefined, + referenceGeometryAspectRatio: referenceGeometryAspectRatio || undefined, preferReferenceAspectRatio: shouldPreferReferenceAspectRatio, promptAspectRatio: promptAspectRatio || undefined, + omittedSizeForAgnesAuto: shouldOmitAgnesAutoSize || undefined, imageSize, requestedImageSize, modelImageSize, diff --git a/lib/types.ts b/lib/types.ts index c5995e81b..7baacf87f 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -56,4 +56,6 @@ export type Attachment = { url: string; contentType: string; assetId?: string; + width?: number; + height?: number; }; diff --git a/store/canvas/index.ts b/store/canvas/index.ts index c626f9fe6..fd3497ae6 100644 --- a/store/canvas/index.ts +++ b/store/canvas/index.ts @@ -45,6 +45,8 @@ interface PendingChatAttachment { url: string; name: string; contentType: string; + width?: number; + height?: number; } // 🆕 拾取的物品信息