diff --git a/lib/ai/providers/openai-image.ts b/lib/ai/providers/openai-image.ts index 128616366..cbf7fa417 100644 --- a/lib/ai/providers/openai-image.ts +++ b/lib/ai/providers/openai-image.ts @@ -240,6 +240,11 @@ function isLongRunningComflyModel(modelId?: string) { return normalized.includes("gpt-image-2") && normalized.includes("4k"); } +function shouldUseComflyChatImageEdit(baseURL: string, modelId?: string) { + const normalizedModelId = String(modelId || "").trim().toLowerCase(); + return isComflyBaseURL(baseURL) && normalizedModelId.includes("gpt-image-2"); +} + function resolveRequestTimeoutMs( baseURL: string, imageSize?: "1K" | "2K" | "4K", @@ -457,6 +462,155 @@ async function parseJsonResponse(response: Response) { } } +function extractImageUrlsFromText(text: string) { + const normalized = String(text || ""); + if (!normalized) return []; + + const urls = new Set(); + const markdownImageRegex = /!\[[^\]]*]\(([^)\s]+)(?:\s+"[^"]*")?\)/g; + for (const match of normalized.matchAll(markdownImageRegex)) { + if (match[1]) urls.add(match[1].trim()); + } + + const htmlImageRegex = /]*\bsrc=["']([^"']+)["'][^>]*>/gi; + for (const match of normalized.matchAll(htmlImageRegex)) { + if (match[1]) urls.add(match[1].trim()); + } + + const dataImageRegex = /data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g; + for (const match of normalized.matchAll(dataImageRegex)) { + if (match[0]) urls.add(match[0].trim()); + } + + const rawUrlRegex = /https?:\/\/[^\s"'<>()[\]{}]+/g; + for (const match of normalized.matchAll(rawUrlRegex)) { + if (match[0]) urls.add(match[0].replace(/[),.;]+$/g, "").trim()); + } + + return Array.from(urls).filter(Boolean); +} + +function collectImageCandidates(value: any, candidates = new Set()) { + if (!value) return candidates; + + if (typeof value === "string") { + for (const url of extractImageUrlsFromText(value)) { + candidates.add(url); + } + return candidates; + } + + if (Array.isArray(value)) { + for (const item of value) collectImageCandidates(item, candidates); + return candidates; + } + + if (typeof value !== "object") return candidates; + + const directUrl = + value.url || + value.image_url?.url || + value.imageUrl || + value.imageURL || + value.output_url || + value.outputUrl || + value.file_url || + value.fileUrl; + if (typeof directUrl === "string") { + candidates.add(directUrl.trim()); + } + + const base64Data = + value.b64_json || + value.base64 || + value.base64Data || + value.image_base64 || + value.imageBase64 || + value.inline_data?.data || + value.inlineData?.data; + if (typeof base64Data === "string" && base64Data.trim()) { + const mimeType = + value.mime_type || + value.mimeType || + value.inline_data?.mime_type || + value.inlineData?.mimeType || + "image/png"; + if (base64Data.startsWith("data:image/")) { + candidates.add(base64Data.trim()); + } else { + candidates.add(`data:${mimeType};base64,${base64Data.trim()}`); + } + } + + for (const key of [ + "content", + "message", + "choices", + "data", + "images", + "results", + "result", + "output", + "outputs", + "image", + "image_url", + ]) { + if (value[key]) collectImageCandidates(value[key], candidates); + } + + return candidates; +} + +async function requestComflyChatImageEdit(params: { + baseURL: string; + apiKey: string; + model: string; + prompt: string; + imageUrls: string[]; + signal: AbortSignal; +}) { + const { baseURL, apiKey, model, prompt, imageUrls, signal } = params; + const endpoint = joinEndpoint(baseURL, "/chat/completions"); + const content = [ + { type: "text", text: prompt }, + ...imageUrls.map((url) => ({ + type: "image_url", + image_url: { url }, + })), + ]; + + console.log("[OpenAI Image] Using Comfly chat image edit", { + model, + endpoint, + imageCount: imageUrls.length, + }); + + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + model, + stream: false, + messages: [ + { + role: "user", + content, + }, + ], + }), + signal, + }); + const payload = await parseJsonResponse(response); + if (!response.ok) { + throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); + } + return payload; +} + async function requestOpenAIImageGeneration(params: { baseURL: string; apiKey: string; @@ -495,6 +649,17 @@ async function requestOpenAIImageGeneration(params: { try { if (imageUrls.length > 0) { + if (shouldUseComflyChatImageEdit(baseURL, model)) { + return requestComflyChatImageEdit({ + baseURL, + apiKey, + model, + prompt, + imageUrls, + signal: controller.signal, + }); + } + const endpoint = joinEndpoint(baseURL, "/images/edits"); const formData = new FormData(); formData.append("model", model); @@ -570,18 +735,44 @@ async function requestOpenAIImageGeneration(params: { async function extractUploadedImages(payload: any): Promise { const data = Array.isArray(payload?.data) ? payload.data : []; const uploadedImages: UploadedImage[] = []; + const handledSources = new Set(); for (const item of data) { if (item?.url && typeof item.url === "string") { + handledSources.add(item.url); uploadedImages.push(await downloadAndUploadImage(item.url)); continue; } if (item?.b64_json && typeof item.b64_json === "string") { + handledSources.add(item.b64_json); uploadedImages.push(await uploadBase64ToS3(item.b64_json, "image/png")); } } + const candidates = Array.from(collectImageCandidates(payload)); + for (const candidate of candidates) { + if (handledSources.has(candidate)) continue; + + try { + if (candidate.startsWith("data:image/")) { + const match = candidate.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/); + if (match) { + handledSources.add(candidate); + uploadedImages.push(await uploadBase64ToS3(match[2], match[1] || "image/png")); + } + } else if (/^https?:\/\//i.test(candidate)) { + handledSources.add(candidate); + uploadedImages.push(await downloadAndUploadImage(candidate)); + } + } catch (error) { + console.warn("[OpenAI Image] Failed to ingest image candidate:", { + candidate, + error: error instanceof Error ? error.message : String(error), + }); + } + } + return uploadedImages; }