import "server-only"; import { resolveGrsaiGptImageSize } from "@/lib/ai/grsai-gpt-image-sizes"; import { buildReferenceImageOrderPrompt } from "@/lib/ai/providers/reference-image-order"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; import { createLogger } from "@/lib/observability/logger"; const logger = createLogger("gemini-image"); interface GeminiImageConfig { apiKey: string; baseURL: string; } /** * 压缩图片生成缩略图(使用 sharp) */ async function createThumbnail(buffer: Buffer, mimeType: string): Promise { try { const sharp = await import("sharp"); // 创建缩略图(宽度 500px,高度按比例,质量 80%) const thumbnailBuffer = await sharp.default(buffer) .resize(500, null, { fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 80 }) .toBuffer(); logger.debug(`[Gemini Image] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`); return thumbnailBuffer; } catch (error) { logger.error("[Gemini Image] Failed to create thumbnail:", error); // 如果失败,返回原图 return buffer; } } /** * 确保图片是公网可访问的 URL(如果是 Base64,上传到 S3) */ async function ensureImageUrl(url: string): Promise { if (url.startsWith("http")) return url; if (url.startsWith("data:")) { try { logger.debug("[Gemini Image] Converting Base64 reference image to S3 URL..."); // 解析 Base64 const matches = url.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/); if (!matches) { logger.warn("[Gemini Image] Invalid Base64 format, skipping upload"); return url; } const mimeType = matches[1]; const base64Data = matches[2]; const buffer = Buffer.from(base64Data, "base64"); const ext = mimeType.split("/")[1] || "png"; const filename = `ref-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`; // 上传到 S3(需要转换为 ArrayBuffer) const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer; const result = await uploadFile(filename, arrayBuffer); logger.debug(`[Gemini Image] Uploaded reference image to: ${result.url}`); return result.url; } catch (error) { logger.error("[Gemini Image] Failed to upload reference image to S3:", error); // 如果上传失败,返回原 Base64(虽然可能也会失败,但没别的办法) return url; } } return url; } async function inferImageAspectRatioFromUrl(url: string): Promise { const normalized = String(url || "").trim(); if (!normalized) return null; try { let buffer: Buffer | null = null; if (normalized.startsWith("data:")) { const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/); if (!match) return null; buffer = Buffer.from(match[2], "base64"); } else if (/^https?:\/\//i.test(normalized)) { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 20000); try { const response = await fetch(normalized, { signal: controller.signal, headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", }, }); if (!response.ok) return null; buffer = Buffer.from(await response.arrayBuffer()); } finally { clearTimeout(timeoutId); } } if (!buffer?.length) return null; const sharp = await import("sharp"); const metadata = await sharp.default(buffer).metadata(); if (!metadata.width || !metadata.height || metadata.height <= 0) { return null; } return `${metadata.width}:${metadata.height}`; } catch (error) { logger.warn("[Gemini Image] Failed to infer reference aspect ratio:", error); return null; } } /** * 将 base64 图片上传到 S3,返回缩略图和原图的 URL */ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jpeg"): Promise<{ thumbnail: string; original: string }> { try { // 解析 base64 数据 const buffer = Buffer.from(base64Data, "base64"); // 生成文件名 const ext = mimeType.split("/")[1] || "jpg"; const timestamp = Date.now(); const randomStr = Math.random().toString(36).slice(2, 8); const thumbnailFilename = `gemini-thumb-${timestamp}-${randomStr}.${ext}`; const originalFilename = `gemini-orig-${timestamp}-${randomStr}.${ext}`; logger.debug(`[Gemini Image] Original size: ${buffer.length} bytes`); // 生成缩略图 const thumbnailBuffer = await createThumbnail(buffer, mimeType); // 转换为 ArrayBuffer const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength ) as ArrayBuffer; const originalArrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; // 并行上传缩略图和原图到 S3 logger.debug(`[Gemini Image] Uploading thumbnail and original to S3...`); const [thumbnailResult, originalResult] = await Promise.all([ uploadFile(thumbnailFilename, thumbnailArrayBuffer), uploadFile(originalFilename, originalArrayBuffer), ]); logger.debug(`[Gemini Image] Uploaded to S3:`); logger.debug(` - Thumbnail: ${thumbnailResult.url}`); logger.debug(` - Original: ${originalResult.url}`); return { thumbnail: thumbnailResult.url, original: originalResult.url, }; } catch (error) { logger.error("[Gemini Image] Failed to upload to S3:", error); // 如果上传失败,抛出错误,不要返回 Base64,否则会导致前端崩溃和数据库保存失败 throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); } } /** * 下载远程URL图片并上传到 S3,返回缩略图和原图的 URL * 带重试机制,确保成功下载并上传到国内中转 */ async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> { const deferred = await createDeferredUploadedImage(imageUrl, { providerType: "gemini-image", }); if (deferred) return deferred; for (let attempt = 0; attempt < retries; attempt++) { try { logger.debug(`[Gemini Image] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`); // 下载图片(增加超时时间到 30 秒) const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 30000); const response = await fetch(imageUrl, { signal: controller.signal, headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", }, }); clearTimeout(timeoutId); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } // 获取图片数据 const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); if (buffer.length === 0) { throw new Error("下载的图片数据为空"); } // 从响应头或URL获取MIME类型 const contentType = response.headers.get("content-type") || "image/jpeg"; const ext = contentType.split("/")[1] || "jpg"; // 生成文件名 const timestamp = Date.now(); const randomStr = Math.random().toString(36).slice(2, 8); const thumbnailFilename = `gemini-thumb-${timestamp}-${randomStr}.${ext}`; const originalFilename = `gemini-orig-${timestamp}-${randomStr}.${ext}`; logger.debug(`[Gemini Image] Downloaded size: ${buffer.length} bytes`); // 生成缩略图 const thumbnailBuffer = await createThumbnail(buffer, contentType); // 转换为 ArrayBuffer const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength ) as ArrayBuffer; const originalArrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; // 并行上传缩略图和原图到 S3 logger.debug(`[Gemini Image] Uploading downloaded image to S3...`); const [thumbnailResult, originalResult] = await Promise.all([ uploadFile(thumbnailFilename, thumbnailArrayBuffer), uploadFile(originalFilename, originalArrayBuffer), ]); logger.debug(`[Gemini Image] Uploaded to S3:`); logger.debug(` - Thumbnail: ${thumbnailResult.url}`); logger.debug(` - Original: ${originalResult.url}`); return { thumbnail: thumbnailResult.url, original: originalResult.url, }; } catch (error) { const isLastAttempt = attempt === retries - 1; logger.error( `[Gemini Image] Download attempt ${attempt + 1}/${retries} failed:`, error instanceof Error ? error.message : String(error) ); if (isLastAttempt) { // 最后一次尝试失败,抛出错误(不要 fallback 到原始 URL) throw new Error( `图片下载并上传失败(已重试 ${retries} 次): ${error instanceof Error ? error.message : "未知错误"}` ); } // 等待后重试(指数退避) const delay = Math.pow(2, attempt) * 1000; // 1s, 2s, 4s... logger.debug(`[Gemini Image] Retrying in ${delay}ms...`); await new Promise((resolve) => setTimeout(resolve, delay)); } } // 理论上不会到达这里,但 TypeScript 需要 throw new Error("下载失败:未知错误"); } // Gemini API 响应格式(参考灵魂助手3.0的格式) interface GeminiResponse { candidates: Array<{ content: { parts: Array<{ text?: string; inline_data?: { mime_type: string; data: string; }; inlineData?: { mimeType: string; data: string; }; }>; }; }>; } function joinOpenAICompatibleBaseUrl(baseURL: string, path: string) { const normalized = String(baseURL || "").trim().replace(/\/+$/, ""); const withoutVersion = normalized.replace(/\/v1$/i, ""); return `${withoutVersion}${path}`; } async function loadImageAsBlob(imageUrl: string): Promise<{ blob: Blob; filename: string }> { const normalized = String(imageUrl || "").trim(); if (!normalized) { throw new Error("参考图地址为空"); } if (normalized.startsWith("data:")) { const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/); if (!match) { throw new Error("参考图 Base64 格式无效"); } const mimeType = match[1] || "image/png"; const ext = mimeType.split("/")[1] || "png"; return { blob: new Blob([Buffer.from(match[2], "base64")], { type: mimeType }), filename: `reference.${ext}`, }; } const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 30000); try { const response = await fetch(normalized, { signal: controller.signal, headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", }, }); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } const contentType = response.headers.get("content-type") || "image/png"; const extFromMime = contentType.split("/")[1]?.split(";")[0] || "png"; const extFromUrl = normalized .split("?")[0] .match(/\.([A-Za-z0-9]+)$/)?.[1]; const ext = extFromUrl || extFromMime; return { blob: new Blob([await response.arrayBuffer()], { type: contentType }), filename: `reference.${ext}`, }; } finally { clearTimeout(timeoutId); } } /** * 从文本中提取图片数据 * 返回 { url, isBase64, mimeType, base64Data } 数组 */ function extractImageDataFromText(text: string): Array<{ url: string; isBase64: boolean; mimeType?: string; base64Data?: string }> { const images: Array<{ url: string; isBase64: boolean; mimeType?: string; base64Data?: string }> = []; // 匹配所有 Markdown 图片格式: ![任意文本](URL) const allImageMatches = text.matchAll(/!\[[^\]]*\]\(([^)]+)\)/g); for (const match of allImageMatches) { const url = match[1]; // 检查是否是 base64 格式 const base64Match = url.match(/^data:image\/([^;]+);base64,(.+)$/); if (base64Match) { images.push({ url, isBase64: true, mimeType: `image/${base64Match[1]}`, base64Data: base64Match[2], }); } else { images.push({ url, isBase64: false }); } } return images; } function parseDataImageUrl(value: string): { mimeType: string; base64Data: string } | null { const match = String(value || "").trim().match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/); if (!match) return null; return { mimeType: match[1] || "image/png", base64Data: match[2], }; } /** * 从 API 响应中提取图片并上传 base64 到 S3 * 返回图片数据数组(包含缩略图和原图 URL) */ async function extractAndUploadImages(response: any): Promise> { logger.debug("[Gemini Image] Extracting images from response:", JSON.stringify(response).substring(0, 500)); // 🔍 调试日志 const images: Array<{ thumbnail: string; original: string }> = []; const uploadPromises: Promise[] = []; const queuedUrls = new Set(); const queueRemoteImageUrl = (imageUrl: unknown, source: string) => { const normalized = String(imageUrl || "").trim(); if (!normalized || queuedUrls.has(normalized)) return; queuedUrls.add(normalized); if (normalized.startsWith("data:")) { const imageData = parseDataImageUrl(normalized); if (imageData?.base64Data) { const uploadPromise = uploadBase64ToS3(imageData.base64Data, imageData.mimeType) .then((urls) => { images.push(urls); }); uploadPromises.push(uploadPromise); } return; } if (!/^https?:\/\//i.test(normalized)) { for (const imageData of extractImageDataFromText(normalized)) { if (imageData.isBase64 && imageData.base64Data) { const uploadPromise = uploadBase64ToS3(imageData.base64Data, imageData.mimeType) .then((urls) => { images.push(urls); }); uploadPromises.push(uploadPromise); } else { queueRemoteImageUrl(imageData.url, source); } } return; } logger.debug( `[Gemini Image] Found ${source} image URL: ${normalized.substring(0, 100)}...` ); const uploadPromise = downloadAndUploadToS3(normalized) .then((urls) => { images.push(urls); }) .catch((error) => { logger.error(`[Gemini Image] Failed to download/upload ${source} URL after retries:`, error); throw new Error( `无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}` ); }); uploadPromises.push(uploadPromise); }; const queueImageValue = (value: unknown, source: string) => { if (!value) return; if (Array.isArray(value)) { value.forEach((entry) => queueImageValue(entry, source)); return; } if (typeof value === "string") { queueRemoteImageUrl(value, source); return; } if (typeof value === "object") { const item = value as Record; [ item.url, item.imageUrl, item.image_url, item.image, item.original, item.originalUrl, item.original_url, item.thumbnail, item.outputUrl, item.output_url, item.resultUrl, item.result_url, item.fileUrl, item.file_url, ].forEach((candidate) => queueRemoteImageUrl(candidate, source)); const base64Data = item.b64_json || item.base64 || item.base64Data || item.image_base64 || item.imageBase64; if (typeof base64Data === "string" && base64Data.trim()) { const mimeType = item.mime_type || item.mimeType || "image/png"; queueRemoteImageUrl( base64Data.startsWith("data:image/") ? base64Data : `data:${mimeType};base64,${base64Data.trim()}`, source ); } } }; queueImageValue(response?.url, "direct"); queueImageValue(response?.imageUrl, "direct"); queueImageValue(response?.image_url, "direct"); queueImageValue(response?.data?.url, "data"); queueImageValue(response?.data?.imageUrl, "data"); queueImageValue(response?.data?.image_url, "data"); const directArrays = [ { value: response?.images, source: "images" }, { value: response?.data?.images, source: "data.images" }, { value: response?.data?.results, source: "data.results" }, { value: response?.imageUrls, source: "imageUrls" }, { value: response?.image_urls, source: "image_urls" }, { value: response?.data?.imageUrls, source: "data.imageUrls" }, { value: response?.data?.image_urls, source: "data.image_urls" }, { value: response?.output, source: "output" }, { value: response?.outputs, source: "outputs" }, { value: response?.data?.output, source: "data.output" }, { value: response?.data?.outputs, source: "data.outputs" }, ]; for (const item of directArrays) { queueImageValue(item.value, item.source); } // 1. 尝试 OpenAI Images API 格式 (data[].b64_json / data[].url) if (Array.isArray(response.data)) { logger.debug("[Gemini Image] Parsing OpenAI Images format, data count:", response.data.length); for (const item of response.data) { if (item?.b64_json && typeof item.b64_json === "string") { const base64Data = item.b64_json.startsWith("data:") ? item.b64_json.split(",", 2)[1] || item.b64_json : item.b64_json; const uploadPromise = uploadBase64ToS3(base64Data, "image/png") .then((urls) => { images.push(urls); }); uploadPromises.push(uploadPromise); } else if (item?.url && typeof item.url === "string") { queueRemoteImageUrl(item.url, "OpenAI Images"); } } } // 2. 尝试 OpenAI Chat 格式 (choices[0].message.content) if (response.choices && response.choices.length > 0) { const content = response.choices[0]?.message?.content; if (content && typeof content === "string") { logger.debug("[Gemini Image] Parsing OpenAI format, content:", content.substring(0, 100) + "..."); const imageData = extractImageDataFromText(content); for (const img of imageData) { if (img.isBase64 && img.base64Data) { // 异步上传 base64 到 S3(生成缩略图和原图) const uploadPromise = uploadBase64ToS3(img.base64Data, img.mimeType) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } else { // 非 base64 图片(OpenAI URL 等),必须下载并上传到 S3(国内中转) logger.debug("[Gemini Image] Detected external URL, will download and upload to S3:", img.url.substring(0, 100) + "..."); const uploadPromise = downloadAndUploadToS3(img.url) .then(urls => { logger.debug("[Gemini Image] Successfully downloaded and uploaded external image to S3"); images.push(urls); }) .catch(error => { logger.error("[Gemini Image] Failed to download/upload URL after retries:", error); // 不再 fallback 到原始 URL,而是抛出错误 throw new Error(`无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}`); }); uploadPromises.push(uploadPromise); } } } } // 3. 尝试 Nano Banana API 格式 (results[].url) if (response.results && Array.isArray(response.results)) { logger.debug("[Gemini Image] Parsing Nano Banana format, results count:", response.results.length); for (const result of response.results) { if (result.url && typeof result.url === "string") { queueRemoteImageUrl(result.url, "Nano Banana"); } } } // 4. 尝试 Gemini 格式 (candidates[0].content.parts) if (response.candidates && response.candidates.length > 0) { const candidate = response.candidates[0]; if (candidate.content && candidate.content.parts) { for (const part of candidate.content.parts) { if (part.text) { const imageData = extractImageDataFromText(part.text); for (const img of imageData) { if (img.isBase64 && img.base64Data) { const uploadPromise = uploadBase64ToS3(img.base64Data, img.mimeType) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } else { // 非 base64 图片(OpenAI URL 等),必须下载并上传到 S3(国内中转) logger.debug("[Gemini Image] Detected external URL in Gemini response, will download and upload to S3:", img.url.substring(0, 100) + "..."); const uploadPromise = downloadAndUploadToS3(img.url) .then(urls => { logger.debug("[Gemini Image] Successfully downloaded and uploaded Gemini external image to S3"); images.push(urls); }) .catch(error => { logger.error("[Gemini Image] Failed to download/upload URL in Gemini response after retries:", error); // 不再 fallback 到原始 URL,而是抛出错误 throw new Error(`无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}`); }); uploadPromises.push(uploadPromise); } } } // inline_data 格式 if (part.inline_data && part.inline_data.data) { const mimeType = part.inline_data.mime_type || "image/png"; const uploadPromise = uploadBase64ToS3(part.inline_data.data, mimeType) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } else if (part.inlineData && part.inlineData.data) { const mimeType = part.inlineData.mimeType || "image/png"; const uploadPromise = uploadBase64ToS3(part.inlineData.data, mimeType) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } } } } // 等待所有上传完成 await Promise.all(uploadPromises); logger.debug("[Gemini Image] Extracted and uploaded", images.length, "images (thumbnail + original)"); return images; } /** * 将图片数据转换为 Markdown 格式 * 使用特殊格式传递缩略图和原图 URL:![thumb|||原图URL|||imageSize](缩略图URL) * 前端的 ImagePreview 组件会解析这个格式 * * 注意:不能使用 $$ 作为分隔符,因为会被 Markdown 解析器当作 LaTeX 数学公式! * 使用 ||| (三个竖线) 作为分隔符,不会与任何 Markdown 语法冲突 */ function formatImageResponse( images: Array<{ thumbnail: string; original: string }>, imageSize: string = "2K" ): string { if (images.length === 0) { throw new Error("生成图片失败,未找到图片数据"); } // 返回 Markdown 格式的图片,在 alt 文本中编码原图 URL 和 imageSize // 使用 ||| 作为分隔符,避免与 Markdown 表格语法和 LaTeX 数学公式冲突 return images.map((img) => `![thumb|||${img.original}|||${imageSize}](${img.thumbnail})`).join("\n\n"); } const GRSAI_NANO_BANANA_SUBMIT_TIMEOUT_MS = 30000; const GRSAI_NANO_BANANA_POLL_TIMEOUT_MS = 30000; const GRSAI_NANO_BANANA_POLL_INTERVAL_MS = 5000; const GRSAI_GPT_IMAGE_TIMEOUT_MS = 10 * 60 * 1000; const GRSAI_GPT_IMAGE_4K_TIMEOUT_MS = 10 * 60 * 1000; function isRecord(value: unknown): value is Record { return Boolean(value && typeof value === "object" && !Array.isArray(value)); } function normalizeGrsaiPayload(payload: any): any { if (isRecord(payload?.data)) { return payload.data; } return payload; } async function sendGrsaiGptImageRequest(params: { baseURL: string; apiKey: string; modelId: string; prompt: string; size: string; imageUrls: string[]; timeout: number; apiLogPrefix: string; }): Promise { const { baseURL, apiKey, modelId, prompt, size, imageUrls, timeout, apiLogPrefix } = params; const endpoint = `${baseURL.replace(/\/+$/, "")}/v1/draw/completions`; const requestBody: Record = { model: modelId, prompt, size, shutProgress: false, }; if (imageUrls.length > 0) { requestBody.urls = imageUrls; } const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeout); try { logger.info(`${apiLogPrefix} Native submit`, { endpoint, modelId, size, referenceImageCount: imageUrls.length, }); const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify(requestBody), signal: controller.signal, }); clearTimeout(timeoutId); const responseText = await response.text(); const data = parseGrsaiJsonOrStreamResponse(responseText); if (!response.ok) { logger.error( `${apiLogPrefix} Native request failed:`, response.status, responseText.substring(0, 1000) ); const providerError = getGrsaiErrorMessage(data); const errorMessage = providerError !== "未知错误" ? providerError : responseText || `${response.status} ${response.statusText}`; throw new Error(`图片生成失败: ${errorMessage}`); } logger.info(`${apiLogPrefix} Native response received`, { status: response.status, hasImages: hasGrsaiImages(data), progress: getGrsaiProgress(data), responseKeys: isRecord(data) ? Object.keys(data).slice(0, 12) : [], }); if (!hasGrsaiImages(data)) { const status = getGrsaiStatus(data); if (status === "failed" || status === "error") { throw new Error(`图片生成失败: ${getGrsaiErrorMessage(data)}`); } const taskId = getGrsaiTaskId(data); if (taskId) { logger.info(`${apiLogPrefix} Native response has no images, polling task result`, { taskId, progress: getGrsaiProgress(data), status: status || null, }); return pollGrsaiDrawResult({ baseURL, apiKey, taskId, timeoutMs: timeout, apiLogPrefix, timeoutLabel: "Grsai GPT Image", }); } const providerError = getGrsaiErrorMessage(data); throw new Error( `图片生成失败: 未返回任务 ID 或可用图片${ providerError !== "未知错误" ? `: ${providerError}` : "" }` ); } return data; } catch (error) { clearTimeout(timeoutId); if (error instanceof Error && error.name === "AbortError") { const timeoutSeconds = Math.round(timeout / 1000); logger.error(`${apiLogPrefix} Request timeout after ${timeoutSeconds} seconds`); throw new Error(`Grsai GPT Image 请求超时(${timeoutSeconds}秒),将自动切换备用模型`); } logger.error(`${apiLogPrefix} Native request error:`, { name: error instanceof Error ? error.name : "UnknownError", message: error instanceof Error ? error.message : String(error), cause: error instanceof Error && "cause" in error ? error.cause instanceof Error ? error.cause.message : String(error.cause) : undefined, endpoint, modelId, size, referenceImageCount: imageUrls.length, }); throw error; } } function parseGrsaiJsonOrStreamResponse(responseText: string): any { if (!responseText.trim()) return {}; try { return JSON.parse(responseText); } catch { // Grsai draw/completions may stream progress as SSE-like "data: {...}" lines. const payloads = responseText .split(/\r?\n/) .map((line) => line.trim()) .filter((line) => line.startsWith("data:")) .map((line) => line.replace(/^data:\s*/, "").trim()) .filter((line) => line && line !== "[DONE]") .map((line) => { try { return JSON.parse(line); } catch { return null; } }) .filter(Boolean); if (payloads.length > 0) { return payloads[payloads.length - 1]; } } return { raw: responseText }; } function getGrsaiTaskId(payload: any): string | null { const normalized = normalizeGrsaiPayload(payload); const candidates = [ normalized?.id, normalized?.taskId, normalized?.task_id, normalized?.task?.id, normalized?.data?.id, payload?.id, payload?.taskId, payload?.task_id, ]; for (const candidate of candidates) { const value = String(candidate || "").trim(); if (value) return value; } return null; } function getGrsaiStatus(payload: any): string { const normalized = normalizeGrsaiPayload(payload); return String( normalized?.status || normalized?.state || normalized?.taskStatus || payload?.status || "" ).toLowerCase(); } function getGrsaiProgress(payload: any): number | null { const normalized = normalizeGrsaiPayload(payload); const value = normalized?.progress ?? normalized?.percentage ?? normalized?.percent ?? payload?.progress; const numeric = Number(value); return Number.isFinite(numeric) ? numeric : null; } function getGrsaiErrorMessage(payload: any): string { const normalized = normalizeGrsaiPayload(payload); const normalizePart = (part: unknown) => { if (typeof part === "string" || typeof part === "number") { return String(part).trim(); } if (!isRecord(part)) return ""; return String( part.message || part.msg || part.detail || part.reason || part.code || part.type || "" ).trim(); }; const messages = [ normalized?.failure_reason, normalized?.error?.message, normalized?.error?.msg, normalized?.error?.code, normalized?.error, normalized?.message, normalized?.msg, normalized?.detail, normalized?.reason, normalized?.code, payload?.error?.message, payload?.error?.msg, payload?.error?.code, payload?.msg, payload?.message, payload?.error, payload?.detail, payload?.reason, payload?.code, ] .map(normalizePart) .filter(Boolean); return Array.from(new Set(messages)).join(" / ") || "未知错误"; } function hasGrsaiImages(payload: any): boolean { const normalized = normalizeGrsaiPayload(payload); return ( Array.isArray(normalized?.results) || Array.isArray(normalized?.images) || Boolean(normalized?.url || normalized?.imageUrl || normalized?.image_url) ); } async function fetchWithTimeout( url: string, init: RequestInit, timeoutMs: number ): Promise { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeoutMs); try { return await fetch(url, { ...init, signal: controller.signal, }); } finally { clearTimeout(timeoutId); } } async function readJsonResponse(response: Response, label: string): Promise { const text = await response.text(); const payload = parseGrsaiJsonOrStreamResponse(text); if (!response.ok) { logger.error(`${label} failed:`, response.status, text.substring(0, 500)); const providerError = getGrsaiErrorMessage(payload); const detail = providerError !== "未知错误" ? providerError : String(payload?.raw || text || "").trim().slice(0, 300); throw new Error( `${label} 接口错误:${response.status} ${response.statusText}${ detail ? `: ${detail}` : "" }` ); } if (isRecord(payload) && !("raw" in payload)) { return payload; } if (!isRecord(payload)) { return payload; } if ("raw" in payload) { logger.error(`${label} returned non-JSON response:`, text.substring(0, 500)); throw new Error(`${label} 返回格式异常,未能解析任务结果`); } return payload; } async function pollGrsaiNanoBananaResult(params: { baseURL: string; apiKey: string; taskId: string; timeoutMs: number; apiLogPrefix: string; }): Promise { return pollGrsaiDrawResult({ ...params, timeoutLabel: "GRSAI Nano Banana", }); } async function pollGrsaiDrawResult(params: { baseURL: string; apiKey: string; taskId: string; timeoutMs: number; apiLogPrefix: string; timeoutLabel: string; }): Promise { const resultUrl = `${params.baseURL.replace(/\/+$/, "")}/v1/draw/result`; const deadline = Date.now() + params.timeoutMs; let lastPayload: any = null; let lastProgress: number | null = null; while (Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, GRSAI_NANO_BANANA_POLL_INTERVAL_MS)); const response = await fetchWithTimeout( resultUrl, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.apiKey}`, }, body: JSON.stringify({ id: params.taskId }), }, GRSAI_NANO_BANANA_POLL_TIMEOUT_MS ); const payload = await readJsonResponse(response, `${params.apiLogPrefix} result`); lastPayload = payload; const normalized = normalizeGrsaiPayload(payload); const status = getGrsaiStatus(payload); const progress = getGrsaiProgress(payload); if (progress !== null && progress !== lastProgress) { lastProgress = progress; logger.debug(`${params.apiLogPrefix} Async task progress: ${progress}%`); } if ( status === "succeeded" || status === "success" || status === "completed" || (progress !== null && progress >= 100 && hasGrsaiImages(payload)) ) { logger.debug(`${params.apiLogPrefix} Async task succeeded:`, { taskId: params.taskId, progress, }); return normalized; } if (status === "failed" || status === "error") { throw new Error(`图片生成失败: ${getGrsaiErrorMessage(payload)}`); } } logger.error(`${params.apiLogPrefix} Async task timeout:`, { taskId: params.taskId, lastPayload: JSON.stringify(lastPayload || {}).substring(0, 500), }); throw new Error( `${params.timeoutLabel} 异步任务超时(${Math.round(params.timeoutMs / 1000)}秒),请稍后重试或切换备用模型` ); } async function submitAndPollGrsaiNanoBanana(params: { baseURL: string; apiKey: string; requestBody: Record; imageParams?: { aspectRatio?: string; imageSize?: string; }; apiLogPrefix: string; }): Promise { const submitUrl = `${params.baseURL.replace(/\/+$/, "")}/v1/draw/nano-banana`; const body: Record = { ...params.requestBody, webHook: "-1", }; const imageSize = String(params.imageParams?.imageSize || "").toUpperCase(); const pollTimeoutMs = 600000; logger.debug(`${params.apiLogPrefix} Submitting async Nano Banana task:`, { model: body.model, imageSize: params.imageParams?.imageSize, aspectRatio: params.imageParams?.aspectRatio, }); const response = await fetchWithTimeout( submitUrl, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.apiKey}`, }, body: JSON.stringify(body), }, GRSAI_NANO_BANANA_SUBMIT_TIMEOUT_MS ); const submitPayload = await readJsonResponse(response, `${params.apiLogPrefix} submit`); const submitStatus = getGrsaiStatus(submitPayload); const normalizedSubmitPayload = normalizeGrsaiPayload(submitPayload); if ( submitStatus === "succeeded" || submitStatus === "success" || submitStatus === "completed" || hasGrsaiImages(submitPayload) ) { return normalizedSubmitPayload; } if (submitStatus === "failed" || submitStatus === "error") { throw new Error(`图片生成失败: ${getGrsaiErrorMessage(submitPayload)}`); } const taskId = getGrsaiTaskId(submitPayload); if (!taskId) { const providerError = getGrsaiErrorMessage(submitPayload); logger.error( `${params.apiLogPrefix} Async submit response has no task id:`, JSON.stringify(submitPayload).substring(0, 500) ); throw new Error( `GRSAI Nano Banana 异步任务提交失败:未返回任务 ID${ providerError !== "未知错误" ? `: ${providerError}` : "" }` ); } logger.debug(`${params.apiLogPrefix} Async task submitted:`, { taskId }); return pollGrsaiNanoBananaResult({ baseURL: params.baseURL, apiKey: params.apiKey, taskId, timeoutMs: pollTimeoutMs, apiLogPrefix: params.apiLogPrefix, }); } /** * 发送 Gemini 图像生成/编辑请求 * 使用独立的请求格式,不依赖 OpenAI SDK * 支持发送完整的消息数组(包括文本和图片) */ async function sendGeminiRequest( baseURL: string, apiKey: string, messages: any[], modelId: string, imageParams?: { aspectRatio?: string; imageSize?: string; } ): Promise { const normalizeAspectRatio = (aspectRatio: string) => { const normalized = String(aspectRatio || "").trim(); if (!normalized || normalized.toLowerCase() === "auto") { return undefined; } if (!/^\d+\s*:\s*\d+$/i.test(normalized)) { return undefined; } const [widthRaw, heightRaw] = normalized.split(":").map((item) => item.trim()); return `${Number(widthRaw)}:${Number(heightRaw)}`; }; const isGptImageModel = /^gpt-image/i.test(modelId); const shouldUseGrsaiAsyncPolling = !isGptImageModel && /nano-banana/i.test(modelId) && /(?:grsai|dakka)/i.test(baseURL); const apiLogPrefix = isGptImageModel ? "[Grsai GPT Image]" : "[Gemini Image]"; // GPT Image 和 Nano Banana 在 Grsai 上是两套接口,不能混用同一个 endpoint。 const url = isGptImageModel ? `${baseURL}/v1/draw/completions` : `${baseURL}/v1/draw/nano-banana`; // 提取最后一条消息的文本作为prompt const lastMessage = messages[messages.length - 1]; const prompt = typeof lastMessage.content === 'string' ? lastMessage.content : lastMessage.content?.find((c: any) => c.type === 'text')?.text || '生成图片'; // 提取所有参考图 (urls) const urls: string[] = []; // 遍历所有消息提取图片(通常用户会发送多张参考图) messages.forEach((msg) => { if (msg.role === "user" && Array.isArray(msg.content)) { msg.content.forEach((contentPart: any) => { const candidateImageUrl = contentPart.type === "image_url" ? contentPart.image_url?.url : contentPart.type === "file" ? contentPart.url || contentPart.data : contentPart.type === "image" ? contentPart.image || contentPart.url : undefined; if (!candidateImageUrl || typeof candidateImageUrl !== "string") { return; } urls.push(candidateImageUrl); }); } }); // 🆕 处理参考图:上传 Base64 到 S3 const processedUrls: string[] = []; if (urls.length > 0) { logger.debug(`${apiLogPrefix} Processing ${urls.length} reference images...`); for (const url of urls) { const s3Url = await ensureImageUrl(url); processedUrls.push(s3Url); } } const promptForRequest = buildReferenceImageOrderPrompt(prompt, processedUrls); logger.debug(`${apiLogPrefix} Sending request to:`, url); logger.debug(`${apiLogPrefix} Using model:`, modelId); logger.debug(`${apiLogPrefix} Prompt:`, promptForRequest.substring(0, 100)); logger.debug(`${apiLogPrefix} Reference image count:`, urls.length); logger.debug(`${apiLogPrefix} Image params:`, imageParams); // 使用对应模型的原生格式 const requestBody: any = { model: modelId, prompt: promptForRequest, }; if (isGptImageModel) { const requestedAspectRatio = imageParams?.aspectRatio || "auto"; const inferredAspectRatio = requestedAspectRatio === "auto" && processedUrls.length > 0 ? await inferImageAspectRatioFromUrl(processedUrls[0]) : null; const resolvedAspectRatio = inferredAspectRatio || requestedAspectRatio; const resolvedSize = resolveGrsaiGptImageSize(resolvedAspectRatio, imageParams?.imageSize); // Grsai GPT-image-2-vip 只接受固定像素 size;继续传比例或档位字段会触发 // “does not support 1:1/3:4 parameter, please use pixel value input”。 requestBody.size = resolvedSize; requestBody.shutProgress = false; if (processedUrls.length > 0) { requestBody.image = processedUrls[0]; requestBody.imageUrls = processedUrls; } logger.debug( `${apiLogPrefix} GPT image payload mapping: requestedAspectRatio=${requestedAspectRatio}, resolvedAspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"}, resolvedSize=${resolvedSize}` ); if (inferredAspectRatio) { logger.debug(`${apiLogPrefix} Inferred reference aspect ratio: ${inferredAspectRatio}`); } logger.debug( `${apiLogPrefix} Resolved size: aspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"} => ${resolvedSize}` ); const timeout = imageParams?.imageSize === "4K" ? GRSAI_GPT_IMAGE_4K_TIMEOUT_MS : GRSAI_GPT_IMAGE_TIMEOUT_MS; logger.debug(`${apiLogPrefix} Timeout set to ${timeout / 1000} seconds for ${imageParams?.imageSize || "default"}`); return sendGrsaiGptImageRequest({ baseURL, apiKey, modelId, prompt: promptForRequest, size: resolvedSize, imageUrls: processedUrls, timeout, apiLogPrefix, }); } else { // 添加 Nano Banana 特定参数 if (processedUrls.length > 0) { requestBody.urls = processedUrls; } if (imageParams?.aspectRatio) { requestBody.aspectRatio = imageParams.aspectRatio; } if (imageParams?.imageSize) { requestBody.imageSize = imageParams.imageSize; } } logger.debug(`${apiLogPrefix} Request body (truncated):`, JSON.stringify(requestBody).substring(0, 500) + "..."); if (shouldUseGrsaiAsyncPolling) { return submitAndPollGrsaiNanoBanana({ baseURL, apiKey, requestBody, imageParams, apiLogPrefix, }); } // 根据分辨率动态设置超时时间 const timeout = 600000; // 所有生图模型最长等待 10 分钟 logger.debug(`${apiLogPrefix} Timeout set to ${timeout/1000} seconds for ${imageParams?.imageSize || "default"}`); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeout); try { const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify(requestBody), signal: controller.signal, }); clearTimeout(timeoutId); if (!response.ok) { const errorText = await response.text(); logger.error(`${apiLogPrefix} Request failed:`, response.status, errorText); throw new Error( `${isGptImageModel ? "Grsai GPT Image" : "Gemini"} API error: ${response.status} ${response.statusText}` ); } // 处理流式响应(SSE格式) logger.debug(`${apiLogPrefix} Reading stream response...`); const reader = response.body?.getReader(); const decoder = new TextDecoder(); let buffer = ""; let lastValidData: any = null; if (!reader) { throw new Error("Response body is null"); } while (true) { const { done, value } = await reader.read(); if (done) { logger.debug("[Gemini Image] Stream completed"); break; } // 将新数据添加到缓冲区 buffer += decoder.decode(value, { stream: true }); // 按行分割 const lines = buffer.split("\n"); buffer = lines.pop() || ""; // 保留最后一个不完整的行 for (const line of lines) { const trimmedLine = line.trim(); // 解析 SSE 格式: "data: {...}" if (trimmedLine.startsWith("data: ")) { try { const jsonStr = trimmedLine.substring(6); // 移除 "data: " 前缀 const data = JSON.parse(jsonStr); // 保存最新的有效数据 lastValidData = data; // 记录进度 if (data.progress !== undefined) { logger.debug(`[Gemini Image] Progress: ${data.progress}%`); } // 如果成功完成,立即返回 if (data.status === "succeeded") { logger.debug("[Gemini Image] Generation succeeded!"); logger.debug("[Gemini Image] Full success data:", JSON.stringify(data).substring(0, 1000)); // 🔍 打印完整成功数据 return data as GeminiResponse; } // 如果失败,抛出错误(不被catch捕获) if (data.status === "failed") { const failureReason = String(data.failure_reason || "").trim(); const providerError = String(data.error || "").trim(); const detailParts = [failureReason, providerError].filter( (part, index, arr) => part && arr.findIndex((candidate) => candidate === part) === index ); const errorMsg = `图片生成失败: ${detailParts.join(" / ") || "未知错误"}`; logger.error("[Gemini Image] Generation failed:", data.failure_reason, data.error); clearTimeout(timeoutId); throw new Error(errorMsg); } } catch (e) { // 只捕获JSON解析错误,重新抛出其他错误 if (e instanceof Error && e.message.includes("图片生成失败")) { throw e; // 重新抛出失败错误 } logger.warn("[Gemini Image] Failed to parse SSE line:", trimmedLine); } } } } // 流结束后,返回最后一个有效数据 if (lastValidData) { logger.debug("[Gemini Image] Returning last valid data"); return lastValidData as GeminiResponse; } throw new Error("No valid data received from stream"); } catch (error) { clearTimeout(timeoutId); if (error instanceof Error && error.name === "AbortError") { const timeoutSeconds = Math.round(timeout / 1000); logger.error(`${apiLogPrefix} Request timeout after ${timeoutSeconds} seconds`); throw new Error( `${isGptImageModel ? "Grsai GPT Image" : "Gemini"} 请求超时(${timeoutSeconds}秒),将自动切换备用模型` ); } throw error; } } /** * 创建 Gemini 图像生成 Provider * 使用与 local-qwen.ts 相同的格式 */ export function createGeminiImage({ apiKey, baseURL }: GeminiImageConfig) { return { chat: (modelId: string) => ({ doGenerate: async (options: any) => { logger.debug("[Gemini Image] doGenerate called!"); // 调试日志 logger.debug("[Gemini Image] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "..."); // 🆕 支持覆盖 modelId(用于备用模型切换) const effectiveModelId = options.overrideModelId || modelId; logger.debug("[Gemini Image] Using modelId:", effectiveModelId, options.overrideModelId ? "(overridden)" : "(default)"); logger.debug("[Gemini Image] Image params from options:", options.imageParams); try { // 直接传递完整的消息数组(包括文本和图片) // 确保imageSize有值,优先级:用户选择 > 默认2K const imageParams = { aspectRatio: options.imageParams?.aspectRatio || "auto", imageSize: options.imageParams?.imageSize || "2K" }; // 🆕 获取用户选择的图片数量,默认 1 张 const imageCount = Math.min(Math.max(options.imageParams?.imageCount || 1, 1), 2); // Gemini 限制 1-2 张 logger.debug("[Gemini Image] Final imageParams after defaults:", imageParams); logger.debug(`[Gemini Image] Image count: ${imageCount}`); // 🆕 根据用户选择的数量发起请求 logger.debug(`[Gemini Image] Starting ${imageCount} concurrent request(s)...`); const startTime = Date.now(); // 创建请求数组(使用 effectiveModelId) const requestPromises = Array.from({ length: imageCount }, () => sendGeminiRequest(baseURL, apiKey, options.messages, effectiveModelId, imageParams) ); const responses = await Promise.all(requestPromises); logger.debug(`[Gemini Image] All ${imageCount} request(s) completed in ${Date.now() - startTime}ms`); // 提取图片并上传 base64 到 S3(并行处理) const extractPromises = responses.map(response => extractAndUploadImages(response)); const imageUrlsArrays = await Promise.all(extractPromises); // 合并所有请求的图片结果 const allImageUrls = imageUrlsArrays.flat(); const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize); logger.debug( `[Gemini Image] Generated ${allImageUrls.length} images from ${imageCount} concurrent request(s) (uploaded to S3)` ); // 返回 AI SDK v1 格式 return { text: markdownText, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: markdownText.length, }, }; } catch (error) { logger.error("[Gemini Image] Generation failed:", error); // ❌ 抛出错误以触发故障转移(不要吞掉错误) throw error; } }, doStream: async function* (options: any) { logger.debug("[Gemini Image] ===== doStream ENTERED! ====="); // 调试日志 logger.debug("[Gemini Image] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "..."); // 🆕 支持覆盖 modelId(用于备用模型切换) const effectiveModelId = options.overrideModelId || modelId; logger.debug("[Gemini Image] Using modelId:", effectiveModelId, options.overrideModelId ? "(overridden)" : "(default)"); logger.debug("[Gemini Image] Image params from options:", options.imageParams); try { // 直接传递完整的消息数组(包括文本和图片) // 确保imageSize有值,优先级:用户选择 > 默认2K const imageParams = { aspectRatio: options.imageParams?.aspectRatio || "auto", imageSize: options.imageParams?.imageSize || "2K" }; // 🆕 获取用户选择的图片数量,默认 1 张 const imageCount = Math.min(Math.max(options.imageParams?.imageCount || 1, 1), 2); // Gemini 限制 1-2 张 logger.debug("[Gemini Image] Final imageParams after defaults:", imageParams); logger.debug(`[Gemini Image Stream] Image count: ${imageCount}`); // 🆕 根据用户选择的数量发起请求 logger.debug(`[Gemini Image Stream] Starting ${imageCount} concurrent request(s)...`); const startTime = Date.now(); // 创建请求数组(使用 effectiveModelId) const requestPromises = Array.from({ length: imageCount }, () => sendGeminiRequest(baseURL, apiKey, options.messages, effectiveModelId, imageParams) ); const responses = await Promise.all(requestPromises); logger.debug(`[Gemini Image Stream] All ${imageCount} request(s) completed in ${Date.now() - startTime}ms`); // 提取图片并上传 base64 到 S3(并行处理) const extractPromises = responses.map(response => extractAndUploadImages(response)); const imageUrlsArrays = await Promise.all(extractPromises); // 合并所有请求的图片结果 const allImageUrls = imageUrlsArrays.flat(); const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize); logger.debug( `[Gemini Image Stream] Generated ${allImageUrls.length} images from ${imageCount} concurrent request(s) (uploaded to S3)` ); // 一次性发送整个图片 markdown(不再分块,因为 base64 图片很大) yield { type: "text-delta" as const, textDelta: markdownText }; logger.debug("[Gemini Image Stream] Sent complete markdown"); // 发送完成信号 yield { type: "finish" as const, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: markdownText.length, }, }; } catch (error) { logger.error("[Gemini Image Stream] Generation failed:", error); const errorMessage = error instanceof Error ? `图片生成失败: ${error.message}` : "图片生成失败,请稍后重试"; // 发送错误信息 yield { type: "text-delta" as const, textDelta: errorMessage }; yield { type: "finish" as const, finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } }, modelId, provider: "gemini-image", specificationVersion: "v1" as const, defaultObjectGenerationMode: "json" as const, }), }; }