831 lines
33 KiB
TypeScript
831 lines
33 KiB
TypeScript
import "server-only";
|
||
|
||
import { resolveGrsaiGptImageSize } from "@/lib/ai/grsai-gpt-image-sizes";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import { createLogger } from "@/lib/observability/logger";
|
||
|
||
const logger = createLogger("gemini-image");
|
||
interface GeminiImageConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
/**
|
||
* 压缩图片生成缩略图（使用 sharp）
|
||
*/
|
||
async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer> {
|
||
try {
|
||
const sharp = await import("sharp");
|
||
|
||
// 创建缩略图（宽度 500px，高度按比例，质量 80%）
|
||
const thumbnailBuffer = await sharp.default(buffer)
|
||
.resize(500, null, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
|
||
logger.debug(`[Gemini Image] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`);
|
||
return thumbnailBuffer;
|
||
} catch (error) {
|
||
logger.error("[Gemini Image] Failed to create thumbnail:", error);
|
||
// 如果失败，返回原图
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 确保图片是公网可访问的 URL（如果是 Base64，上传到 S3）
|
||
*/
|
||
async function ensureImageUrl(url: string): Promise<string> {
|
||
if (url.startsWith("http")) return url;
|
||
|
||
if (url.startsWith("data:")) {
|
||
try {
|
||
logger.debug("[Gemini Image] Converting Base64 reference image to S3 URL...");
|
||
// 解析 Base64
|
||
const matches = url.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/);
|
||
if (!matches) {
|
||
logger.warn("[Gemini Image] Invalid Base64 format, skipping upload");
|
||
return url;
|
||
}
|
||
|
||
const mimeType = matches[1];
|
||
const base64Data = matches[2];
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
const ext = mimeType.split("/")[1] || "png";
|
||
const filename = `ref-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||
|
||
// 上传到 S3（需要转换为 ArrayBuffer）
|
||
const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer;
|
||
const result = await uploadFile(filename, arrayBuffer);
|
||
logger.debug(`[Gemini Image] Uploaded reference image to: ${result.url}`);
|
||
return result.url;
|
||
} catch (error) {
|
||
logger.error("[Gemini Image] Failed to upload reference image to S3:", error);
|
||
// 如果上传失败，返回原 Base64（虽然可能也会失败，但没别的办法）
|
||
return url;
|
||
}
|
||
}
|
||
|
||
return url;
|
||
}
|
||
|
||
async function inferImageAspectRatioFromUrl(url: string): Promise<string | null> {
|
||
const normalized = String(url || "").trim();
|
||
if (!normalized) return null;
|
||
|
||
try {
|
||
let buffer: Buffer | null = null;
|
||
if (normalized.startsWith("data:")) {
|
||
const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||
if (!match) return null;
|
||
buffer = Buffer.from(match[2], "base64");
|
||
} else if (/^https?:\/\//i.test(normalized)) {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 20000);
|
||
try {
|
||
const response = await fetch(normalized, {
|
||
signal: controller.signal,
|
||
headers: {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||
},
|
||
});
|
||
if (!response.ok) return null;
|
||
buffer = Buffer.from(await response.arrayBuffer());
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
if (!buffer?.length) return null;
|
||
const sharp = await import("sharp");
|
||
const metadata = await sharp.default(buffer).metadata();
|
||
if (!metadata.width || !metadata.height || metadata.height <= 0) {
|
||
return null;
|
||
}
|
||
|
||
return `${metadata.width}:${metadata.height}`;
|
||
} catch (error) {
|
||
logger.warn("[Gemini Image] Failed to infer reference aspect ratio:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 将 base64 图片上传到 S3，返回缩略图和原图的 URL
|
||
*/
|
||
async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jpeg"): Promise<{ thumbnail: string; original: string }> {
|
||
try {
|
||
// 解析 base64 数据
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
|
||
// 生成文件名
|
||
const ext = mimeType.split("/")[1] || "jpg";
|
||
const timestamp = Date.now();
|
||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||
const thumbnailFilename = `gemini-thumb-${timestamp}-${randomStr}.${ext}`;
|
||
const originalFilename = `gemini-orig-${timestamp}-${randomStr}.${ext}`;
|
||
|
||
logger.debug(`[Gemini Image] Original size: ${buffer.length} bytes`);
|
||
|
||
// 生成缩略图
|
||
const thumbnailBuffer = await createThumbnail(buffer, mimeType);
|
||
|
||
// 转换为 ArrayBuffer
|
||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||
thumbnailBuffer.byteOffset,
|
||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||
) as ArrayBuffer;
|
||
const originalArrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
|
||
// 并行上传缩略图和原图到 S3
|
||
logger.debug(`[Gemini Image] Uploading thumbnail and original to S3...`);
|
||
const [thumbnailResult, originalResult] = await Promise.all([
|
||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||
uploadFile(originalFilename, originalArrayBuffer),
|
||
]);
|
||
|
||
logger.debug(`[Gemini Image] Uploaded to S3:`);
|
||
logger.debug(` - Thumbnail: ${thumbnailResult.url}`);
|
||
logger.debug(` - Original: ${originalResult.url}`);
|
||
|
||
return {
|
||
thumbnail: thumbnailResult.url,
|
||
original: originalResult.url,
|
||
};
|
||
} catch (error) {
|
||
logger.error("[Gemini Image] Failed to upload to S3:", error);
|
||
// 如果上传失败，抛出错误，不要返回 Base64，否则会导致前端崩溃和数据库保存失败
|
||
throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 下载远程URL图片并上传到 S3，返回缩略图和原图的 URL
|
||
* 带重试机制，确保成功下载并上传到国内中转
|
||
*/
|
||
async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> {
|
||
for (let attempt = 0; attempt < retries; attempt++) {
|
||
try {
|
||
logger.debug(`[Gemini Image] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`);
|
||
|
||
// 下载图片（增加超时时间到 30 秒）
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 30000);
|
||
|
||
const response = await fetch(imageUrl, {
|
||
signal: controller.signal,
|
||
headers: {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||
},
|
||
});
|
||
|
||
clearTimeout(timeoutId);
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||
}
|
||
|
||
// 获取图片数据
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
|
||
if (buffer.length === 0) {
|
||
throw new Error("下载的图片数据为空");
|
||
}
|
||
|
||
// 从响应头或URL获取MIME类型
|
||
const contentType = response.headers.get("content-type") || "image/jpeg";
|
||
const ext = contentType.split("/")[1] || "jpg";
|
||
|
||
// 生成文件名
|
||
const timestamp = Date.now();
|
||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||
const thumbnailFilename = `gemini-thumb-${timestamp}-${randomStr}.${ext}`;
|
||
const originalFilename = `gemini-orig-${timestamp}-${randomStr}.${ext}`;
|
||
|
||
logger.debug(`[Gemini Image] Downloaded size: ${buffer.length} bytes`);
|
||
|
||
// 生成缩略图
|
||
const thumbnailBuffer = await createThumbnail(buffer, contentType);
|
||
|
||
// 转换为 ArrayBuffer
|
||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||
thumbnailBuffer.byteOffset,
|
||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||
) as ArrayBuffer;
|
||
const originalArrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
|
||
// 并行上传缩略图和原图到 S3
|
||
logger.debug(`[Gemini Image] Uploading downloaded image to S3...`);
|
||
const [thumbnailResult, originalResult] = await Promise.all([
|
||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||
uploadFile(originalFilename, originalArrayBuffer),
|
||
]);
|
||
|
||
logger.debug(`[Gemini Image] Uploaded to S3:`);
|
||
logger.debug(` - Thumbnail: ${thumbnailResult.url}`);
|
||
logger.debug(` - Original: ${originalResult.url}`);
|
||
|
||
return {
|
||
thumbnail: thumbnailResult.url,
|
||
original: originalResult.url,
|
||
};
|
||
} catch (error) {
|
||
const isLastAttempt = attempt === retries - 1;
|
||
logger.error(
|
||
`[Gemini Image] Download attempt ${attempt + 1}/${retries} failed:`,
|
||
error instanceof Error ? error.message : String(error)
|
||
);
|
||
|
||
if (isLastAttempt) {
|
||
// 最后一次尝试失败，抛出错误（不要 fallback 到原始 URL）
|
||
throw new Error(
|
||
`图片下载并上传失败（已重试 ${retries} 次）: ${error instanceof Error ? error.message : "未知错误"}`
|
||
);
|
||
}
|
||
|
||
// 等待后重试（指数退避）
|
||
const delay = Math.pow(2, attempt) * 1000; // 1s, 2s, 4s...
|
||
logger.debug(`[Gemini Image] Retrying in ${delay}ms...`);
|
||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||
}
|
||
}
|
||
|
||
// 理论上不会到达这里，但 TypeScript 需要
|
||
throw new Error("下载失败：未知错误");
|
||
}
|
||
|
||
// Gemini API 响应格式（参考灵魂助手3.0的格式）
|
||
interface GeminiResponse {
|
||
candidates: Array<{
|
||
content: {
|
||
parts: Array<{
|
||
text?: string;
|
||
inline_data?: {
|
||
mime_type: string;
|
||
data: string;
|
||
};
|
||
inlineData?: {
|
||
mimeType: string;
|
||
data: string;
|
||
};
|
||
}>;
|
||
};
|
||
}>;
|
||
}
|
||
|
||
/**
|
||
* 从文本中提取图片数据
|
||
* 返回 { url, isBase64, mimeType, base64Data } 数组
|
||
*/
|
||
function extractImageDataFromText(text: string): Array<{ url: string; isBase64: boolean; mimeType?: string; base64Data?: string }> {
|
||
const images: Array<{ url: string; isBase64: boolean; mimeType?: string; base64Data?: string }> = [];
|
||
|
||
// 匹配所有 Markdown 图片格式: 
|
||
const allImageMatches = text.matchAll(/!\[[^\]]*\]\(([^)]+)\)/g);
|
||
for (const match of allImageMatches) {
|
||
const url = match[1];
|
||
// 检查是否是 base64 格式
|
||
const base64Match = url.match(/^data:image\/([^;]+);base64,(.+)$/);
|
||
if (base64Match) {
|
||
images.push({
|
||
url,
|
||
isBase64: true,
|
||
mimeType: `image/${base64Match[1]}`,
|
||
base64Data: base64Match[2],
|
||
});
|
||
} else {
|
||
images.push({ url, isBase64: false });
|
||
}
|
||
}
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* 从 API 响应中提取图片并上传 base64 到 S3
|
||
* 返回图片数据数组（包含缩略图和原图 URL）
|
||
*/
|
||
async function extractAndUploadImages(response: any): Promise<Array<{ thumbnail: string; original: string }>> {
|
||
logger.debug("[Gemini Image] Extracting images from response:", JSON.stringify(response).substring(0, 500)); // 🔍 调试日志
|
||
const images: Array<{ thumbnail: string; original: string }> = [];
|
||
const uploadPromises: Promise<void>[] = [];
|
||
|
||
// 1. 尝试 OpenAI 格式 (choices[0].message.content)
|
||
if (response.choices && response.choices.length > 0) {
|
||
const content = response.choices[0]?.message?.content;
|
||
if (content && typeof content === "string") {
|
||
logger.debug("[Gemini Image] Parsing OpenAI format, content:", content.substring(0, 100) + "...");
|
||
const imageData = extractImageDataFromText(content);
|
||
|
||
for (const img of imageData) {
|
||
if (img.isBase64 && img.base64Data) {
|
||
// 异步上传 base64 到 S3（生成缩略图和原图）
|
||
const uploadPromise = uploadBase64ToS3(img.base64Data, img.mimeType)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
} else {
|
||
// 非 base64 图片（OpenAI URL 等），必须下载并上传到 S3（国内中转）
|
||
logger.debug("[Gemini Image] Detected external URL, will download and upload to S3:", img.url.substring(0, 100) + "...");
|
||
const uploadPromise = downloadAndUploadToS3(img.url)
|
||
.then(urls => {
|
||
logger.debug("[Gemini Image] Successfully downloaded and uploaded external image to S3");
|
||
images.push(urls);
|
||
})
|
||
.catch(error => {
|
||
logger.error("[Gemini Image] Failed to download/upload URL after retries:", error);
|
||
// 不再 fallback 到原始 URL，而是抛出错误
|
||
throw new Error(`无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}`);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 2. 尝试 Nano Banana API 格式 (results[].url)
|
||
if (response.results && Array.isArray(response.results)) {
|
||
logger.debug("[Gemini Image] Parsing Nano Banana format, results count:", response.results.length);
|
||
for (const result of response.results) {
|
||
if (result.url && typeof result.url === "string") {
|
||
logger.debug("[Gemini Image] Found Nano Banana image URL:", result.url.substring(0, 100) + "...");
|
||
// Nano Banana 返回的是外部URL，必须下载并上传到 S3（国内中转）
|
||
// 如果下载失败，抛出错误，不要使用原始URL（官方地址可能无法访问）
|
||
const uploadPromise = downloadAndUploadToS3(result.url)
|
||
.then(urls => {
|
||
logger.debug("[Gemini Image] Successfully downloaded and uploaded Nano Banana image to S3");
|
||
images.push(urls);
|
||
})
|
||
.catch(error => {
|
||
logger.error("[Gemini Image] Failed to download/upload Nano Banana URL after retries:", error);
|
||
// 不再 fallback 到原始 URL，而是抛出错误
|
||
throw new Error(`无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}`);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 3. 尝试 Gemini 格式 (candidates[0].content.parts)
|
||
if (response.candidates && response.candidates.length > 0) {
|
||
const candidate = response.candidates[0];
|
||
if (candidate.content && candidate.content.parts) {
|
||
for (const part of candidate.content.parts) {
|
||
if (part.text) {
|
||
const imageData = extractImageDataFromText(part.text);
|
||
for (const img of imageData) {
|
||
if (img.isBase64 && img.base64Data) {
|
||
const uploadPromise = uploadBase64ToS3(img.base64Data, img.mimeType)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
} else {
|
||
// 非 base64 图片（OpenAI URL 等），必须下载并上传到 S3（国内中转）
|
||
logger.debug("[Gemini Image] Detected external URL in Gemini response, will download and upload to S3:", img.url.substring(0, 100) + "...");
|
||
const uploadPromise = downloadAndUploadToS3(img.url)
|
||
.then(urls => {
|
||
logger.debug("[Gemini Image] Successfully downloaded and uploaded Gemini external image to S3");
|
||
images.push(urls);
|
||
})
|
||
.catch(error => {
|
||
logger.error("[Gemini Image] Failed to download/upload URL in Gemini response after retries:", error);
|
||
// 不再 fallback 到原始 URL，而是抛出错误
|
||
throw new Error(`无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}`);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
}
|
||
}
|
||
// inline_data 格式
|
||
if (part.inline_data && part.inline_data.data) {
|
||
const mimeType = part.inline_data.mime_type || "image/png";
|
||
const uploadPromise = uploadBase64ToS3(part.inline_data.data, mimeType)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
} else if (part.inlineData && part.inlineData.data) {
|
||
const mimeType = part.inlineData.mimeType || "image/png";
|
||
const uploadPromise = uploadBase64ToS3(part.inlineData.data, mimeType)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 等待所有上传完成
|
||
await Promise.all(uploadPromises);
|
||
|
||
logger.debug("[Gemini Image] Extracted and uploaded", images.length, "images (thumbnail + original)");
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* 将图片数据转换为 Markdown 格式
|
||
* 使用特殊格式传递缩略图和原图 URL：
|
||
* 前端的 ImagePreview 组件会解析这个格式
|
||
*
|
||
* 注意：不能使用 $$ 作为分隔符，因为会被 Markdown 解析器当作 LaTeX 数学公式！
|
||
* 使用 ||| (三个竖线) 作为分隔符，不会与任何 Markdown 语法冲突
|
||
*/
|
||
function formatImageResponse(
|
||
images: Array<{ thumbnail: string; original: string }>,
|
||
imageSize: string = "2K"
|
||
): string {
|
||
if (images.length === 0) {
|
||
return "生成图片失败，未找到图片数据";
|
||
}
|
||
|
||
// 返回 Markdown 格式的图片，在 alt 文本中编码原图 URL 和 imageSize
|
||
// 使用 ||| 作为分隔符，避免与 Markdown 表格语法和 LaTeX 数学公式冲突
|
||
return images.map((img) => ``).join("\n\n");
|
||
}
|
||
|
||
/**
|
||
* 发送 Gemini 图像生成/编辑请求
|
||
* 使用独立的请求格式，不依赖 OpenAI SDK
|
||
* 支持发送完整的消息数组（包括文本和图片）
|
||
*/
|
||
async function sendGeminiRequest(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
messages: any[],
|
||
modelId: string,
|
||
imageParams?: {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
}
|
||
): Promise<GeminiResponse> {
|
||
const isGptImageModel = /^gpt-image/i.test(modelId);
|
||
const apiLogPrefix = isGptImageModel ? "[Grsai GPT Image]" : "[Gemini Image]";
|
||
// GPT Image 和 Nano Banana 在 Grsai 上是两套接口，不能混用同一个 endpoint。
|
||
const url = isGptImageModel
|
||
? `${baseURL}/v1/draw/completions`
|
||
: `${baseURL}/v1/draw/nano-banana`;
|
||
|
||
// 提取最后一条消息的文本作为prompt
|
||
const lastMessage = messages[messages.length - 1];
|
||
const prompt = typeof lastMessage.content === 'string'
|
||
? lastMessage.content
|
||
: lastMessage.content?.find((c: any) => c.type === 'text')?.text || '生成图片';
|
||
|
||
// 提取所有参考图 (urls)
|
||
const urls: string[] = [];
|
||
|
||
// 遍历所有消息提取图片（通常用户会发送多张参考图）
|
||
messages.forEach((msg) => {
|
||
if (msg.role === "user" && Array.isArray(msg.content)) {
|
||
msg.content.forEach((contentPart: any) => {
|
||
// 支持标准 image_url
|
||
if (contentPart.type === "image_url" && contentPart.image_url?.url) {
|
||
urls.push(contentPart.image_url.url);
|
||
}
|
||
// 🆕 支持 CoreMessage 的 image 类型 (可能来自 Attachment)
|
||
else if (contentPart.type === "image" && contentPart.image) {
|
||
if (typeof contentPart.image === 'string') {
|
||
urls.push(contentPart.image);
|
||
} else if (contentPart.image instanceof URL) {
|
||
urls.push(contentPart.image.toString());
|
||
}
|
||
}
|
||
});
|
||
}
|
||
});
|
||
|
||
// 🆕 处理参考图：上传 Base64 到 S3
|
||
const processedUrls: string[] = [];
|
||
if (urls.length > 0) {
|
||
logger.debug(`${apiLogPrefix} Processing ${urls.length} reference images...`);
|
||
for (const url of urls) {
|
||
const s3Url = await ensureImageUrl(url);
|
||
processedUrls.push(s3Url);
|
||
}
|
||
}
|
||
|
||
logger.debug(`${apiLogPrefix} Sending request to:`, url);
|
||
logger.debug(`${apiLogPrefix} Using model:`, modelId);
|
||
logger.debug(`${apiLogPrefix} Prompt:`, prompt.substring(0, 100));
|
||
logger.debug(`${apiLogPrefix} Reference image count:`, urls.length);
|
||
logger.debug(`${apiLogPrefix} Image params:`, imageParams);
|
||
|
||
// 使用对应模型的原生格式
|
||
const requestBody: any = {
|
||
model: modelId,
|
||
prompt: prompt,
|
||
};
|
||
|
||
// 添加参考图参数
|
||
if (processedUrls.length > 0) {
|
||
requestBody.urls = processedUrls;
|
||
}
|
||
|
||
if (isGptImageModel) {
|
||
const requestedAspectRatio = imageParams?.aspectRatio || "auto";
|
||
const inferredAspectRatio =
|
||
requestedAspectRatio === "auto" && processedUrls.length > 0
|
||
? await inferImageAspectRatioFromUrl(processedUrls[0])
|
||
: null;
|
||
const resolvedAspectRatio = inferredAspectRatio || requestedAspectRatio;
|
||
requestBody.size = resolveGrsaiGptImageSize(resolvedAspectRatio, imageParams?.imageSize);
|
||
requestBody.shutProgress = false;
|
||
if (inferredAspectRatio) {
|
||
logger.debug(`${apiLogPrefix} Inferred reference aspect ratio: ${inferredAspectRatio}`);
|
||
}
|
||
logger.debug(
|
||
`${apiLogPrefix} Resolved size: aspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"} => ${requestBody.size}`
|
||
);
|
||
} else {
|
||
// 添加 Nano Banana 特定参数
|
||
if (imageParams?.aspectRatio) {
|
||
requestBody.aspectRatio = imageParams.aspectRatio;
|
||
}
|
||
if (imageParams?.imageSize) {
|
||
requestBody.imageSize = imageParams.imageSize;
|
||
}
|
||
}
|
||
|
||
logger.debug(`${apiLogPrefix} Request body (truncated):`, JSON.stringify(requestBody).substring(0, 500) + "...");
|
||
|
||
// 根据分辨率动态设置超时时间
|
||
const timeout = imageParams?.imageSize === "4K" ? 300000 : 120000; // 4K: 5分钟, 其他: 2分钟
|
||
logger.debug(`${apiLogPrefix} Timeout set to ${timeout/1000} seconds for ${imageParams?.imageSize || "default"}`);
|
||
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
||
|
||
try {
|
||
const response = await fetch(url, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
signal: controller.signal,
|
||
});
|
||
|
||
clearTimeout(timeoutId);
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text();
|
||
logger.error(`${apiLogPrefix} Request failed:`, response.status, errorText);
|
||
throw new Error(
|
||
`${isGptImageModel ? "Grsai GPT Image" : "Gemini"} API error: ${response.status} ${response.statusText}`
|
||
);
|
||
}
|
||
|
||
// 处理流式响应（SSE格式）
|
||
logger.debug(`${apiLogPrefix} Reading stream response...`);
|
||
const reader = response.body?.getReader();
|
||
const decoder = new TextDecoder();
|
||
let buffer = "";
|
||
let lastValidData: any = null;
|
||
|
||
if (!reader) {
|
||
throw new Error("Response body is null");
|
||
}
|
||
|
||
while (true) {
|
||
const { done, value } = await reader.read();
|
||
|
||
if (done) {
|
||
logger.debug("[Gemini Image] Stream completed");
|
||
break;
|
||
}
|
||
|
||
// 将新数据添加到缓冲区
|
||
buffer += decoder.decode(value, { stream: true });
|
||
|
||
// 按行分割
|
||
const lines = buffer.split("\n");
|
||
buffer = lines.pop() || ""; // 保留最后一个不完整的行
|
||
|
||
for (const line of lines) {
|
||
const trimmedLine = line.trim();
|
||
|
||
// 解析 SSE 格式: "data: {...}"
|
||
if (trimmedLine.startsWith("data: ")) {
|
||
try {
|
||
const jsonStr = trimmedLine.substring(6); // 移除 "data: " 前缀
|
||
const data = JSON.parse(jsonStr);
|
||
|
||
// 保存最新的有效数据
|
||
lastValidData = data;
|
||
|
||
// 记录进度
|
||
if (data.progress !== undefined) {
|
||
logger.debug(`[Gemini Image] Progress: ${data.progress}%`);
|
||
}
|
||
|
||
// 如果成功完成，立即返回
|
||
if (data.status === "succeeded") {
|
||
logger.debug("[Gemini Image] Generation succeeded!");
|
||
logger.debug("[Gemini Image] Full success data:", JSON.stringify(data).substring(0, 1000)); // 🔍 打印完整成功数据
|
||
return data as GeminiResponse;
|
||
}
|
||
|
||
// 如果失败，抛出错误（不被catch捕获）
|
||
if (data.status === "failed") {
|
||
const failureReason = String(data.failure_reason || "").trim();
|
||
const providerError = String(data.error || "").trim();
|
||
const detailParts = [failureReason, providerError].filter(
|
||
(part, index, arr) =>
|
||
part && arr.findIndex((candidate) => candidate === part) === index
|
||
);
|
||
const errorMsg = `图片生成失败: ${detailParts.join(" / ") || "未知错误"}`;
|
||
logger.error("[Gemini Image] Generation failed:", data.failure_reason, data.error);
|
||
clearTimeout(timeoutId);
|
||
throw new Error(errorMsg);
|
||
}
|
||
} catch (e) {
|
||
// 只捕获JSON解析错误，重新抛出其他错误
|
||
if (e instanceof Error && e.message.includes("图片生成失败")) {
|
||
throw e; // 重新抛出失败错误
|
||
}
|
||
logger.warn("[Gemini Image] Failed to parse SSE line:", trimmedLine);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 流结束后，返回最后一个有效数据
|
||
if (lastValidData) {
|
||
logger.debug("[Gemini Image] Returning last valid data");
|
||
return lastValidData as GeminiResponse;
|
||
}
|
||
|
||
throw new Error("No valid data received from stream");
|
||
} catch (error) {
|
||
clearTimeout(timeoutId);
|
||
if (error instanceof Error && error.name === "AbortError") {
|
||
const timeoutSeconds = Math.round(timeout / 1000);
|
||
logger.error(`${apiLogPrefix} Request timeout after ${timeoutSeconds} seconds`);
|
||
throw new Error(
|
||
`${isGptImageModel ? "Grsai GPT Image" : "Gemini"} 请求超时（${timeoutSeconds}秒），将自动切换备用模型`
|
||
);
|
||
}
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 创建 Gemini 图像生成 Provider
|
||
* 使用与 local-qwen.ts 相同的格式
|
||
*/
|
||
export function createGeminiImage({ apiKey, baseURL }: GeminiImageConfig) {
|
||
return {
|
||
chat: (modelId: string) => ({
|
||
doGenerate: async (options: any) => {
|
||
logger.debug("[Gemini Image] doGenerate called!"); // 调试日志
|
||
logger.debug("[Gemini Image] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "...");
|
||
// 🆕 支持覆盖 modelId（用于备用模型切换）
|
||
const effectiveModelId = options.overrideModelId || modelId;
|
||
logger.debug("[Gemini Image] Using modelId:", effectiveModelId, options.overrideModelId ? "(overridden)" : "(default)");
|
||
logger.debug("[Gemini Image] Image params from options:", options.imageParams);
|
||
|
||
try {
|
||
// 直接传递完整的消息数组（包括文本和图片）
|
||
// 确保imageSize有值，优先级：用户选择 > 默认2K
|
||
const imageParams = {
|
||
aspectRatio: options.imageParams?.aspectRatio || "auto",
|
||
imageSize: options.imageParams?.imageSize || "2K"
|
||
};
|
||
// 🆕 获取用户选择的图片数量，默认 1 张
|
||
const imageCount = Math.min(Math.max(options.imageParams?.imageCount || 1, 1), 2); // Gemini 限制 1-2 张
|
||
logger.debug("[Gemini Image] Final imageParams after defaults:", imageParams);
|
||
logger.debug(`[Gemini Image] Image count: ${imageCount}`);
|
||
|
||
// 🆕 根据用户选择的数量发起请求
|
||
logger.debug(`[Gemini Image] Starting ${imageCount} concurrent request(s)...`);
|
||
const startTime = Date.now();
|
||
|
||
// 创建请求数组（使用 effectiveModelId）
|
||
const requestPromises = Array.from({ length: imageCount }, () =>
|
||
sendGeminiRequest(baseURL, apiKey, options.messages, effectiveModelId, imageParams)
|
||
);
|
||
|
||
const responses = await Promise.all(requestPromises);
|
||
|
||
logger.debug(`[Gemini Image] All ${imageCount} request(s) completed in ${Date.now() - startTime}ms`);
|
||
|
||
// 提取图片并上传 base64 到 S3（并行处理）
|
||
const extractPromises = responses.map(response => extractAndUploadImages(response));
|
||
const imageUrlsArrays = await Promise.all(extractPromises);
|
||
|
||
// 合并所有请求的图片结果
|
||
const allImageUrls = imageUrlsArrays.flat();
|
||
const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize);
|
||
|
||
logger.debug(
|
||
`[Gemini Image] Generated ${allImageUrls.length} images from ${imageCount} concurrent request(s) (uploaded to S3)`
|
||
);
|
||
|
||
// 返回 AI SDK v1 格式
|
||
return {
|
||
text: markdownText,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
logger.error("[Gemini Image] Generation failed:", error);
|
||
|
||
// ❌ 抛出错误以触发故障转移（不要吞掉错误）
|
||
throw error;
|
||
}
|
||
},
|
||
|
||
doStream: async function* (options: any) {
|
||
logger.debug("[Gemini Image] ===== doStream ENTERED! ====="); // 调试日志
|
||
logger.debug("[Gemini Image] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "...");
|
||
// 🆕 支持覆盖 modelId（用于备用模型切换）
|
||
const effectiveModelId = options.overrideModelId || modelId;
|
||
logger.debug("[Gemini Image] Using modelId:", effectiveModelId, options.overrideModelId ? "(overridden)" : "(default)");
|
||
logger.debug("[Gemini Image] Image params from options:", options.imageParams);
|
||
|
||
try {
|
||
// 直接传递完整的消息数组（包括文本和图片）
|
||
// 确保imageSize有值，优先级：用户选择 > 默认2K
|
||
const imageParams = {
|
||
aspectRatio: options.imageParams?.aspectRatio || "auto",
|
||
imageSize: options.imageParams?.imageSize || "2K"
|
||
};
|
||
// 🆕 获取用户选择的图片数量，默认 1 张
|
||
const imageCount = Math.min(Math.max(options.imageParams?.imageCount || 1, 1), 2); // Gemini 限制 1-2 张
|
||
logger.debug("[Gemini Image] Final imageParams after defaults:", imageParams);
|
||
logger.debug(`[Gemini Image Stream] Image count: ${imageCount}`);
|
||
|
||
// 🆕 根据用户选择的数量发起请求
|
||
logger.debug(`[Gemini Image Stream] Starting ${imageCount} concurrent request(s)...`);
|
||
const startTime = Date.now();
|
||
|
||
// 创建请求数组（使用 effectiveModelId）
|
||
const requestPromises = Array.from({ length: imageCount }, () =>
|
||
sendGeminiRequest(baseURL, apiKey, options.messages, effectiveModelId, imageParams)
|
||
);
|
||
|
||
const responses = await Promise.all(requestPromises);
|
||
|
||
logger.debug(`[Gemini Image Stream] All ${imageCount} request(s) completed in ${Date.now() - startTime}ms`);
|
||
|
||
// 提取图片并上传 base64 到 S3（并行处理）
|
||
const extractPromises = responses.map(response => extractAndUploadImages(response));
|
||
const imageUrlsArrays = await Promise.all(extractPromises);
|
||
|
||
// 合并所有请求的图片结果
|
||
const allImageUrls = imageUrlsArrays.flat();
|
||
const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize);
|
||
|
||
logger.debug(
|
||
`[Gemini Image Stream] Generated ${allImageUrls.length} images from ${imageCount} concurrent request(s) (uploaded to S3)`
|
||
);
|
||
|
||
// 一次性发送整个图片 markdown（不再分块，因为 base64 图片很大）
|
||
yield { type: "text-delta" as const, textDelta: markdownText };
|
||
|
||
logger.debug("[Gemini Image Stream] Sent complete markdown");
|
||
|
||
// 发送完成信号
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
logger.error("[Gemini Image Stream] Generation failed:", error);
|
||
|
||
const errorMessage =
|
||
error instanceof Error
|
||
? `图片生成失败: ${error.message}`
|
||
: "图片生成失败，请稍后重试";
|
||
|
||
// 发送错误信息
|
||
yield { type: "text-delta" as const, textDelta: errorMessage };
|
||
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "error" as const,
|
||
usage: { promptTokens: 0, completionTokens: 0 },
|
||
};
|
||
}
|
||
},
|
||
|
||
modelId,
|
||
provider: "gemini-image",
|
||
specificationVersion: "v1" as const,
|
||
defaultObjectGenerationMode: "json" as const,
|
||
}),
|
||
};
|
||
}
|