Fix Gemini image reference forwarding for grsai
This commit is contained in:
Your Name
parent
0ba1ea521f
commit
37e48b7878
@ -581,45 +581,64 @@ export async function convertMessagesWithImages(
|
||||
if (hasAnnotation) {
|
||||
chatApiLogger.info("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示");
|
||||
}
|
||||
|
||||
// 对于 Gemini，需要特殊处理图片
|
||||
const convertedMessages = [];
|
||||
|
||||
|
||||
for (const message of preparedMessages) {
|
||||
const content: any[] = [];
|
||||
|
||||
|
||||
// 🆕 如果是用户消息且有标注图片，在文本前添加系统提示
|
||||
let hasAddedAnnotationPrompt = false;
|
||||
|
||||
|
||||
// 处理消息的各个部分
|
||||
for (const part of message.parts) {
|
||||
if (part.type === "text" && part.text) {
|
||||
let textContent = part.text;
|
||||
|
||||
|
||||
// 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示
|
||||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) {
|
||||
textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent;
|
||||
hasAddedAnnotationPrompt = true;
|
||||
chatApiLogger.info("[Message Convert] 🎨 已注入标注提示词");
|
||||
}
|
||||
|
||||
|
||||
content.push({
|
||||
type: "text",
|
||||
text: textContent,
|
||||
});
|
||||
} else if (part.type === "file" && part.url) {
|
||||
// 直接使用 URL 格式（不再转换为 Base64）
|
||||
// gemini-image 和 huanke-gemini provider 内部会处理图片上传
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidateImageUrl =
|
||||
part.type === "file"
|
||||
? part.url || part.data
|
||||
: part.type === "image"
|
||||
? part.image || part.url
|
||||
: part.type === "image_url"
|
||||
? part.image_url?.url || part.url
|
||||
: undefined;
|
||||
|
||||
if (typeof candidateImageUrl !== "string" || !candidateImageUrl.trim()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalizedImageUrl = candidateImageUrl.trim();
|
||||
|
||||
// 兼容 file/image/image_url 三类输入；Gemini 内部会将图片交给 S3 后再请求生图网关。
|
||||
if (part.type === "file" || part.type === "image" || part.type === "image_url") {
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: part.url,
|
||||
url: normalizedImageUrl,
|
||||
},
|
||||
});
|
||||
chatApiLogger.info("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "...");
|
||||
chatApiLogger.info(
|
||||
"[Message Convert] Image kept as URL format:",
|
||||
normalizedImageUrl.substring(0, 80) + "..."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本
|
||||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) {
|
||||
content.unshift({
|
||||
@ -628,12 +647,10 @@ export async function convertMessagesWithImages(
|
||||
});
|
||||
chatApiLogger.info("[Message Convert] 🎨 已在消息开头注入标注提示词");
|
||||
}
|
||||
|
||||
|
||||
convertedMessages.push({
|
||||
role: message.role,
|
||||
content: content.length === 1 && content[0].type === "text"
|
||||
? content[0].text
|
||||
: content,
|
||||
content: content.length === 1 && content[0].type === "text" ? content[0].text : content,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -715,6 +715,18 @@ async function sendGeminiRequest(
|
||||
imageSize?: string;
|
||||
}
|
||||
): Promise<GeminiResponse> {
|
||||
const normalizeAspectRatio = (aspectRatio: string) => {
|
||||
const normalized = String(aspectRatio || "").trim();
|
||||
if (!normalized || normalized.toLowerCase() === "auto") {
|
||||
return undefined;
|
||||
}
|
||||
if (!/^\d+\s*:\s*\d+$/i.test(normalized)) {
|
||||
return undefined;
|
||||
}
|
||||
const [widthRaw, heightRaw] = normalized.split(":").map((item) => item.trim());
|
||||
return `${Number(widthRaw)}:${Number(heightRaw)}`;
|
||||
};
|
||||
|
||||
const isGptImageModel = /^gpt-image/i.test(modelId);
|
||||
const shouldUseGrsaiAsyncPolling =
|
||||
!isGptImageModel &&
|
||||
@ -739,18 +751,20 @@ async function sendGeminiRequest(
|
||||
messages.forEach((msg) => {
|
||||
if (msg.role === "user" && Array.isArray(msg.content)) {
|
||||
msg.content.forEach((contentPart: any) => {
|
||||
// 支持标准 image_url
|
||||
if (contentPart.type === "image_url" && contentPart.image_url?.url) {
|
||||
urls.push(contentPart.image_url.url);
|
||||
}
|
||||
// 🆕 支持 CoreMessage 的 image 类型 (可能来自 Attachment)
|
||||
else if (contentPart.type === "image" && contentPart.image) {
|
||||
if (typeof contentPart.image === 'string') {
|
||||
urls.push(contentPart.image);
|
||||
} else if (contentPart.image instanceof URL) {
|
||||
urls.push(contentPart.image.toString());
|
||||
}
|
||||
const candidateImageUrl =
|
||||
contentPart.type === "image_url"
|
||||
? contentPart.image_url?.url
|
||||
: contentPart.type === "file"
|
||||
? contentPart.url || contentPart.data
|
||||
: contentPart.type === "image"
|
||||
? contentPart.image || contentPart.url
|
||||
: undefined;
|
||||
|
||||
if (!candidateImageUrl || typeof candidateImageUrl !== "string") {
|
||||
return;
|
||||
}
|
||||
|
||||
urls.push(candidateImageUrl);
|
||||
});
|
||||
}
|
||||
});
|
||||
@ -789,16 +803,39 @@ async function sendGeminiRequest(
|
||||
? await inferImageAspectRatioFromUrl(processedUrls[0])
|
||||
: null;
|
||||
const resolvedAspectRatio = inferredAspectRatio || requestedAspectRatio;
|
||||
requestBody.size = resolveGrsaiGptImageSize(resolvedAspectRatio, imageParams?.imageSize);
|
||||
const normalizedRequestedAspectRatio = normalizeAspectRatio(requestedAspectRatio);
|
||||
const resolvedSize = resolveGrsaiGptImageSize(resolvedAspectRatio, imageParams?.imageSize);
|
||||
|
||||
// GPT-image（如 gpt-image-2-vip）在不同网关中参数差异较大：
|
||||
// - 有的网关只识别 size；
|
||||
// - 有的网关同时支持 aspectRatio/aspect_ratio；
|
||||
// - 有的会读取 imageSize/image_size；
|
||||
// 为避免接口兼容问题，保留核心 size 后补齐兼容字段。
|
||||
requestBody.size = resolvedSize;
|
||||
requestBody.imageSize = imageParams?.imageSize;
|
||||
requestBody.image_size = imageParams?.imageSize;
|
||||
if (normalizedRequestedAspectRatio) {
|
||||
requestBody.aspectRatio = normalizedRequestedAspectRatio;
|
||||
requestBody.aspect_ratio = normalizedRequestedAspectRatio;
|
||||
requestBody.aspectRatioRaw = requestedAspectRatio;
|
||||
}
|
||||
requestBody.shutProgress = false;
|
||||
|
||||
if (processedUrls.length > 0) {
|
||||
requestBody.images = processedUrls;
|
||||
if (processedUrls.length === 1) {
|
||||
requestBody.image = processedUrls[0];
|
||||
}
|
||||
requestBody.imageUrls = processedUrls;
|
||||
}
|
||||
logger.debug(
|
||||
`${apiLogPrefix} GPT image payload mapping: requestedAspectRatio=${requestedAspectRatio}, resolvedAspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"}, resolvedSize=${requestBody.size}`
|
||||
`${apiLogPrefix} GPT image payload mapping: requestedAspectRatio=${requestedAspectRatio}, normalizedRequestedAspectRatio=${normalizedRequestedAspectRatio || "n/a"}, resolvedAspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"}, resolvedSize=${resolvedSize}`
|
||||
);
|
||||
if (inferredAspectRatio) {
|
||||
logger.debug(`${apiLogPrefix} Inferred reference aspect ratio: ${inferredAspectRatio}`);
|
||||
}
|
||||
logger.debug(
|
||||
`${apiLogPrefix} Resolved size: aspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"} => ${requestBody.size}`
|
||||
`${apiLogPrefix} Resolved size: aspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"} => ${resolvedSize}`
|
||||
);
|
||||
} else {
|
||||
// 添加 Nano Banana 特定参数
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user