1003 lines
31 KiB
TypeScript
1003 lines
31 KiB
TypeScript
import { createLogger } from "@/lib/observability/logger";
|
||
import { convertToModelMessages } from "ai";
|
||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||
import { extractImageUrlsFromText } from "@/lib/image-url";
|
||
import { parseImageAltMetadata } from "@/lib/ai/runtime/generated-image-persistence";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import type { ChatMessage } from "@/lib/types";
|
||
import { ANNOTATION_SYSTEM_PROMPT, hasAnnotatedImage } from "./message-preparation";
|
||
|
||
const chatApiLogger = createLogger("chat.imageMedia");
|
||
|
||
|
||
export function guessImageMediaType(url: string): string {
|
||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg";
|
||
if (cleanUrl.endsWith(".webp")) return "image/webp";
|
||
if (cleanUrl.endsWith(".gif")) return "image/gif";
|
||
if (cleanUrl.endsWith(".bmp")) return "image/bmp";
|
||
if (cleanUrl.endsWith(".svg")) return "image/svg+xml";
|
||
return "image/png";
|
||
}
|
||
|
||
export function injectHistoryImagesToLastUserMessage(
|
||
messages: ChatMessage[],
|
||
historyImages: string[]
|
||
): ChatMessage[] {
|
||
if (!historyImages || historyImages.length === 0) return messages;
|
||
|
||
let lastUserIndex = -1;
|
||
for (let i = messages.length - 1; i >= 0; i--) {
|
||
if (messages[i]?.role === "user") {
|
||
lastUserIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (lastUserIndex < 0) return messages;
|
||
|
||
const lastUserMessage = messages[lastUserIndex];
|
||
const hasExistingImage = lastUserMessage.parts?.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
if (hasExistingImage) return messages;
|
||
|
||
const imageParts = historyImages.map((url, index) => ({
|
||
type: "file" as const,
|
||
url,
|
||
name: `history-image-${index + 1}`,
|
||
mediaType: guessImageMediaType(url),
|
||
mimeType: guessImageMediaType(url),
|
||
}));
|
||
|
||
const updatedMessage: ChatMessage = {
|
||
...lastUserMessage,
|
||
parts: [...lastUserMessage.parts, ...imageParts],
|
||
};
|
||
|
||
return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
|
||
}
|
||
|
||
export function isImageInputPart(part: any) {
|
||
return part?.type === "file" || part?.type === "image" || part?.type === "image_url";
|
||
}
|
||
|
||
export function normalizeResolvedTargetImageUrls(imageUrls?: string[] | null) {
|
||
return Array.from(
|
||
new Set(
|
||
(imageUrls || [])
|
||
.map((url) => String(url || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
)
|
||
).slice(0, 8);
|
||
}
|
||
|
||
export function summarizeTargetImageUrl(url?: string) {
|
||
const normalized = String(url || "").trim();
|
||
if (!normalized) return undefined;
|
||
try {
|
||
const parsed = new URL(normalized);
|
||
return `${parsed.origin}${parsed.pathname}`.slice(0, 160);
|
||
} catch {
|
||
return normalized.slice(0, 160);
|
||
}
|
||
}
|
||
|
||
export function applyResolvedTargetImagesToLastUserMessage(
|
||
messages: ChatMessage[],
|
||
targetImageUrls: string[]
|
||
): ChatMessage[] {
|
||
const normalizedTargetImageUrls = normalizeResolvedTargetImageUrls(targetImageUrls);
|
||
if (normalizedTargetImageUrls.length === 0) return messages;
|
||
|
||
let lastUserIndex = -1;
|
||
for (let i = messages.length - 1; i >= 0; i--) {
|
||
if (messages[i]?.role === "user") {
|
||
lastUserIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (lastUserIndex < 0) return messages;
|
||
const lastUserMessage = messages[lastUserIndex];
|
||
const nonImageParts = (lastUserMessage.parts || []).filter((part: any) => !isImageInputPart(part));
|
||
const imageParts = normalizedTargetImageUrls.map((url, index) => ({
|
||
type: "file" as const,
|
||
url,
|
||
name: `target-image-${index + 1}`,
|
||
mediaType: guessImageMediaType(url),
|
||
mimeType: guessImageMediaType(url),
|
||
}));
|
||
|
||
const updatedMessage: ChatMessage = {
|
||
...lastUserMessage,
|
||
parts: [...imageParts, ...nonImageParts],
|
||
};
|
||
|
||
return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
|
||
}
|
||
|
||
export function compactAutoDemand(text: string, maxLength = 56): string {
|
||
const normalized = text.replace(/\s+/g, " ").trim();
|
||
if (!normalized) return "按你刚才的描述执行";
|
||
if (normalized.length <= maxLength) return normalized;
|
||
return `${normalized.slice(0, maxLength)}...`;
|
||
}
|
||
|
||
export function buildAutoModeFallbackMessage(params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
fromHistory: boolean;
|
||
imageSummary?: string;
|
||
}): string {
|
||
const demand = compactAutoDemand(params.text);
|
||
|
||
if (params.hasImage) {
|
||
const sourceLine = params.fromHistory
|
||
? "基于上一轮图片继续处理。"
|
||
: "基于当前上传图片处理。";
|
||
|
||
return [
|
||
"### 意图分析",
|
||
`- ${demand}`,
|
||
"### 图片分析",
|
||
params.imageSummary
|
||
? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}`
|
||
: `- ${sourceLine}`,
|
||
"- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。",
|
||
"### 任务规划",
|
||
"- 先完成本轮核心改动，再检查边缘与光影一致性。",
|
||
"- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。",
|
||
"### 输出设置",
|
||
"- 默认单图输出，优先稳定命中。",
|
||
].join("\n");
|
||
}
|
||
|
||
return [
|
||
"### 意图分析",
|
||
`- ${demand}`,
|
||
"### 任务规划",
|
||
"- 先锁定主体与场景，再细化风格、材质与光影。",
|
||
"### 输出设置",
|
||
"- 默认单图输出，优先稳定命中。",
|
||
].join("\n");
|
||
}
|
||
|
||
export function sanitizePromptOnlyUserIntent(text: string): string {
|
||
return String(text || "")
|
||
.replace(/先?分析(?:一下)?(?:这张)?(?:图片|图像|这张图)?/g, "")
|
||
.replace(/帮我|给我|先给我|只给我|仅给我/g, "")
|
||
.replace(/优化(?:一下)?提示词|提示词优化|提示词|prompt/gi, "")
|
||
.replace(/不要生图|先别生图|不需要生图|不要生成|别生成|不直接生成/g, "")
|
||
.replace(/[，。；：,.!！?？]/g, " ")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
}
|
||
|
||
export function buildPromptOnlyOptimizationSeed(params: {
|
||
text: string;
|
||
imageSummary?: string;
|
||
}): string {
|
||
const cleanedIntent = sanitizePromptOnlyUserIntent(params.text);
|
||
const summary = String(params.imageSummary || "").replace(/\s+/g, " ").trim();
|
||
|
||
if (cleanedIntent && summary) {
|
||
return `${cleanedIntent}。参考图关键特征：${summary}`;
|
||
}
|
||
if (summary) {
|
||
return `基于参考图生成一段可直接复现核心主体、场景、风格与光线特征的中文提示词。参考图关键特征：${summary}`;
|
||
}
|
||
if (cleanedIntent) {
|
||
return cleanedIntent;
|
||
}
|
||
return "基于当前参考图整理一段可直接使用的中文提示词。";
|
||
}
|
||
|
||
export function buildPromptOnlyReply(params: {
|
||
userText: string;
|
||
imageSummary?: string;
|
||
optimizedPrompt: string;
|
||
}): string {
|
||
const demand = sanitizePromptOnlyUserIntent(params.userText) || "你希望我先分析这张图并整理可直接使用的提示词";
|
||
const summaryLines = String(params.imageSummary || "")
|
||
.split(/\r?\n/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean);
|
||
const formattedSummary =
|
||
summaryLines.length > 0
|
||
? summaryLines.map((line) => `- ${line}`).join("\n")
|
||
: "- 我已基于当前图片梳理主体、场景、风格与光线特征。";
|
||
|
||
return [
|
||
"【需求理解】",
|
||
`- ${demand}`,
|
||
"【图片分析】",
|
||
formattedSummary,
|
||
"【优化后提示词】",
|
||
`- ${params.optimizedPrompt.replace(/\s+/g, " ").trim()}`,
|
||
"【使用建议】",
|
||
"- 本轮仅输出优化后的提示词，不直接执行生图。",
|
||
].join("\n");
|
||
}
|
||
|
||
export function hasStructuredAgentSections(message?: string): boolean {
|
||
if (!message) return false;
|
||
return /(?:###\s*意图分析|【\s*意图分析\s*】|###\s*任务规划|【\s*任务规划\s*】|###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
|
||
message,
|
||
);
|
||
}
|
||
|
||
export function ensureStructuredAgentFriendlyMessage(params: {
|
||
message?: string;
|
||
text: string;
|
||
hasImage: boolean;
|
||
fromHistory: boolean;
|
||
}): string {
|
||
const existingMessage = String(params.message || "").trim();
|
||
if (hasStructuredAgentSections(existingMessage)) {
|
||
return existingMessage;
|
||
}
|
||
|
||
const fallback = buildAutoModeFallbackMessage({
|
||
text: params.text,
|
||
hasImage: params.hasImage,
|
||
fromHistory: params.fromHistory,
|
||
});
|
||
|
||
if (!existingMessage) {
|
||
return fallback;
|
||
}
|
||
|
||
const normalizedSupplement = existingMessage.replace(/\s+/g, " ").trim();
|
||
if (!normalizedSupplement) {
|
||
return fallback;
|
||
}
|
||
|
||
if (fallback.includes("### 任务规划\n")) {
|
||
return fallback.replace(
|
||
"### 任务规划\n",
|
||
`### 任务规划\n- 补充说明：${normalizedSupplement}\n`,
|
||
);
|
||
}
|
||
|
||
return fallback.replace(
|
||
"### 执行计划\n",
|
||
`### 执行计划\n- 补充说明：${normalizedSupplement}\n`,
|
||
);
|
||
}
|
||
|
||
export function extractMessageImageUrls(parts: any[]): string[] {
|
||
return (parts || [])
|
||
.map((part: any) => {
|
||
if (!part) return null;
|
||
if (part.type === "file") return part.url || part.data || null;
|
||
if (part.type === "image") return part.url || part.image || null;
|
||
if (part.type === "image_url") return part.image_url?.url || null;
|
||
return null;
|
||
})
|
||
.filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url));
|
||
}
|
||
|
||
export function extractMessageImageAssetIds(parts: any[]): string[] {
|
||
return Array.from(
|
||
new Set(
|
||
(parts || [])
|
||
.map((part: any) => {
|
||
if (!part || typeof part !== "object") return null;
|
||
return typeof part.assetId === "string" && part.assetId.trim()
|
||
? part.assetId.trim()
|
||
: null;
|
||
})
|
||
.filter((assetId: any): assetId is string => typeof assetId === "string" && assetId.length > 0),
|
||
),
|
||
);
|
||
}
|
||
|
||
export function shouldEnrichImageAnalysisText(message?: string): boolean {
|
||
if (!message) return false;
|
||
if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false;
|
||
return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message);
|
||
}
|
||
|
||
export function injectImageAnalysisSummary(message: string, summary: string): string {
|
||
const normalizedSummaryLines = summary
|
||
.split(/\r?\n/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean);
|
||
if (normalizedSummaryLines.length === 0) return message;
|
||
const lines = message.split("\n");
|
||
const analysisHeaderIndex = lines.findIndex((line) =>
|
||
/^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim())
|
||
);
|
||
if (analysisHeaderIndex < 0) return message;
|
||
|
||
for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) {
|
||
const current = lines[i]?.trim() || "";
|
||
if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break;
|
||
if (/^- /.test(current)) {
|
||
lines.splice(i, 1, ...normalizedSummaryLines.map((line) => `- ${line}`));
|
||
return lines.join("\n");
|
||
}
|
||
}
|
||
|
||
const insertIndex = analysisHeaderIndex + 1;
|
||
lines.splice(insertIndex, 0, ...normalizedSummaryLines.map((line) => `- ${line}`));
|
||
return lines.join("\n");
|
||
}
|
||
|
||
export function ensureImageAnalysisSummarySection(message: string, summary: string): string {
|
||
const normalizedSummaryLines = summary
|
||
.split(/\r?\n/)
|
||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||
.filter(Boolean);
|
||
if (normalizedSummaryLines.length === 0) return message;
|
||
|
||
if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) {
|
||
return injectImageAnalysisSummary(message, normalizedSummaryLines.join("\n"));
|
||
}
|
||
|
||
const trimmed = message.trim();
|
||
const summaryBlock = normalizedSummaryLines.map((line) => `- ${line}`).join("\n");
|
||
if (!trimmed) return `【图片分析】\n${summaryBlock}`;
|
||
return `${trimmed}\n【图片分析】\n${summaryBlock}`;
|
||
}
|
||
|
||
/**
|
||
* 自定义消息转换函数
|
||
* 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64）
|
||
* 因为这些 provider 内部会自行处理图片上传到 S3
|
||
*/
|
||
export async function convertMessagesWithImages(
|
||
messages: ChatMessage[],
|
||
providerType:
|
||
| string
|
||
| {
|
||
providerType?: string;
|
||
providerBaseUrl?: string | null;
|
||
modelId?: string | null;
|
||
displayName?: string | null;
|
||
forceInlineImageUrls?: boolean;
|
||
loggerPrefix?: string;
|
||
},
|
||
) {
|
||
type ImageConversionTarget = {
|
||
providerType?: string;
|
||
providerBaseUrl?: string | null;
|
||
modelId?: string | null;
|
||
displayName?: string | null;
|
||
forceInlineImageUrls?: boolean;
|
||
loggerPrefix?: string;
|
||
};
|
||
|
||
const normalizeTarget = (
|
||
target: string | ImageConversionTarget,
|
||
): Required<Pick<ImageConversionTarget, "providerType" | "forceInlineImageUrls" | "loggerPrefix">> &
|
||
Omit<ImageConversionTarget, "providerType" | "forceInlineImageUrls" | "loggerPrefix"> => {
|
||
if (typeof target === "string") {
|
||
return {
|
||
providerType: target,
|
||
providerBaseUrl: null,
|
||
modelId: null,
|
||
displayName: null,
|
||
forceInlineImageUrls: false,
|
||
loggerPrefix: "[Chat API] Vision inline",
|
||
};
|
||
}
|
||
|
||
return {
|
||
providerType: String(target.providerType || ""),
|
||
providerBaseUrl: target.providerBaseUrl || null,
|
||
modelId: target.modelId || null,
|
||
displayName: target.displayName || null,
|
||
forceInlineImageUrls: target.forceInlineImageUrls === true,
|
||
loggerPrefix: target.loggerPrefix || "[Chat API] Vision inline",
|
||
};
|
||
};
|
||
|
||
const extractImagePartUrl = (part: unknown): string | null => {
|
||
const partAny = part as any;
|
||
const candidateUrl =
|
||
partAny?.type === "file"
|
||
? partAny.url || partAny.data
|
||
: partAny?.type === "image"
|
||
? partAny.image || partAny.url
|
||
: partAny?.type === "image_url"
|
||
? partAny.image_url?.url
|
||
: undefined;
|
||
|
||
if (typeof candidateUrl !== "string") return null;
|
||
const normalized = candidateUrl.trim();
|
||
if (!/^https?:\/\//i.test(normalized)) return null;
|
||
return normalized;
|
||
};
|
||
|
||
const replaceImagePartUrl = (part: unknown, nextUrl: string): unknown => {
|
||
const partAny = part as any;
|
||
if (partAny?.type === "file") {
|
||
return {
|
||
...partAny,
|
||
url: nextUrl,
|
||
data: nextUrl,
|
||
mediaType: String(partAny.mediaType || "").startsWith("image/")
|
||
? partAny.mediaType
|
||
: "image/jpeg",
|
||
mimeType: String(partAny.mimeType || "").startsWith("image/")
|
||
? partAny.mimeType
|
||
: "image/jpeg",
|
||
};
|
||
}
|
||
if (partAny?.type === "image") {
|
||
return {
|
||
...partAny,
|
||
image: nextUrl,
|
||
url: nextUrl,
|
||
};
|
||
}
|
||
if (partAny?.type === "image_url") {
|
||
return {
|
||
...partAny,
|
||
image_url: {
|
||
...(partAny.image_url || {}),
|
||
url: nextUrl,
|
||
},
|
||
};
|
||
}
|
||
return part;
|
||
};
|
||
|
||
const inlineImageUrlsInMessages = async (
|
||
sourceMessages: ChatMessage[],
|
||
target: ReturnType<typeof normalizeTarget>,
|
||
) => {
|
||
const uniqueUrls = Array.from(
|
||
new Set(
|
||
sourceMessages
|
||
.flatMap((message) => message.parts || [])
|
||
.map((part) => extractImagePartUrl(part))
|
||
.filter((url): url is string => Boolean(url)),
|
||
),
|
||
);
|
||
|
||
if (uniqueUrls.length === 0) {
|
||
return sourceMessages;
|
||
}
|
||
|
||
const { inlineDataUrls } = await inlineVisionImages(uniqueUrls, {
|
||
maxImages: Math.min(8, uniqueUrls.length),
|
||
maxSide: 1200,
|
||
timeoutMs: 12000,
|
||
retries: 2,
|
||
loggerPrefix: target.loggerPrefix,
|
||
}).catch((error) => {
|
||
chatApiLogger.warn(`${target.loggerPrefix} failed to inline message images:`, error);
|
||
return { inlineDataUrls: [], skipped: [] };
|
||
});
|
||
|
||
if (inlineDataUrls.length === 0) {
|
||
return sourceMessages;
|
||
}
|
||
|
||
const inlineMap = new Map<string, string>();
|
||
for (let index = 0; index < inlineDataUrls.length; index += 1) {
|
||
const originalUrl = uniqueUrls[index];
|
||
const inlineUrl = inlineDataUrls[index];
|
||
if (originalUrl && inlineUrl) {
|
||
inlineMap.set(originalUrl, inlineUrl);
|
||
}
|
||
}
|
||
|
||
if (inlineMap.size === 0) {
|
||
return sourceMessages;
|
||
}
|
||
|
||
chatApiLogger.info(`${target.loggerPrefix} inlined chat message images:`, {
|
||
total: uniqueUrls.length,
|
||
inlined: inlineMap.size,
|
||
providerType: target.providerType,
|
||
modelId: target.modelId,
|
||
displayName: target.displayName,
|
||
});
|
||
|
||
return sourceMessages.map((message) => ({
|
||
...message,
|
||
parts: (message.parts || []).map((part) => {
|
||
const originalUrl = extractImagePartUrl(part);
|
||
if (!originalUrl) return part;
|
||
const inlineUrl = inlineMap.get(originalUrl);
|
||
return inlineUrl ? replaceImagePartUrl(part, inlineUrl) : part;
|
||
}) as typeof message.parts,
|
||
})) as ChatMessage[];
|
||
};
|
||
|
||
const target = normalizeTarget(providerType);
|
||
let preparedMessages = messages;
|
||
|
||
if (target.forceInlineImageUrls) {
|
||
preparedMessages = await inlineImageUrlsInMessages(preparedMessages, target);
|
||
}
|
||
|
||
const normalizedProviderType = target.providerType;
|
||
|
||
if (
|
||
normalizedProviderType === "bfl-image" ||
|
||
normalizedProviderType === "ruxa-image" ||
|
||
normalizedProviderType === "fal-image" ||
|
||
normalizedProviderType === "snapgen-image" ||
|
||
normalizedProviderType === "openai-image" ||
|
||
normalizedProviderType === "bananaimage" ||
|
||
normalizedProviderType === "visionary" ||
|
||
normalizedProviderType === "atlascloud" ||
|
||
normalizedProviderType === "vip98k" ||
|
||
normalizedProviderType === "stepfun-image" ||
|
||
normalizedProviderType === "midjourney" ||
|
||
normalizedProviderType === "runway" ||
|
||
normalizedProviderType === "krea"
|
||
) {
|
||
const convertedMessages = [];
|
||
|
||
for (const message of preparedMessages) {
|
||
const content: any[] = [];
|
||
|
||
for (const part of message.parts) {
|
||
const partAny = part as any;
|
||
|
||
if (partAny.type === "text" && partAny.text) {
|
||
content.push({
|
||
type: "text",
|
||
text: partAny.text,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const candidateUrl =
|
||
partAny.type === "file"
|
||
? (partAny.url || partAny.data)
|
||
: partAny.type === "image"
|
||
? (partAny.image || partAny.url)
|
||
: partAny.type === "image_url"
|
||
? partAny.image_url?.url
|
||
: undefined;
|
||
|
||
if (!candidateUrl || typeof candidateUrl !== "string") continue;
|
||
if (candidateUrl.startsWith("data:")) {
|
||
throw new Error("当前模型不支持 Base64 图片传输，请先上传图片后再编辑");
|
||
}
|
||
if (!candidateUrl.startsWith("http")) continue;
|
||
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: candidateUrl,
|
||
},
|
||
});
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text" ? content[0].text : content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
// 所有 provider 都使用 URL 格式传输图片
|
||
// gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL
|
||
// 这里不再进行 Base64 转换，直接返回标准格式
|
||
if (normalizedProviderType !== "gemini") {
|
||
return convertToModelMessages(preparedMessages);
|
||
}
|
||
|
||
// 🆕 检测是否有标注图片
|
||
const hasAnnotation = hasAnnotatedImage(messages);
|
||
if (hasAnnotation) {
|
||
chatApiLogger.info("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示");
|
||
}
|
||
// 对于 Gemini，需要特殊处理图片
|
||
const convertedMessages = [];
|
||
|
||
for (const message of preparedMessages) {
|
||
const content: any[] = [];
|
||
|
||
// 🆕 如果是用户消息且有标注图片，在文本前添加系统提示
|
||
let hasAddedAnnotationPrompt = false;
|
||
|
||
// 处理消息的各个部分
|
||
for (const part of message.parts) {
|
||
if (part.type === "text" && part.text) {
|
||
let textContent = part.text;
|
||
|
||
// 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) {
|
||
textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent;
|
||
hasAddedAnnotationPrompt = true;
|
||
chatApiLogger.info("[Message Convert] 🎨 已注入标注提示词");
|
||
}
|
||
|
||
content.push({
|
||
type: "text",
|
||
text: textContent,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const typedPart = part as {
|
||
type: string;
|
||
url?: string;
|
||
image?: string;
|
||
image_url?: { url?: string };
|
||
data?: string;
|
||
};
|
||
const candidateImageUrl =
|
||
typedPart.type === "file"
|
||
? typedPart.url || typedPart.data
|
||
: typedPart.type === "image"
|
||
? typedPart.image || typedPart.url
|
||
: typedPart.type === "image_url"
|
||
? typedPart.image_url?.url || typedPart.url
|
||
: undefined;
|
||
|
||
if (typeof candidateImageUrl !== "string" || !candidateImageUrl.trim()) {
|
||
continue;
|
||
}
|
||
|
||
const normalizedImageUrl = candidateImageUrl.trim();
|
||
|
||
// 兼容 file/image/image_url 三类输入；Gemini 内部会将图片交给 S3 后再请求生图网关。
|
||
if (typedPart.type === "file" || typedPart.type === "image" || typedPart.type === "image_url") {
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: normalizedImageUrl,
|
||
},
|
||
});
|
||
chatApiLogger.info(
|
||
"[Message Convert] Image kept as URL format:",
|
||
normalizedImageUrl.substring(0, 80) + "..."
|
||
);
|
||
}
|
||
}
|
||
|
||
// 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) {
|
||
content.unshift({
|
||
type: "text",
|
||
text: ANNOTATION_SYSTEM_PROMPT,
|
||
});
|
||
chatApiLogger.info("[Message Convert] 🎨 已在消息开头注入标注提示词");
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text" ? content[0].text : content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
export function extractOriginalImageUrlsFromText(text: string): string[] {
|
||
return extractImageUrlsFromText(text);
|
||
}
|
||
|
||
export function normalizeImageInputRef(rawValue: unknown, requestOrigin?: string): string | null {
|
||
if (typeof rawValue !== "string") return null;
|
||
const trimmed = rawValue.trim();
|
||
if (!trimmed) return null;
|
||
|
||
if (/^data:image\/[a-z0-9+.-]+;base64,/i.test(trimmed)) {
|
||
return trimmed.replace(/\s+/g, "");
|
||
}
|
||
|
||
if (/^https?:\/\//i.test(trimmed)) {
|
||
return trimmed;
|
||
}
|
||
|
||
if (requestOrigin && trimmed.startsWith("/")) {
|
||
try {
|
||
return new URL(trimmed, requestOrigin).toString();
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
export function collectImageInputsFromMessageParts(
|
||
parts: Array<Record<string, any>> | undefined,
|
||
requestOrigin?: string,
|
||
): string[] {
|
||
if (!Array.isArray(parts) || parts.length === 0) return [];
|
||
|
||
const result: string[] = [];
|
||
const pushInput = (rawValue: unknown) => {
|
||
const normalized = normalizeImageInputRef(rawValue, requestOrigin);
|
||
if (!normalized) return;
|
||
if (!result.includes(normalized)) {
|
||
result.push(normalized);
|
||
}
|
||
};
|
||
|
||
for (const part of parts) {
|
||
if (!part) continue;
|
||
|
||
if (part.type === "file") {
|
||
pushInput(part.url);
|
||
pushInput(part.data);
|
||
continue;
|
||
}
|
||
|
||
if (part.type === "image") {
|
||
pushInput(part.url);
|
||
pushInput(part.image);
|
||
pushInput(part.data);
|
||
continue;
|
||
}
|
||
|
||
if (part.type === "image_url") {
|
||
pushInput(part.image_url?.url);
|
||
pushInput(part.url);
|
||
continue;
|
||
}
|
||
|
||
if (part.type === "text" && typeof part.text === "string") {
|
||
const parsedUrls = extractOriginalImageUrlsFromText(part.text);
|
||
for (const parsedUrl of parsedUrls) {
|
||
pushInput(parsedUrl);
|
||
}
|
||
}
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
export function mimeTypeToExtension(mimeType: string) {
|
||
const normalized = mimeType.toLowerCase();
|
||
if (normalized.includes("png")) return "png";
|
||
if (normalized.includes("webp")) return "webp";
|
||
if (normalized.includes("gif")) return "gif";
|
||
if (normalized.includes("bmp")) return "bmp";
|
||
return "jpg";
|
||
}
|
||
|
||
export function getExtensionFromUrl(url: string): string {
|
||
try {
|
||
const pathname = new URL(url).pathname;
|
||
const ext = pathname.split(".").pop()?.toLowerCase();
|
||
if (!ext) return "jpg";
|
||
if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext;
|
||
} catch {
|
||
// ignore
|
||
}
|
||
return "jpg";
|
||
}
|
||
|
||
export async function uploadRemoteImageToStorage(imageUrl: string): Promise<string> {
|
||
const response = await fetch(imageUrl, {
|
||
method: "GET",
|
||
signal: AbortSignal.timeout(30_000),
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`);
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
||
if (contentType && !contentType.startsWith("image/")) {
|
||
throw new Error(`Remote url is not an image: ${contentType}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
|
||
throw new Error("Remote image content is empty");
|
||
}
|
||
|
||
const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl);
|
||
const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
export async function isRemoteImageAccessible(imageUrl: string): Promise<boolean> {
|
||
try {
|
||
const headResponse = await fetch(imageUrl, {
|
||
method: "HEAD",
|
||
signal: AbortSignal.timeout(10_000),
|
||
});
|
||
|
||
if (headResponse.ok) {
|
||
return true;
|
||
}
|
||
|
||
if (headResponse.status !== 405 && headResponse.status !== 403) {
|
||
return false;
|
||
}
|
||
|
||
const probeResponse = await fetch(imageUrl, {
|
||
method: "GET",
|
||
headers: {
|
||
Range: "bytes=0-0",
|
||
},
|
||
signal: AbortSignal.timeout(10_000),
|
||
});
|
||
return probeResponse.ok;
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
export async function ensureHostedImageUrl(imageUrl: string): Promise<string> {
|
||
const normalizedUrl = imageUrl.trim();
|
||
if (!normalizedUrl) {
|
||
throw new Error("Image URL is empty");
|
||
}
|
||
|
||
if (normalizedUrl.startsWith("data:image")) {
|
||
return uploadDataUrlImage(normalizedUrl);
|
||
}
|
||
|
||
if (!/^https?:\/\//i.test(normalizedUrl)) {
|
||
throw new Error(`Unsupported image URL: ${normalizedUrl}`);
|
||
}
|
||
|
||
if (/\/uploads\//i.test(normalizedUrl)) {
|
||
const accessible = await isRemoteImageAccessible(normalizedUrl);
|
||
if (accessible) {
|
||
return normalizedUrl;
|
||
}
|
||
}
|
||
|
||
return uploadRemoteImageToStorage(normalizedUrl);
|
||
}
|
||
|
||
export async function uploadDataUrlImage(dataUrl: string): Promise<string> {
|
||
const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i);
|
||
if (!match) {
|
||
throw new Error("Invalid data URL image format");
|
||
}
|
||
|
||
const mimeType = match[1] || "image/jpeg";
|
||
const base64Data = (match[2] || "").replace(/\s+/g, "");
|
||
if (!base64Data) {
|
||
throw new Error("Empty base64 image content");
|
||
}
|
||
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
const fileExt = mimeTypeToExtension(mimeType);
|
||
const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`;
|
||
const arrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
export async function normalizeAssistantImageMarkdown(
|
||
text: string,
|
||
imageSize = "2K"
|
||
): Promise<string> {
|
||
if (!text) return text;
|
||
|
||
const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim;
|
||
const markdownMatches = [...text.matchAll(markdownDataUrlRegex)];
|
||
let normalizedText = text;
|
||
const uploadedUrls: string[] = [];
|
||
|
||
if (markdownMatches.length > 0) {
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of markdownMatches) {
|
||
const matchText = match[0];
|
||
const dataUrl = match[2];
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += normalizedText.slice(cursor, start);
|
||
|
||
try {
|
||
const uploadedUrl = await uploadDataUrlImage(dataUrl);
|
||
uploadedUrls.push(uploadedUrl);
|
||
rebuilt += ``;
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Failed to upload inline data URL image:", error);
|
||
rebuilt += matchText;
|
||
}
|
||
|
||
cursor = start + matchText.length;
|
||
}
|
||
|
||
rebuilt += normalizedText.slice(cursor);
|
||
normalizedText = rebuilt;
|
||
}
|
||
|
||
if (normalizedText.includes("data:image")) {
|
||
const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim;
|
||
const rawMatches = normalizedText.match(rawDataUrlRegex) ?? [];
|
||
|
||
for (const rawDataUrl of rawMatches) {
|
||
try {
|
||
const uploadedUrl = await uploadDataUrlImage(rawDataUrl);
|
||
uploadedUrls.push(uploadedUrl);
|
||
normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl);
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Failed to normalize raw data URL image:", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim;
|
||
const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)];
|
||
|
||
if (markdownHttpMatches.length > 0) {
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of markdownHttpMatches) {
|
||
const matchText = match[0];
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = (match[2] || "").trim();
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += normalizedText.slice(cursor, start);
|
||
|
||
try {
|
||
const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata(
|
||
altText,
|
||
thumbnailUrl
|
||
);
|
||
|
||
let hostedThumbnailUrl = thumbnailUrl;
|
||
let hostedOriginalUrl =
|
||
originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl;
|
||
|
||
try {
|
||
hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl);
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Failed to normalize remote thumbnail URL:", error);
|
||
}
|
||
|
||
if (originalUrl && originalUrl !== thumbnailUrl) {
|
||
try {
|
||
hostedOriginalUrl = await ensureHostedImageUrl(originalUrl);
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Failed to normalize remote original URL:", error);
|
||
hostedOriginalUrl = hostedThumbnailUrl || originalUrl;
|
||
}
|
||
} else {
|
||
hostedOriginalUrl = hostedThumbnailUrl;
|
||
}
|
||
|
||
const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl;
|
||
const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl;
|
||
|
||
uploadedUrls.push(finalThumbnailUrl);
|
||
uploadedUrls.push(finalOriginalUrl);
|
||
rebuilt += ``;
|
||
} catch (error) {
|
||
chatApiLogger.error("[Chat API] Failed to parse remote image markdown metadata:", error);
|
||
rebuilt += matchText;
|
||
}
|
||
|
||
cursor = start + matchText.length;
|
||
}
|
||
|
||
rebuilt += normalizedText.slice(cursor);
|
||
normalizedText = rebuilt;
|
||
}
|
||
|
||
const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText);
|
||
if (!hasMarkdownImage && uploadedUrls.length > 0) {
|
||
const uniqueUrls = [...new Set(uploadedUrls)];
|
||
chatApiLogger.info(`[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)`);
|
||
return uniqueUrls
|
||
.map((url) => ``)
|
||
.join("\n");
|
||
}
|
||
|
||
return normalizedText;
|
||
}
|