import { geolocation } from "@vercel/functions"; import { convertToModelMessages, createUIMessageStream, generateText, JsonToSseTransformStream, smoothStream, stepCountIs, streamText, } from "ai"; import { unstable_cache as cache } from "next/cache"; import { after, NextResponse } from "next/server"; import { createResumableStreamContext, type ResumableStreamContext, } from "resumable-stream"; import type { ModelCatalog } from "tokenlens/core"; import { fetchModels } from "tokenlens/fetch"; import { getUsage } from "tokenlens/helpers"; import { auth, type UserType } from "@/app/(auth)/auth"; import type { VisibilityType } from "@/components/visibility-selector"; import { entitlementsByUserType } from "@/lib/ai/entitlements"; import type { ChatModel } from "@/lib/ai/models"; import { type RequestHints, systemPrompt } from "@/lib/ai/prompts"; import { myProvider } from "@/lib/ai/providers"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { createDocument } from "@/lib/ai/tools/create-document"; import { getWeather } from "@/lib/ai/tools/get-weather"; import { requestSuggestions } from "@/lib/ai/tools/request-suggestions"; import { updateDocument } from "@/lib/ai/tools/update-document"; import { generateVideo } from "@/lib/ai/tools/generate-video"; import { getEnabledTools } from "@/lib/ai/tools-registry"; import { isProductionEnvironment } from "@/lib/constants"; import { createStreamId, deleteChatById, getChatById, getMessageCountByUserId, getMessagesByChatId, saveChat, saveMessages, updateChatLastContextById, getAiModelById, getEnabledAiModelsWithProvider, getSubModelsByParentModelId, getAiProviderById, checkUserImageQuota, } from "@/lib/db/queries"; import { db } from "@/lib/db"; import type { DBMessage } from "@/lib/db/schema"; import { generatedImage } from "@/lib/db/schema"; import { ChatSDKError } from "@/lib/errors"; import { uploadFile } from "@/lib/storage/service"; import type { ChatMessage } from "@/lib/types"; import type { AppUsage } from "@/lib/usage"; import { convertToUIMessages, generateUUID } from "@/lib/utils"; import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags"; import { generateTitleFromUserMessage } from "../../actions"; import { type PostRequestBody, postRequestBodySchema } from "./schema"; /** * 将图片 URL 转换为 base64 格式 * 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API */ async function convertImageUrlToBase64(url: string): Promise { try { console.log("[Image Convert] Fetching image from:", url); const response = await fetch(url); if (!response.ok) { throw new Error(`Failed to fetch image: ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const base64 = buffer.toString("base64"); // 获取 MIME 类型 const contentType = response.headers.get("content-type") || "image/jpeg"; const dataUrl = `data:${contentType};base64,${base64}`; console.log("[Image Convert] Converted to base64, size:", base64.length); return dataUrl; } catch (error) { console.error("[Image Convert] Failed:", error); throw error; } } /** * 检测消息中是否包含用户标注过的图片(文件名以 annotated- 开头) */ function hasAnnotatedImage(messages: ChatMessage[]): boolean { for (const message of messages) { for (const part of message.parts) { if (part.type === "file" && "name" in part && typeof part.name === "string") { if (part.name.startsWith("annotated-")) { return true; } } } } return false; } /** * 标注图片的系统提示词(用户不可见,后台自动添加) */ const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条(如箭头、圆圈、画笔痕迹或文字说明),这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义,按照标注指示进行编辑,但在生成的新图片中不要保留这些标注线条,输出干净的编辑结果。"; function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: string): ChatMessage[] { if (!hiddenPrompt || !hiddenPrompt.trim()) return messages; const cloned = messages.map((msg) => ({ ...msg, parts: msg.parts.map((part) => ({ ...part })), })); for (let i = cloned.length - 1; i >= 0; i--) { const msg = cloned[i]; if (msg.role !== "user") continue; const textPart = msg.parts.find((part) => part.type === "text") as { type: "text"; text: string } | undefined; if (textPart) { textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim(); } else { msg.parts.unshift({ type: "text", text: hiddenPrompt }); } break; } return cloned; } function applyOptimizedPromptToLatestUserMessage( messages: ChatMessage[], optimizedPrompt?: string, hiddenPromptPrefix = "", ): ChatMessage[] { const prompt = optimizedPrompt?.trim(); if (!prompt) return messages; const cloned = messages.map((msg) => ({ ...msg, parts: msg.parts.map((part) => ({ ...part })), })); for (let i = cloned.length - 1; i >= 0; i -= 1) { const msg = cloned[i]; if (msg.role !== "user") continue; const textPart = msg.parts.find( (part): part is { type: "text"; text: string } => part.type === "text", ); const originalText = textPart?.text?.trim() || ""; const normalizedOriginal = hiddenPromptPrefix && originalText.startsWith(hiddenPromptPrefix) ? originalText.slice(hiddenPromptPrefix.length).trim() : originalText; const hasOriginalDemand = normalizedOriginal.length > 0; const hasOriginalInOptimized = hasOriginalDemand && (prompt.includes(normalizedOriginal) || normalizedOriginal.includes(prompt)); const finalPrompt = hasOriginalDemand && !hasOriginalInOptimized ? [ hiddenPromptPrefix.trim(), `【核心目标(必须优先满足)】${normalizedOriginal}`, `【执行优化细节】${prompt}`, ] .filter(Boolean) .join("\n\n") .trim() : `${hiddenPromptPrefix}${prompt}`.trim(); console.log("[Prompt Merge] Final image prompt prepared:", { original: normalizedOriginal.substring(0, 120), optimized: prompt.substring(0, 120), merged: finalPrompt.substring(0, 180), forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized, }); if (textPart) { textPart.text = finalPrompt; } else { msg.parts.unshift({ type: "text", text: finalPrompt }); } break; } return cloned; } function guessImageMediaType(url: string): string { const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg"; if (cleanUrl.endsWith(".webp")) return "image/webp"; if (cleanUrl.endsWith(".gif")) return "image/gif"; if (cleanUrl.endsWith(".bmp")) return "image/bmp"; if (cleanUrl.endsWith(".svg")) return "image/svg+xml"; return "image/png"; } function injectHistoryImagesToLastUserMessage( messages: ChatMessage[], historyImages: string[] ): ChatMessage[] { if (!historyImages || historyImages.length === 0) return messages; let lastUserIndex = -1; for (let i = messages.length - 1; i >= 0; i--) { if (messages[i]?.role === "user") { lastUserIndex = i; break; } } if (lastUserIndex < 0) return messages; const lastUserMessage = messages[lastUserIndex]; const hasExistingImage = lastUserMessage.parts?.some( (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" ); if (hasExistingImage) return messages; const imageParts = historyImages.map((url, index) => ({ type: "file" as const, url, name: `history-image-${index + 1}`, mediaType: guessImageMediaType(url), mimeType: guessImageMediaType(url), })); const updatedMessage: ChatMessage = { ...lastUserMessage, parts: [...lastUserMessage.parts, ...imageParts], }; return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg)); } function compactAutoDemand(text: string, maxLength = 56): string { const normalized = text.replace(/\s+/g, " ").trim(); if (!normalized) return "按你刚才的描述执行"; if (normalized.length <= maxLength) return normalized; return `${normalized.slice(0, maxLength)}...`; } function buildAutoModeFallbackMessage(params: { text: string; hasImage: boolean; fromHistory: boolean; }): string { const demand = compactAutoDemand(params.text); if (params.hasImage) { const sourceLine = params.fromHistory ? "我会基于你上一轮图片继续处理。" : "我会基于你当前上传图片处理。"; return [ "收到,我先帮你梳理这次修改目标。", `- 需求理解:${demand}`, `- 执行基准:${sourceLine}`, "- 执行方式:优先只改你指定区域,主体、构图和风格尽量保持一致。", "- 继续迭代:生成后你可直接说“保留A,改B”,我会按最新要求继续优化。", ].join("\n"); } return [ "收到,我来按自动模式帮你生成。", `- 需求理解:${demand}`, "- 执行方式:先锁定主体和场景,再细化风格、材质与光影。", "- 继续迭代:你可以继续补充偏好(如构图、镜头、色调、质感)。", ].join("\n"); } /** * 自定义消息转换函数 * 注意:gemini-image 和 huanke-gemini 都需要 URL 格式(不是 Base64) * 因为这些 provider 内部会自行处理图片上传到 S3 */ async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) { if (providerType === "ruxa-image") { const convertedMessages = []; for (const message of messages) { const content: any[] = []; for (const part of message.parts) { const partAny = part as any; if (partAny.type === "text" && partAny.text) { content.push({ type: "text", text: partAny.text, }); continue; } const candidateUrl = partAny.type === "file" ? (partAny.url || partAny.data) : partAny.type === "image" ? (partAny.image || partAny.url) : partAny.type === "image_url" ? partAny.image_url?.url : undefined; if (!candidateUrl || typeof candidateUrl !== "string") continue; if (candidateUrl.startsWith("data:")) { throw new Error("RUXA 不支持 Base64 图片传输,请先上传图片后再编辑"); } if (!candidateUrl.startsWith("http")) continue; content.push({ type: "image_url", image_url: { url: candidateUrl, }, }); } convertedMessages.push({ role: message.role, content: content.length === 1 && content[0].type === "text" ? content[0].text : content, }); } return convertedMessages; } // 所有 provider 都使用 URL 格式传输图片 // gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL // 这里不再进行 Base64 转换,直接返回标准格式 if (providerType !== "gemini") { return convertToModelMessages(messages); } // 🆕 检测是否有标注图片 const hasAnnotation = hasAnnotatedImage(messages); if (hasAnnotation) { console.log("[Message Convert] 🎨 检测到标注图片,将自动添加系统提示"); } // 对于 Gemini,需要特殊处理图片 const convertedMessages = []; for (const message of messages) { const content: any[] = []; // 🆕 如果是用户消息且有标注图片,在文本前添加系统提示 let hasAddedAnnotationPrompt = false; // 处理消息的各个部分 for (const part of message.parts) { if (part.type === "text" && part.text) { let textContent = part.text; // 🆕 如果有标注图片且是用户消息,在用户文本前添加系统提示 if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) { textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求:" + textContent; hasAddedAnnotationPrompt = true; console.log("[Message Convert] 🎨 已注入标注提示词"); } content.push({ type: "text", text: textContent, }); } else if (part.type === "file" && part.url) { // 直接使用 URL 格式(不再转换为 Base64) // gemini-image 和 huanke-gemini provider 内部会处理图片上传 content.push({ type: "image_url", image_url: { url: part.url, }, }); console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "..."); } } // 🆕 如果有标注图片但消息中没有文本部分,添加一个纯提示文本 if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) { content.unshift({ type: "text", text: ANNOTATION_SYSTEM_PROMPT, }); console.log("[Message Convert] 🎨 已在消息开头注入标注提示词"); } convertedMessages.push({ role: message.role, content: content.length === 1 && content[0].type === "text" ? content[0].text : content, }); } return convertedMessages; } function parseImageAltMetadata(altText: string, thumbnailUrl: string): { originalUrl: string; imageSize: string; } { let originalUrl = thumbnailUrl; let imageSize = "2K"; // 格式1: thumb|||originalUrl|||size (新格式) if (altText.includes("thumb|||")) { const content = altText.substring(altText.indexOf("thumb|||") + 8); const parts = content.split("|||"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 格式2: thumb$$originalUrl$$size (中间格式,可能被 LaTeX 解析器破坏) else if (altText.includes("thumb$$")) { const content = altText.substring(altText.indexOf("thumb$$") + 7); const parts = content.split("$$"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 格式3: thumb|originalUrl|size (旧格式) else if (altText.includes("thumb|")) { const content = altText.substring(altText.indexOf("thumb|") + 6); const parts = content.split("|"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 回退策略:从缩略图 URL 推断原图 URL if (originalUrl === thumbnailUrl) { if (thumbnailUrl.includes("gemini-thumb-")) { originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-"); } else if (thumbnailUrl.includes("openai-thumb-")) { originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-"); } else if (thumbnailUrl.includes("volcano-thumb-")) { originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-"); } else if (thumbnailUrl.includes("alibaba-image-thumb-")) { originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-"); } else if (thumbnailUrl.includes("ruxa-image-thumb-")) { originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-"); } } return { originalUrl, imageSize }; } function extractOriginalImageUrlsFromText(text: string): string[] { const imageRegex = /!\[([^\]]*)\]\((https?:\/\/[^\s)]+)\)/g; const imageUrls: string[] = []; const added = new Set(); let hasMarkdownImages = false; let match: RegExpExecArray | null; while ((match = imageRegex.exec(text)) !== null) { hasMarkdownImages = true; const altText = match[1] || ""; const thumbnailUrl = match[2]; const { originalUrl } = parseImageAltMetadata(altText, thumbnailUrl); const finalUrl = originalUrl || thumbnailUrl; if (finalUrl && /^https?:\/\//i.test(finalUrl) && !added.has(finalUrl)) { added.add(finalUrl); imageUrls.push(finalUrl); } } if (!hasMarkdownImages) { const urlRegex = /https?:\/\/[^\s"'<>]+\.(?:jpg|jpeg|png|gif|webp|bmp|svg)(?:\?[^\s"'<>]*)?/gi; const matchedUrls = text.match(urlRegex) || []; for (const url of matchedUrls) { if (!added.has(url)) { added.add(url); imageUrls.push(url); } } } return imageUrls; } function mimeTypeToExtension(mimeType: string) { const normalized = mimeType.toLowerCase(); if (normalized.includes("png")) return "png"; if (normalized.includes("webp")) return "webp"; if (normalized.includes("gif")) return "gif"; if (normalized.includes("bmp")) return "bmp"; return "jpg"; } function getExtensionFromUrl(url: string): string { try { const pathname = new URL(url).pathname; const ext = pathname.split(".").pop()?.toLowerCase(); if (!ext) return "jpg"; if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext; } catch { // ignore } return "jpg"; } async function uploadRemoteImageToStorage(imageUrl: string): Promise { const response = await fetch(imageUrl, { method: "GET", signal: AbortSignal.timeout(30_000), }); if (!response.ok) { throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`); } const contentType = response.headers.get("content-type")?.toLowerCase() || ""; if (contentType && !contentType.startsWith("image/")) { throw new Error(`Remote url is not an image: ${contentType}`); } const arrayBuffer = await response.arrayBuffer(); if (!arrayBuffer || arrayBuffer.byteLength === 0) { throw new Error("Remote image content is empty"); } const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl); const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`; const uploaded = await uploadFile(fileName, arrayBuffer); return uploaded.url; } async function isRemoteImageAccessible(imageUrl: string): Promise { try { const headResponse = await fetch(imageUrl, { method: "HEAD", signal: AbortSignal.timeout(10_000), }); if (headResponse.ok) { return true; } if (headResponse.status !== 405 && headResponse.status !== 403) { return false; } const probeResponse = await fetch(imageUrl, { method: "GET", headers: { Range: "bytes=0-0", }, signal: AbortSignal.timeout(10_000), }); return probeResponse.ok; } catch { return false; } } async function ensureHostedImageUrl(imageUrl: string): Promise { const normalizedUrl = imageUrl.trim(); if (!normalizedUrl) { throw new Error("Image URL is empty"); } if (normalizedUrl.startsWith("data:image")) { return uploadDataUrlImage(normalizedUrl); } if (!/^https?:\/\//i.test(normalizedUrl)) { throw new Error(`Unsupported image URL: ${normalizedUrl}`); } if (/\/uploads\//i.test(normalizedUrl)) { const accessible = await isRemoteImageAccessible(normalizedUrl); if (accessible) { return normalizedUrl; } } return uploadRemoteImageToStorage(normalizedUrl); } async function uploadDataUrlImage(dataUrl: string): Promise { const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i); if (!match) { throw new Error("Invalid data URL image format"); } const mimeType = match[1] || "image/jpeg"; const base64Data = (match[2] || "").replace(/\s+/g, ""); if (!base64Data) { throw new Error("Empty base64 image content"); } const buffer = Buffer.from(base64Data, "base64"); const fileExt = mimeTypeToExtension(mimeType); const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`; const arrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; const uploaded = await uploadFile(fileName, arrayBuffer); return uploaded.url; } async function normalizeAssistantImageMarkdown( text: string, imageSize = "2K" ): Promise { if (!text) return text; const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim; const markdownMatches = [...text.matchAll(markdownDataUrlRegex)]; let normalizedText = text; const uploadedUrls: string[] = []; if (markdownMatches.length > 0) { let rebuilt = ""; let cursor = 0; for (const match of markdownMatches) { const matchText = match[0]; const dataUrl = match[2]; const start = match.index ?? -1; if (start < 0) continue; rebuilt += normalizedText.slice(cursor, start); try { const uploadedUrl = await uploadDataUrlImage(dataUrl); uploadedUrls.push(uploadedUrl); rebuilt += `![thumb|||${uploadedUrl}|||${imageSize}](${uploadedUrl})`; } catch (error) { console.error("[Chat API] Failed to upload inline data URL image:", error); rebuilt += matchText; } cursor = start + matchText.length; } rebuilt += normalizedText.slice(cursor); normalizedText = rebuilt; } if (normalizedText.includes("data:image")) { const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim; const rawMatches = normalizedText.match(rawDataUrlRegex) ?? []; for (const rawDataUrl of rawMatches) { try { const uploadedUrl = await uploadDataUrlImage(rawDataUrl); uploadedUrls.push(uploadedUrl); normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl); } catch (error) { console.error("[Chat API] Failed to normalize raw data URL image:", error); } } } const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim; const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)]; if (markdownHttpMatches.length > 0) { let rebuilt = ""; let cursor = 0; for (const match of markdownHttpMatches) { const matchText = match[0]; const altText = match[1] || ""; const thumbnailUrl = (match[2] || "").trim(); const start = match.index ?? -1; if (start < 0) continue; rebuilt += normalizedText.slice(cursor, start); try { const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata( altText, thumbnailUrl ); const hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl); const hostedOriginalUrl = originalUrl && originalUrl !== thumbnailUrl ? await ensureHostedImageUrl(originalUrl) : hostedThumbnailUrl; uploadedUrls.push(hostedThumbnailUrl); uploadedUrls.push(hostedOriginalUrl); rebuilt += `![thumb|||${hostedOriginalUrl}|||${parsedImageSize || imageSize}](${hostedThumbnailUrl})`; } catch (error) { console.error("[Chat API] Failed to normalize remote image URL:", error); rebuilt += ""; } cursor = start + matchText.length; } rebuilt += normalizedText.slice(cursor); normalizedText = rebuilt; } const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText); if (!hasMarkdownImage && uploadedUrls.length > 0) { const uniqueUrls = [...new Set(uploadedUrls)]; console.log( `[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)` ); return uniqueUrls .map((url) => `![thumb|||${url}|||${imageSize}](${url})`) .join("\n"); } return normalizedText; } /** * 解析生图 Markdown 并保存图片记录到数据库 * 支持三种格式:thumb|||url|||size、thumb$$url$$size、thumb|url|size */ async function saveGeneratedImages( imageMarkdown: string, userId: string, chatId: string, messageId: string, prompt: string, modelId: string ): Promise { // 匹配 Markdown 图片格式: ![alt](url) const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; const matches = [...imageMarkdown.matchAll(imageRegex)]; for (const match of matches) { const altText = match[1] || ""; const thumbnailUrl = match[2]; const { originalUrl, imageSize } = parseImageAltMetadata(altText, thumbnailUrl); try { await db.insert(generatedImage).values({ userId, chatId, messageId, thumbnailUrl, originalUrl, imageSize, prompt, modelId, }); console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`); } catch (error) { console.error("[Gallery] Failed to save image record:", error); } } } export const maxDuration = 600; // 10 分钟,用于支持图片编辑等耗时任务 let globalStreamContext: ResumableStreamContext | null = null; const getTokenlensCatalog = cache( async (): Promise => { try { return await fetchModels(); } catch (err) { console.warn( "TokenLens: catalog fetch failed, using default catalog", err ); return; // tokenlens helpers will fall back to defaultCatalog } }, ["tokenlens-catalog"], { revalidate: 24 * 60 * 60 } // 24 hours ); export function getStreamContext() { if (!globalStreamContext) { try { globalStreamContext = createResumableStreamContext({ waitUntil: after, }); } catch (error: any) { if (error.message.includes("REDIS_URL")) { console.log( " > Resumable streams are disabled due to missing REDIS_URL" ); } else { console.error(error); } } } return globalStreamContext; } export async function POST(request: Request) { let requestBody: PostRequestBody; try { const json = await request.json(); console.log("[Chat API] Received request body:", JSON.stringify(json, null, 2)); requestBody = postRequestBodySchema.parse(json); } catch (error) { console.error("[Chat API] Schema validation failed:", error); return new ChatSDKError("bad_request:api").toResponse(); } try { const { id, message, selectedChatModel, selectedVisibilityType, hiddenPrompt, autoContext, }: { id: string; message: ChatMessage; selectedChatModel: ChatModel["id"]; selectedVisibilityType: VisibilityType; hiddenPrompt?: string; autoContext?: string; } = requestBody; let finalModelId = selectedChatModel; // 🆕 存储智能模式的相关信息 let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复 let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词(不修改用户消息,只用于API调用) let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL(支持多张) let smartAutoNeedsClarification = false; // 是否需要先追问用户 let smartAutoClarificationQuestion: string | undefined; let shouldRunAgentConversation = finalModelId === "auto"; let shouldKeepSelectedModel = false; const requestedModelId = selectedChatModel; if (!shouldRunAgentConversation && requestedModelId !== "auto") { const selectedModelIsUUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( requestedModelId ); if (selectedModelIsUUID) { const selectedModelConfig = await getAiModelById({ id: requestedModelId }); if (selectedModelConfig && isAiModelAgentAuto(selectedModelConfig)) { shouldRunAgentConversation = true; shouldKeepSelectedModel = false; console.log( `[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}` ); } else if (selectedModelConfig?.supportsImageGen && !selectedModelConfig?.isSubModel) { shouldRunAgentConversation = true; shouldKeepSelectedModel = true; console.log( `[Smart Auto Mode] 🎯 Agent conversation enabled for fixed image model: ${selectedModelConfig.displayName}` ); } } } // ========== Smart Auto Mode Logic ========== if (shouldRunAgentConversation) { console.log("[Smart Auto Mode] 🚀 Starting intelligent analysis..."); let autoTextForFallback = ""; let autoHasImageForFallback = false; let autoFromHistoryForFallback = false; try { // 🆕 首先获取历史消息(在 Smart Auto Mode 内部单独获取,避免依赖后面的 messagesFromDb) const smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id }); console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB"); // 1. 提取用户消息内容 const lastUserMessage = message; const textContent = lastUserMessage.parts .filter((p): p is { type: "text"; text: string } => p.type === "text") .map((p) => p.text) .join(" "); autoTextForFallback = textContent; const autoContextText = typeof autoContext === "string" ? autoContext.trim() : ""; // @ts-ignore - 检查各种可能的图片类型 let hasImage = lastUserMessage.parts.some( (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" ); autoHasImageForFallback = hasImage; // @ts-ignore - 提取图片URL let imageUrls = lastUserMessage.parts .filter((p: any) => p.type === "file" && p.url) .map((p: any) => p.url as string); // 🆕 历史图片上下文感知:如果用户没有发送新图片,自动使用历史中最近的图片(支持多张) // 逻辑:只要用户在对话中没有附上新图片,就默认继续处理之前的图片 if (!hasImage && smartAutoHistoryMessagesFromDb.length > 0) { console.log("[Smart Auto Mode] 🔍 No image in current message, auto-searching history..."); // 从历史消息中倒序查找最近一轮包含图片的消息 // 收集该消息中的所有图片(支持多图融合编辑) for (let i = smartAutoHistoryMessagesFromDb.length - 1; i >= 0; i--) { const msg = smartAutoHistoryMessagesFromDb[i]; const parts = msg.parts as any[]; if (!parts) continue; // 查找AI生成的图片(从Markdown中提取所有图片) if (msg.role === "assistant") { for (const part of parts) { if (part.type === "text" && part.text) { // 从 markdown 中优先提取原图 URL,避免把缩略图当作下一轮输入导致画质下降 const parsedImageUrls = extractOriginalImageUrlsFromText(part.text); for (const parsedUrl of parsedImageUrls) { if (!smartAutoHistoryImages.includes(parsedUrl)) { smartAutoHistoryImages.push(parsedUrl); } } } } // 如果在这条assistant消息中找到了图片,就停止搜索(使用最近一轮的结果) if (smartAutoHistoryImages.length > 0) { console.log("[Smart Auto Mode] 🖼️ Found AI-generated images:", smartAutoHistoryImages.length); break; } } // 查找用户上传的图片(收集该消息中的所有图片) if (msg.role === "user" && smartAutoHistoryImages.length === 0) { for (const part of parts) { if ((part.type === "file" || part.type === "image") && part.url) { if (!smartAutoHistoryImages.includes(part.url)) { smartAutoHistoryImages.push(part.url); } } } // 如果在这条user消息中找到了图片,就停止搜索 if (smartAutoHistoryImages.length > 0) { console.log("[Smart Auto Mode] 🖼️ Found user-uploaded images:", smartAutoHistoryImages.length); break; } } } // 如果找到了历史图片,标记为有图片 if (smartAutoHistoryImages.length > 0) { hasImage = true; imageUrls = smartAutoHistoryImages; autoHasImageForFallback = true; autoFromHistoryForFallback = true; console.log("[Smart Auto Mode] ✅ Auto-using", smartAutoHistoryImages.length, "history image(s) for context continuity"); } else { console.log("[Smart Auto Mode] ℹ️ No image found in history, proceeding as pure text"); } } console.log("[Smart Auto Mode] Input:", { textContent: textContent.substring(0, 100), hasImage, imageCount: imageUrls.length, fromHistory: smartAutoHistoryImages.length > 0 }); // 🆕 从历史消息中提取对话文本(用于上下文理解) const historyMessages: { role: "user" | "assistant"; content: string }[] = []; for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条 const parts = msg.parts as any[]; if (parts && parts.length > 0) { // 提取文本内容 const textParts = parts.filter((p: any) => p.type === 'text'); const content = textParts.map((p: any) => p.text || '').join(' ').trim(); if (content) { historyMessages.push({ role: msg.role as "user" | "assistant", content: content.substring(0, 300), // 限制长度 }); } } } console.log("[Smart Auto Mode] History context:", historyMessages.length, "messages"); // 2. 使用 LangGraph 驱动的自动模式工作流 const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph"); const autoGraphResult = await runAutoAgentGraph({ text: textContent, hasImage, imageUrls, imageFromHistory: smartAutoHistoryImages.length > 0, historyMessages, context: autoContextText || undefined, }); if (autoGraphResult.analysis) { console.log("[Smart Auto Mode] Analysis result:", { intent: autoGraphResult.analysis.intent, subIntent: autoGraphResult.analysis.subIntent, confidence: autoGraphResult.analysis.confidence, needsClarification: autoGraphResult.analysis.needsClarification, }); smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification); smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0]; } if (autoGraphResult.friendlyMessage) { smartAutoFriendlyMessage = autoGraphResult.friendlyMessage; console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage); } const autoIntent = autoGraphResult.analysis?.intent; const isImageTask = autoIntent === "image_gen" || autoIntent === "image_edit" || hasImage; const needsRicherAutoReply = isImageTask && !smartAutoNeedsClarification && !smartAutoFriendlyMessage; if (needsRicherAutoReply) { if (isImageTask) { smartAutoFriendlyMessage = autoGraphResult.analysis?.conversationalReply || "我先分析你这轮图片里的主体、风格和构图,再给你执行方案。"; } else { smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage, fromHistory: smartAutoHistoryImages.length > 0, }); } console.log("[Smart Auto Mode] 🧩 Filled missing conversational reply"); } if (autoGraphResult.optimizedPrompt) { smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt; console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", { original: textContent.substring(0, 50), optimized: autoGraphResult.optimizedPrompt.substring(0, 100), }); } if (autoGraphResult.route && !shouldKeepSelectedModel) { finalModelId = autoGraphResult.route.id; console.log(`[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`); } else if (autoGraphResult.route && shouldKeepSelectedModel) { finalModelId = requestedModelId; console.log("[Smart Auto Mode] 🔒 Keep user-selected model, skip routing override"); } } catch (autoError) { // 智能分析失败时,回退到简单逻辑 console.error("[Smart Auto Mode] ❌ Error, falling back:", autoError); try { const { analyzeIntent } = await import("@/lib/ai/smart-analyzer"); const rescueAnalysis = await analyzeIntent({ text: autoTextForFallback, hasImage: autoHasImageForFallback || smartAutoHistoryImages.length > 0, imageUrls: smartAutoHistoryImages, imageFromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, historyMessages: [], context: undefined, }); if (rescueAnalysis.conversationalReply?.trim()) { smartAutoFriendlyMessage = rescueAnalysis.conversationalReply.trim(); } } catch (rescueError) { console.error("[Smart Auto Mode] Rescue analysis failed:", rescueError); } const allModels = await getEnabledAiModelsWithProvider(); // @ts-ignore - 检查各种可能的图片类型 const hasImage = autoHasImageForFallback || message.parts.some( (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" ); // 简单回退:有图选图片模型,无图选默认模型 let bestModel = null; if (hasImage) { bestModel = allModels.find(m => m.model.supportsImageGen && !m.model.isSubModel); smartAutoFriendlyMessage = smartAutoFriendlyMessage || "我正在先做这一轮图片分析,接着按你的要求执行修改。"; } else { smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage: false, fromHistory: false, }); } if (!bestModel) { bestModel = allModels.find(m => m.model.isDefault && !m.model.isSubModel); } if (!bestModel && allModels.length > 0) { bestModel = allModels.find(m => !m.model.isSubModel) || allModels[0]; } if (shouldKeepSelectedModel) { finalModelId = requestedModelId; console.log("[Smart Auto Mode] 🔒 Keep user-selected model in fallback mode"); } else if (bestModel) { finalModelId = bestModel.model.id; console.log(`[Smart Auto Mode] Fallback selected: ${bestModel.model.displayName}`); } else { return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse(); } } } // ========== End Smart Auto Mode Logic ========== const session = await auth(); if (!session?.user) { return new ChatSDKError("unauthorized:chat").toResponse(); } const userType: UserType = session.user.type; const messageCount = await getMessageCountByUserId({ id: session.user.id, differenceInHours: 24, }); if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) { return new ChatSDKError("rate_limit:chat").toResponse(); } // ========== 使用时间限制验证(后端安全检查) ========== try { // 从数据库获取网站设置 const { siteSettings } = await import("@/lib/db/schema"); const [settings] = await db.select().from(siteSettings).limit(1); if (settings?.usageTimeEnabled) { const now = new Date(); const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`; const startTime = settings.usageTimeStart || "08:00"; const endTime = settings.usageTimeEnd || "20:00"; const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime; if (!isWithinUsageTime) { console.log(`[Chat API] ⛔ 访问被拒绝:当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`); return NextResponse.json( { error: "usage_time_restricted", message: settings.usageTimeMessage || "当前不在系统使用时间段内", currentTime, allowedTime: `${startTime} - ${endTime}` }, { status: 403 } ); } console.log(`[Chat API] ✅ 时间验证通过:${currentTime} 在允许范围 ${startTime}-${endTime}`); } } catch (timeCheckError) { console.error("[Chat API] ⚠️ 时间验证失败(继续执行):", timeCheckError); // 如果时间检查失败,继续执行(避免因检查逻辑错误导致系统不可用) } const chat = await getChatById({ id }); let messagesFromDb: DBMessage[] = []; if (chat) { if (chat.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } // Only fetch messages if chat already exists messagesFromDb = await getMessagesByChatId({ id }); } else { const title = await generateTitleFromUserMessage({ message, }); await saveChat({ id, userId: session.user.id, title, visibility: selectedVisibilityType, }); // New chat - no need to fetch messages, it's empty } const uiMessages = [...convertToUIMessages(messagesFromDb), message]; const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : ""; const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : ""; const uiMessagesForModel = hiddenPromptText ? applyHiddenPromptToMessages(uiMessages, hiddenPromptText) : uiMessages; const { longitude, latitude, city, country } = geolocation(request); const requestHints: RequestHints = { longitude, latitude, city, country, }; await saveMessages({ messages: [ { chatId: id, id: message.id, role: "user", parts: message.parts, attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); const streamId = generateUUID(); await createStreamId({ streamId, chatId: id }); if (shouldRunAgentConversation && smartAutoNeedsClarification) { const clarificationMessage = smartAutoClarificationQuestion || smartAutoFriendlyMessage || "我先确认一下,你希望我优先处理哪一部分?"; const clarificationStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: clarificationMessage }], }), }); await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: clarificationMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); }, }); return new Response(clarificationStream.pipeThrough(new JsonToSseTransformStream())); } let finalMergedUsage: AppUsage | undefined; // 获取启用的工具列表 const enabledTools = await getEnabledTools(); // 检查是否是 UUID 格式(数据库中的模型 ID) const isUUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( finalModelId ); // 获取语言模型实例 let languageModel; let actualModelId: string; let resolvedModelSupportsImageGen = false; if (isUUID) { // 从数据库获取动态配置的模型 let targetModelId = finalModelId; // 🆕 检查是否是视频主模型,如果是,根据 videoParams.mode 选择子模型 const aiModelData = await getAiModelById({ id: finalModelId }); console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel }); if (aiModelData && !aiModelData.isSubModel) { // 检查是否有子模型 const subModels = await getSubModelsByParentModelId(finalModelId); console.log("[Chat API] Found sub-models:", subModels.length); if (subModels.length > 0) { // 这是一个主模型,需要根据 videoParams 选择子模型 const videoParams = requestBody.videoParams; console.log("[Chat API] Video params:", videoParams); // 如果没有 videoParams 或没有 mode,使用默认模式 const mode = videoParams?.mode || "text-to-video"; console.log("[Chat API] Using mode:", mode); // 根据模式选择对应的子模型 const modeToModelMap: Record = { "text-to-video": "lite-t2v", // 文生视频 → Lite T2V "first-last-frame": "pro", // 首尾帧 → Pro "first-frame": "pro-fast", // 智能多帧 → Pro Fast "reference-images": "lite-i2v", // 主体参考 → Lite I2V }; const targetSubModelKey = modeToModelMap[mode]; if (targetSubModelKey) { // 查找匹配的子模型(通过 modelId 中包含的关键字) const matchedSubModel = subModels.find(({ model }) => model.modelId.toLowerCase().includes(targetSubModelKey) ); if (matchedSubModel) { console.log(`[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName}`); targetModelId = matchedSubModel.model.id; } else { console.warn(`[Chat API] 未找到匹配的子模型: ${targetSubModelKey},将使用第一个可用子模型`); // 如果没找到匹配的,使用第一个子模型作为后备 targetModelId = subModels[0].model.id; } } } } const dynamicModel = await getLanguageModelById(targetModelId); if (!dynamicModel) { return new ChatSDKError( "bad_request:api", "Model not found or disabled" ).toResponse(); } languageModel = dynamicModel; // 对于动态模型,我们需要从数据库获取 modelId const finalModelData = await getAiModelById({ id: targetModelId }); actualModelId = finalModelData?.modelId ?? targetModelId; resolvedModelSupportsImageGen = Boolean(finalModelData?.supportsImageGen); } else { // 使用预定义的模型 languageModel = myProvider.languageModel(finalModelId); actualModelId = languageModel.modelId; } // ========== 图片生成配额检查 ========== const isImageGenerationModel = (languageModel as any).provider === "gemini-image" || (languageModel as any).provider === "volcano-vision" || (languageModel as any).provider === "alibaba-image" || (languageModel as any).provider === "ruxa-image" || (languageModel as any).provider === "huanke-gemini"; if (isImageGenerationModel) { const quota = await checkUserImageQuota(session.user.id); if (!quota.allowed) { console.log(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`); return NextResponse.json( { error: "quota_exceeded", message: `本月图片生成配额已用完(已使用 ${quota.used}/${quota.limit} 张),配额将于下月1日重置,如需更多配额请联系管理员。`, used: quota.used, limit: quota.limit, }, { status: 429 } ); } console.log(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`); } // ========== Gemini 图片生成独立处理 ========== // 支持 gemini-image 和 huanke-gemini 两种图片生成 provider const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider); if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", requestBody.imageParams); // 🆕 获取备用模型配置 const currentModelConfig = await getAiModelById({ id: finalModelId }); const configuredFallbacks = [ currentModelConfig?.fallbackModelId1, currentModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const sameProviderFallbacks = currentModelConfig ? allModels .filter(m => m.provider.id === currentModelConfig.providerId && m.model.supportsImageGen && m.model.isEnabled && !m.model.isSubModel && m.model.modelId !== actualModelId ) .map(m => m.model.modelId) : []; const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks])); console.log("[Chat API] Fallback model IDs:", fallbackModelIds); const geminiStream = createUIMessageStream({ execute: async ({ writer }) => { // 🆕 如果是智能模式且有对话回复,先发送对话消息让用户看到 // 在消息末尾加上特殊标记,告诉前端"图片正在生成中" const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: chatMessageId, role: "assistant", parts: [{ type: "text", text: conversationMessage }], }), }); // 保存对话消息到数据库(不带标记,纯对话) await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 🆕 尝试生成图片的函数(支持备用模型切换) const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise => { console.log(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`); // 只传递最后一条用户消息,避免上下文历史干扰生图效果 const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop(); let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel; messagesForGemini = applyOptimizedPromptToLatestUserMessage( messagesForGemini, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages); console.log("[Chat API] Gemini: Using only last user message (no history context)"); // 转换消息:gemini-image 需要 base64 格式,huanke-gemini 需要 URL 格式 const providerForConvert = (languageModel as any).provider === "huanke-gemini" ? "huanke-gemini" : "gemini"; const modelMessages = await convertMessagesWithImages(messagesForGemini, providerForConvert); console.log("[Chat API] Message conversion provider:", providerForConvert); // 调用 doGenerate 获取图片(如果是备用模型,传入 overrideModelId) const result = await (languageModel as any).doGenerate({ messages: modelMessages, imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K" }, overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID }); return result.text || "图片生成失败"; }; try { let imageMarkdown = ""; // 初始化为空字符串 let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记(用于前端显示占位图) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); try { // 首先尝试主模型 imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Primary model failed:", primaryError); // 🆕 尝试备用模型(使用同一个 provider,但切换 modelId) let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Trying fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Fallback model ${fallbackModelId} failed:`, fallbackError); } } // 如果所有备用模型都失败,抛出原始错误触发火山引擎故障转移 if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Gemini generated, length:", imageMarkdown.length); console.log("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200)); console.log("[Chat API] Used model ID:", usedModelId); // 生成一个统一的消息 ID const messageId = generateUUID(); // 写入消息到流 - 使用正确的 parts 格式 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: imageMarkdown }], }), }); // 🆕 图片生成结束标记(成功) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); // 保存消息到数据库(现在是 S3 URL,可以完整保存) try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdown }], attachments: [], // 必须提供 attachments 字段(数据库 notNull) createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Gemini message saved successfully!"); // 提取用户 prompt(最后一条用户消息的文本) const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdown, session.user.id, id, messageId, userPrompt, actualModelId ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError); console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300)); } } catch (error) { console.error("[Chat API] Gemini generation error:", error); console.log("[Chat API] Attempting failover to Volcano Vision..."); // 故障转移:尝试使用火山引擎模型 try { // 查找火山引擎模型 const allModels = await getEnabledAiModelsWithProvider(); console.log("[Chat API] Total enabled models:", allModels.length); const volcanoModel = allModels.find( (m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled ); if (volcanoModel) { console.log("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id); // 获取火山引擎模型实例 const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id); if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") { console.log("[Chat API] Executing Volcano Vision failover..."); // 转换消息(火山引擎会自动处理 Base64) const volcanoMessages = await convertMessagesWithImages( injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages), "volcano" ); // 调用火山引擎生成(故障转移时只生成 1 张,保持与 Gemini 一致) const volcanoResult = await (volcanoLanguageModel as any).doGenerate({ messages: volcanoMessages, maxImages: 1, // 故障转移时只生成 1 张 }); const imageMarkdown = volcanoResult.text || "图片生成失败"; console.log("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length); // 生成消息 ID const messageId = generateUUID(); // 写入消息到流 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: imageMarkdown }], }), }); // 🆕 图片生成结束标记(故障转移成功) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); // 保存消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdown }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Volcano Vision failover message saved!"); // 提取用户 prompt const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdown, session.user.id, id, messageId, userPrompt, volcanoModel.model.modelId ); } catch (saveError) { console.error("[Chat API] Failed to save failover message:", saveError); } return; // 成功,直接返回 } } console.log("[Chat API] No Volcano Vision model available for failover"); } catch (failoverError) { console.error("[Chat API] Failover to Volcano Vision also failed:", failoverError); } // 如果故障转移也失败,返回错误消息 const errorMessageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: "图片生成失败,请稍后重试" }], }), }); // 🆕 图片生成结束标记(失败) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); // 尝试保存错误消息 try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: "图片生成失败,请稍后重试" }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } }, }); return new Response(geminiStream.pipeThrough(new JsonToSseTransformStream())); } // ========== Gemini 处理结束 ========== // ========== 阿里云千问图片生成独立处理 ========== const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image"].includes((languageModel as any).provider); if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using image generation provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", requestBody.imageParams); // 🆕 获取模型显示名称,用于错误提示 const alibabaModelConfig = await getAiModelById({ id: finalModelId }); const modelDisplayName = alibabaModelConfig?.displayName || actualModelId; const configuredFallbacks = [ alibabaModelConfig?.fallbackModelId1, alibabaModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const sameProviderFallbacks = alibabaModelConfig ? allModels .filter(m => m.provider.id === alibabaModelConfig.providerId && m.model.supportsImageGen && m.model.isEnabled && !m.model.isSubModel && m.model.modelId !== actualModelId ) .map(m => m.model.modelId) : []; const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks])); console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds); const alibabaStream = createUIMessageStream({ execute: async ({ writer }) => { try { // 🆕 如果是智能模式且有对话回复,先发送对话消息让用户看到 const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: chatMessageId, role: "assistant", parts: [{ type: "text", text: conversationMessage }], }), }); // 保存对话消息到数据库(不带标记,纯对话) await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 只传递最后一条用户消息 const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop(); let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessagesForModel; console.log("[Chat API] Alibaba: Using only last user message"); // 🆕 如果有智能模式优化后的提示词,替换消息中的文本(仅用于API调用) if (smartAutoOptimizedPrompt) { console.log("[Chat API] Using optimized prompt for API call"); messagesForAlibaba = applyOptimizedPromptToLatestUserMessage( messagesForAlibaba, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); } messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages); const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => { // 转换消息(阿里生图支持 URL,会自动上传 base64) const providerForConvert = (languageModel as any).provider === "ruxa-image" ? "ruxa-image" : "alibaba-image"; const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert); // 调用 doGenerate 获取图片,传递模型显示名称 const result = await (languageModel as any).doGenerate({ messages: modelMessages, imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 }, modelDisplayName, // 🆕 传递模型显示名称 overrideModelId: isRetry ? modelIdToUse : undefined, }); if (result?.finishReason === "error") { throw new Error(result.text || "图片生成失败"); } return result.text || "图片生成失败"; }; let imageMarkdown = ""; let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记(用于前端显示占位图) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); try { imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Alibaba primary model failed:", primaryError); let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Alibaba fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Alibaba fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Alibaba fallback model ${fallbackModelId} failed:`, fallbackError); } } if (!fallbackSuccess) { const providerPriority: Record = { "volcano-vision": 0, gemini: 1, "huanke-gemini": 2, "alibaba-image": 3, "ruxa-image": 4, }; const crossProviderCandidates = allModels .filter((m) => { if (!m.model.supportsImageGen || !m.model.isEnabled || m.model.isSubModel) { return false; } if (m.model.id === finalModelId) return false; if (m.provider.id === alibabaModelConfig?.providerId) return false; return true; }) .sort((a, b) => { const left = providerPriority[a.provider.type as string] ?? 99; const right = providerPriority[b.provider.type as string] ?? 99; if (left !== right) return left - right; return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0); }); for (const candidate of crossProviderCandidates) { try { console.log( `[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})` ); const candidateLanguageModel = await getLanguageModelById(candidate.model.id); if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") { console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`); continue; } const convertProvider = candidate.provider.type === "ruxa-image" ? "ruxa-image" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" ? "huanke-gemini" : candidate.provider.type === "volcano-vision" ? "volcano" : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForAlibaba, convertProvider ); const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 }, modelDisplayName: candidate.model.displayName, overrideModelId: candidate.model.modelId, }); if (candidateResult?.finishReason === "error") { throw new Error(candidateResult.text || "图片生成失败"); } imageMarkdown = candidateResult.text || "图片生成失败"; usedModelId = candidate.model.modelId; fallbackSuccess = true; console.log( `[Chat API] ✅ Cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})` ); break; } catch (crossError) { console.error( `[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError ); } } } if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Alibaba generated, length:", imageMarkdown.length); console.log("[Chat API] Alibaba used model:", usedModelId); // 生成消息 ID(图片消息) const messageId = generateUUID(); // 写入图片消息到流 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: imageMarkdown }], }), }); // 🆕 图片生成结束标记(成功) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); // 保存图片消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdown }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Alibaba message saved successfully!"); // 提取用户 prompt const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdown, session.user.id, id, messageId, userPrompt, usedModelId ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Alibaba message:", saveError); } } catch (error) { console.error("[Chat API] Alibaba generation error:", error); // 返回错误消息 const errorMessageId = generateUUID(); const errorText = error instanceof Error ? error.message : "图片生成失败,请稍后重试"; writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorText }], }), }); // 🆕 图片生成结束标记(失败) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } }, }); return new Response(alibabaStream.pipeThrough(new JsonToSseTransformStream())); } // ========== 阿里云图片生成处理结束 ========== // ========== 火山引擎视觉图片生成独立处理 ========== if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Volcano Vision image generation"); // 🆕 检测是否有标注图片,如果有则在消息中注入系统提示 const hasAnnotation = hasAnnotatedImage(uiMessages); let messagesForVolcano = applyOptimizedPromptToLatestUserMessage( uiMessagesForModel, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); const volcanoModelConfig = await getAiModelById({ id: finalModelId }); const configuredFallbacks = [ volcanoModelConfig?.fallbackModelId1, volcanoModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const sameProviderFallbacks = volcanoModelConfig ? allModels .filter(m => m.provider.id === volcanoModelConfig.providerId && m.model.supportsImageGen && m.model.isEnabled && !m.model.isSubModel && m.model.modelId !== actualModelId ) .map(m => m.model.modelId) : []; const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks])); console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds); if (hasAnnotation) { console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片,将注入系统提示"); // 创建消息副本,修改最后一条用户消息的文本 messagesForVolcano = messagesForVolcano.map((msg, idx) => { if (msg.role === "user" && idx === messagesForVolcano.length - 1) { // 修改最后一条用户消息 const modifiedParts = msg.parts.map((part) => { if (part.type === "text" && part.text) { return { ...part, text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求:" + part.text, }; } return part; }); return { ...msg, parts: modifiedParts }; } return msg; }); } messagesForVolcano = injectHistoryImagesToLastUserMessage(messagesForVolcano, smartAutoHistoryImages); const volcanoStream = createUIMessageStream({ execute: async ({ writer }) => { try { // 🆕 如果是智能模式且有对话回复,先发送对话消息让用户看到 const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: chatMessageId, role: "assistant", parts: [{ type: "text", text: conversationMessage }], }), }); // 保存对话消息到数据库(不带标记,纯对话) await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 转换消息,火山引擎支持 URL 和 base64 const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano"); const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => { const result = await (languageModel as any).doGenerate({ messages: modelMessages, overrideModelId: isRetry ? modelIdToUse : undefined, }); if (result?.finishReason === "error") { throw new Error(result.text || "图片生成失败"); } return result.text || "图片生成失败"; }; let imageMarkdown = ""; let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记(用于前端显示占位图) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); try { imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Volcano primary model failed:", primaryError); let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Volcano fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Volcano fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Volcano fallback model ${fallbackModelId} failed:`, fallbackError); } } if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Volcano Vision generated, length:", imageMarkdown.length); console.log("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200)); // 生成消息 ID(图片消息) const messageId = generateUUID(); // 写入图片消息到流 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: imageMarkdown }], }), }); // 🆕 图片生成结束标记(成功) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); // 保存图片消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdown }], attachments: [], // 必须提供 attachments 字段(数据库 notNull) createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Volcano Vision message saved successfully!"); // 提取用户 prompt const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdown, session.user.id, id, messageId, userPrompt, usedModelId ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError); console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300)); } } catch (error) { console.error("[Chat API] Volcano Vision generation error:", error); const errorMessageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: "图片生成失败,请稍后重试" }], }), }); // 🆕 图片生成结束标记(失败) writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); // 尝试保存错误消息 try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: "图片生成失败,请稍后重试" }], attachments: [], // 必须提供 attachments 字段 createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } }, }); return new Response(volcanoStream.pipeThrough(new JsonToSseTransformStream())); } // ========== 火山引擎视觉处理结束 ========== // ========== 千问视频生成独立处理 ========== if ((languageModel as any).provider === "qianwen-video" && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Qianwen video generation"); const { scheduleVideoUrlReplacement } = await import("@/lib/ai/providers/qianwen-video"); const qianwenVideoStream = createUIMessageStream({ execute: async ({ writer }) => { try { // 千问视频直接使用原始 URL,不转换为 base64 const modelMessages = convertToModelMessages(uiMessagesForModel); console.log("[Chat API] Qianwen video messages prepared, count:", modelMessages.length); // 调用 doGenerate 获取视频 const result = await (languageModel as any).doGenerate({ messages: modelMessages, }); const videoMarkdown = result.text || "视频生成失败"; console.log("[Chat API] Qianwen video generated, length:", videoMarkdown.length); // 提取临时视频 URL 和提示词(用于后续替换) const videoUrlMatch = videoMarkdown.match(/ { // 构建工具对象,只包含启用的工具 const tools: any = {}; const activeTools: string[] = []; if (enabledTools.includes("getWeather")) { tools.getWeather = getWeather; activeTools.push("getWeather"); } if (enabledTools.includes("createDocument")) { tools.createDocument = createDocument({ session, dataStream }); activeTools.push("createDocument"); } if (enabledTools.includes("updateDocument")) { tools.updateDocument = updateDocument({ session, dataStream }); activeTools.push("updateDocument"); } if (enabledTools.includes("requestSuggestions")) { tools.requestSuggestions = requestSuggestions({ session, dataStream, }); activeTools.push("requestSuggestions"); } if (enabledTools.includes("generateVideo")) { tools.generateVideo = generateVideo({ dataStream }); activeTools.push("generateVideo"); } console.log("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志 console.log("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志 // 转换消息,对于 Gemini 和千问视频将图片 URL 转换为 base64 const convertedMessages = await convertMessagesWithImages( injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages), (languageModel as any).provider || "" ); console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志 // 🆕 检测是否是视频模型 const providerType = (languageModel as any).provider; console.log("[Chat API] Provider type for tool detection:", providerType); const isVolcanoVideoModel = providerType === "volcano-video" || actualModelId.includes("seedance"); const isOpenAICompatImageModel = resolvedModelSupportsImageGen && providerType === "openai.chat"; console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel); // 🆕 处理 OpenAI 兼容的生图模型输出(防止返回 data:image;base64 原文) if (isOpenAICompatImageModel) { console.log("[Chat API] Using generateText for OpenAI-compatible image model"); // 生成开始标记 dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); try { const imageRequestTimeoutMs = 180_000; const generated = await generateText({ model: languageModel, system: systemPrompt({ selectedChatModel: finalModelId, requestHints }), messages: convertedMessages, abortSignal: AbortSignal.timeout(imageRequestTimeoutMs), }); const rawText = generated.text?.trim() || "图片生成失败,请稍后重试"; const normalizedText = await normalizeAssistantImageMarkdown( rawText, requestBody.imageParams?.imageSize || "2K" ); const messageId = generateUUID(); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: normalizedText }], }), }); // 生成结束标记 dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); await saveMessages({ messages: [ { id: messageId, role: "assistant", parts: [{ type: "text", text: normalizedText }], createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); const latestUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = latestUserMessage?.parts?.find( (p): p is { type: "text"; text: string } => p.type === "text" )?.text || ""; await saveGeneratedImages( normalizedText, session.user.id, id, messageId, userPrompt, actualModelId ); return; } catch (error) { console.error("[Chat API] OpenAI-compatible image model generation error:", error); const errorMessageId = generateUUID(); const errorText = error instanceof Error ? error.message : "图片生成失败,请稍后重试"; dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorText }], }), }); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); await saveMessages({ messages: [ { id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorText }], createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); return; } } // 🎬 视频模型使用专门的生成流程 if (isVolcanoVideoModel) { console.log("[Chat API] Using video generation API"); // 导入视频生成函数 const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video"); // 从最后一条用户消息中提取提示词和图片 const lastUserMessage = uiMessages.filter(m => m.role === "user").pop(); let prompt = ""; const imageUrls: string[] = []; // 🐛 调试:打印原始消息结构 console.log("[Chat API] 🔍 原始 lastUserMessage:", JSON.stringify(lastUserMessage, null, 2)); if (lastUserMessage) { // 🆕 处理 parts 数组(AI SDK v4 格式) if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) { console.log("[Chat API] 🔍 Found parts array, length:", (lastUserMessage as any).parts.length); for (const part of (lastUserMessage as any).parts) { console.log("[Chat API] 🔍 Processing part:", JSON.stringify(part, null, 2)); if (part.type === "text" && part.text) { // 提取文本 prompt += part.text; console.log("[Chat API] 🔍 ✅ Extracted text:", part.text); } else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) { // 提取图片 imageUrls.push(part.url); console.log("[Chat API] 🔍 ✅ Extracted image URL:", part.url); } } } // 备用方案1:content 字符串/数组 const legacyContent = (lastUserMessage as any).content; if (typeof legacyContent === "string") { prompt = legacyContent; } else if (Array.isArray(legacyContent)) { for (const part of legacyContent) { if (typeof part === "string") { prompt += part; } else if (part.type === "text") { prompt += part.text; } else if (part.type === "image" && (part as any).image) { imageUrls.push((part as any).image); } } } else if ((lastUserMessage as any).experimental_attachments) { // 备用方案2:experimental_attachments for (const attachment of (lastUserMessage as any).experimental_attachments) { if (attachment.url && attachment.contentType?.startsWith("image/")) { imageUrls.push(attachment.url); } } } } console.log("[Chat API] 🔍 最终提取结果 - prompt:", prompt, "imageCount:", imageUrls.length); console.log("[Chat API] 🔍 imageUrls:", imageUrls); try { console.log("[Chat API] Video generation params:", { prompt, imageCount: imageUrls.length, videoParams: requestBody.videoParams, }); // 🆕 重新执行子模型选择逻辑,确定要使用的模型 UUID let videoModelId = finalModelId; // 默认使用选择的模型 ID // 检查是否是主模型,需要选择子模型 const selectedModelData = await getAiModelById({ id: finalModelId }); if (selectedModelData && !selectedModelData.isSubModel) { // 是主模型,根据 mode 选择子模型 const videoSubModels = await getSubModelsByParentModelId(finalModelId); console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`); if (videoSubModels.length > 0) { const videoMode = requestBody.videoParams?.mode || "text-to-video"; const modeToModelMap: Record = { "text-to-video": "lite-t2v", "first-last-frame": "pro", "first-frame": "pro-fast", "reference-images": "lite-i2v", }; const targetKey = modeToModelMap[videoMode]; const matchedModel = videoSubModels.find(({ model }) => model.modelId.toLowerCase().includes(targetKey) ); if (matchedModel) { videoModelId = matchedModel.model.id; console.log(`[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} for mode ${videoMode}`); } else { videoModelId = videoSubModels[0].model.id; console.log(`[Chat API] Video: Using fallback sub-model ${videoSubModels[0].model.displayName}`); } } } // 构建视频生成选项 const videoOptions: any = { model: actualModelId, prompt: prompt || "生成视频", mode: requestBody.videoParams?.mode || "text-to-video", resolution: requestBody.videoParams?.resolution || "720p", ratio: requestBody.videoParams?.ratio || "16:9", duration: requestBody.videoParams?.duration || 5, fps: requestBody.videoParams?.fps || 24, cameraMove: requestBody.videoParams?.cameraMove || "none", cameraFixed: requestBody.videoParams?.cameraFixed || false, watermark: requestBody.videoParams?.watermark || false, seed: requestBody.videoParams?.seed, }; console.log("[Chat API] Video options constructed:", videoOptions); // 根据模式添加图片 if (videoOptions.mode === "first-frame" && imageUrls[0]) { videoOptions.firstFrameImage = imageUrls[0]; } else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) { videoOptions.firstFrameImage = imageUrls[0]; videoOptions.lastFrameImage = imageUrls[1]; } else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) { videoOptions.referenceImages = imageUrls; } // 获取 provider 配置(使用确定的视频模型 UUID) console.log("[Chat API] Getting model config for:", videoModelId); const aiModelData = await getAiModelById({ id: videoModelId }); if (!aiModelData) { throw new Error("视频模型配置未找到"); } // 如果是子模型,获取父模型的 Provider 配置 let providerModelData = aiModelData; if (aiModelData.isSubModel && aiModelData.parentModelId) { console.log("[Chat API] This is a sub-model, getting parent model config:", aiModelData.parentModelId); const parentModel = await getAiModelById({ id: aiModelData.parentModelId }); if (parentModel) { providerModelData = parentModel; } } console.log("[Chat API] Getting provider config for:", providerModelData.providerId); const providerData = await getAiProviderById({ id: providerModelData.providerId }); if (!providerData) { throw new Error("Provider 配置未找到"); } const apiKey = providerData.apiKey; const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3"; console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey }); // 调用视频生成流 console.log("[Chat API] Starting video generation stream..."); const videoStream = doGenerateStream(apiKey, baseURL, videoOptions); // 收集所有流输出 console.log("[Chat API] Processing video stream..."); let fullText = ""; let chunkCount = 0; for await (const chunk of videoStream) { chunkCount++; fullText += chunk; console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`); } console.log(`[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}`); // 生成消息 ID const messageId = generateUUID(); // 一次性写入完整消息(与千问视频相同) dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: fullText }], attachments: [], }), }); // 保存消息到数据库(提取视频URL用于归档) try { // 从 fullText 中提取视频URL const videoUrlMatch = fullText.match(/]+src="([^"]+)"/); const temporaryVideoUrl = videoUrlMatch ? videoUrlMatch[1] : null; // 构建视频元数据 const videoMetadata = temporaryVideoUrl ? { provider: "volcano-video", mode: requestBody.videoParams?.mode, resolution: requestBody.videoParams?.resolution, ratio: requestBody.videoParams?.ratio, duration: requestBody.videoParams?.duration, createdAt: Date.now(), } : null; await saveMessages({ messages: [{ id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: fullText }], attachments: [], createdAt: new Date(), videoStorageStatus: temporaryVideoUrl ? "temporary" : null, temporaryVideoUrl: temporaryVideoUrl, permanentVideoUrl: null, archivedAt: null, videoMetadata: videoMetadata, }], }); console.log("[Chat API] ✅ Video message saved successfully!"); if (temporaryVideoUrl) { console.log("[Chat API] 📹 临时视频URL已保存,将在8小时后自动归档到S3"); } } catch (saveError) { console.error("[Chat API] ❌ Failed to save video message:", saveError); } console.log("[Chat API] Video generation finished successfully"); return; } catch (error: any) { console.error("[Chat API] ❌ Video generation error:", error); console.error("[Chat API] Error stack:", error.stack); // 生成错误消息 ID const errorMessageId = generateUUID(); const errorMessage = `❌ 视频生成失败:${error.message}\n\n请检查配置或稍后重试。`; // 使用正确的格式发送错误消息 dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], }), }); // 保存错误消息到数据库 try { await saveMessages({ messages: [{ id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } return; } } // 常规文本模型使用 streamText const streamRequestTimeoutMs = resolvedModelSupportsImageGen ? 180_000 : 90_000; const shouldDisableSmoothStream = resolvedModelSupportsImageGen || isVolcanoVideoModel; const result = streamText({ model: languageModel, system: systemPrompt({ selectedChatModel: finalModelId, requestHints }), messages: convertedMessages, stopWhen: stepCountIs(5), experimental_activeTools: finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools, experimental_transform: shouldDisableSmoothStream ? undefined : smoothStream({ chunking: "word" }), tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具 abortSignal: AbortSignal.timeout(streamRequestTimeoutMs), experimental_telemetry: { isEnabled: isProductionEnvironment, functionId: "stream-text", }, onFinish: async ({ usage }) => { try { const providers = await getTokenlensCatalog(); if (!actualModelId) { finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage, }); return; } if (!providers) { finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage, }); return; } const summary = getUsage({ modelId: actualModelId, usage, providers, }); finalMergedUsage = { ...usage, ...summary, modelId: actualModelId, } as AppUsage; dataStream.write({ type: "data-usage", data: finalMergedUsage }); } catch (err) { console.warn("TokenLens enrichment failed", err); finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage }); } }, }); result.consumeStream(); dataStream.merge( result.toUIMessageStream({ sendReasoning: true, }) ); }, generateId: generateUUID, onFinish: async ({ messages }) => { await saveMessages({ messages: messages.map((currentMessage) => ({ id: currentMessage.id, role: currentMessage.role, parts: currentMessage.parts, createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, })), }); // 检测助手消息中是否包含视频,如果有则启动异步替换任务 for (const msg of messages) { if (msg.role === "assistant" && msg.parts) { const textContent = msg.parts .filter((part: any) => part && part.type === "text" && part.text) .map((part: any) => part.text) .join(""); // 检测是否包含视频标签 const videoUrlMatch = textContent?.match(/]+src="([^"]+)"/); if (videoUrlMatch) { const tempVideoUrl = videoUrlMatch[1]; const promptMatch = textContent?.match(/\*\*提示词\*\*:\s*(.+)/); const prompt = promptMatch ? promptMatch[1].trim() : "生成视频"; console.log("[Chat API] 检测到视频消息,启动URL替换任务"); console.log("[Chat API] 临时视频URL:", tempVideoUrl); // 动态导入并启动替换任务(5分钟后执行) import("@/lib/ai/providers/qianwen-video").then(({ scheduleVideoUrlReplacement }) => { scheduleVideoUrlReplacement(id, msg.id, tempVideoUrl, prompt); }).catch((err) => { console.error("[Chat API] 启动视频替换任务失败:", err); }); } } } if (finalMergedUsage) { try { await updateChatLastContextById({ chatId: id, context: finalMergedUsage, }); } catch (err) { console.warn("Unable to persist last usage for chat", id, err); } } }, onError: () => { return "Oops, an error occurred!"; }, }); // const streamContext = getStreamContext(); // if (streamContext) { // return new Response( // await streamContext.resumableStream(streamId, () => // stream.pipeThrough(new JsonToSseTransformStream()) // ) // ); // } return new Response(stream.pipeThrough(new JsonToSseTransformStream())); } catch (error) { const vercelId = request.headers.get("x-vercel-id"); if (error instanceof ChatSDKError) { return error.toResponse(); } // Check for Vercel AI Gateway credit card error if ( error instanceof Error && error.message?.includes( "AI Gateway requires a valid credit card on file to service requests" ) ) { return new ChatSDKError("bad_request:activate_gateway").toResponse(); } console.error("Unhandled error in chat API:", error, { vercelId }); return new ChatSDKError("offline:chat").toResponse(); } } export async function DELETE(request: Request) { const { searchParams } = new URL(request.url); const id = searchParams.get("id"); if (!id) { return new ChatSDKError("bad_request:api").toResponse(); } const session = await auth(); if (!session?.user) { return new ChatSDKError("unauthorized:chat").toResponse(); } const chat = await getChatById({ id }); if (chat?.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } const deletedChat = await deleteChatById({ id }); return Response.json(deletedChat, { status: 200 }); }