import { geolocation } from "@vercel/functions"; import { convertToModelMessages, createUIMessageStream, generateText, JsonToSseTransformStream, smoothStream, stepCountIs, streamText, } from "ai"; import { unstable_cache as cache } from "next/cache"; import { after, NextResponse } from "next/server"; import { createResumableStreamContext, type ResumableStreamContext, } from "resumable-stream"; import type { ModelCatalog } from "tokenlens/core"; import { fetchModels } from "tokenlens/fetch"; import { getUsage } from "tokenlens/helpers"; import { auth, type UserType } from "@/app/(auth)/auth"; import type { VisibilityType } from "@/components/visibility-selector"; import { entitlementsByUserType } from "@/lib/ai/entitlements"; import type { ChatModel } from "@/lib/ai/models"; import { type RequestHints, systemPrompt } from "@/lib/ai/prompts"; import { myProvider } from "@/lib/ai/providers"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { createDocument } from "@/lib/ai/tools/create-document"; import { getWeather } from "@/lib/ai/tools/get-weather"; import { requestSuggestions } from "@/lib/ai/tools/request-suggestions"; import { updateDocument } from "@/lib/ai/tools/update-document"; import { generateVideo } from "@/lib/ai/tools/generate-video"; import { getEnabledTools } from "@/lib/ai/tools-registry"; import { isProductionEnvironment } from "@/lib/constants"; import { createStreamId, deleteChatById, getChatById, getPointsSystemConfig, getUserById, getMessageCountByUserId, getMessagesByChatId, saveChat, saveMessages, updateMessageById, updateChatLastContextById, getAiModelById, getEnabledAiModelsWithProvider, getSubModelsByParentModelId, getAiProviderById, checkUserImageQuota, consumeUserPoints, } from "@/lib/db/queries"; import { db } from "@/lib/db"; import type { DBMessage } from "@/lib/db/schema"; import { generatedImage, message as messageTable } from "@/lib/db/schema"; import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; import { videoArchiver } from "@/lib/storage/video-archiver"; import { uploadFile } from "@/lib/storage/service"; import type { ChatMessage } from "@/lib/types"; import type { AppUsage } from "@/lib/usage"; import { convertToUIMessages, generateUUID } from "@/lib/utils"; import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags"; import { generateTitleFromUserMessage } from "../../actions"; import { type PostRequestBody, postRequestBodySchema } from "./schema"; /** * 将图片 URL 转换为 base64 格式 * 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API */ async function convertImageUrlToBase64(url: string): Promise { try { console.log("[Image Convert] Fetching image from:", url); const response = await fetch(url); if (!response.ok) { throw new Error(`Failed to fetch image: ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const base64 = buffer.toString("base64"); // 获取 MIME 类型 const contentType = response.headers.get("content-type") || "image/jpeg"; const dataUrl = `data:${contentType};base64,${base64}`; console.log("[Image Convert] Converted to base64, size:", base64.length); return dataUrl; } catch (error) { console.error("[Image Convert] Failed:", error); throw error; } } /** * 检测消息中是否包含用户标注过的图片(文件名以 annotated- 开头) */ function hasAnnotatedImage(messages: ChatMessage[]): boolean { for (const message of messages) { for (const part of message.parts) { if (part.type === "file" && "name" in part && typeof part.name === "string") { if (part.name.startsWith("annotated-")) { return true; } } } } return false; } /** * 标注图片的系统提示词(用户不可见,后台自动添加) */ const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条(如箭头、圆圈、画笔痕迹或文字说明),这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义,按照标注指示进行编辑,但在生成的新图片中不要保留这些标注线条,输出干净的编辑结果。"; function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: string): ChatMessage[] { if (!hiddenPrompt || !hiddenPrompt.trim()) return messages; const cloned = messages.map((msg) => ({ ...msg, parts: msg.parts.map((part) => ({ ...part })), })); for (let i = cloned.length - 1; i >= 0; i--) { const msg = cloned[i]; if (msg.role !== "user") continue; const textPart = msg.parts.find((part) => part.type === "text") as { type: "text"; text: string } | undefined; if (textPart) { textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim(); } else { msg.parts.unshift({ type: "text", text: hiddenPrompt }); } break; } return cloned; } function applyOptimizedPromptToLatestUserMessage( messages: ChatMessage[], optimizedPrompt?: string, hiddenPromptPrefix = "", ): ChatMessage[] { const prompt = optimizedPrompt?.trim(); if (!prompt) return messages; const cloned = messages.map((msg) => ({ ...msg, parts: msg.parts.map((part) => ({ ...part })), })); for (let i = cloned.length - 1; i >= 0; i -= 1) { const msg = cloned[i]; if (msg.role !== "user") continue; const textPart = msg.parts.find( (part): part is { type: "text"; text: string } => part.type === "text", ); const originalText = textPart?.text?.trim() || ""; const normalizedOriginal = hiddenPromptPrefix && originalText.startsWith(hiddenPromptPrefix) ? originalText.slice(hiddenPromptPrefix.length).trim() : originalText; const hasOriginalDemand = normalizedOriginal.length > 0; const hasOriginalInOptimized = hasOriginalDemand && (prompt.includes(normalizedOriginal) || normalizedOriginal.includes(prompt)); const finalPrompt = hasOriginalDemand && !hasOriginalInOptimized ? [ hiddenPromptPrefix.trim(), `【核心目标(必须优先满足)】${normalizedOriginal}`, `【执行优化细节】${prompt}`, ] .filter(Boolean) .join("\n\n") .trim() : `${hiddenPromptPrefix}${prompt}`.trim(); console.log("[Prompt Merge] Final image prompt prepared:", { original: normalizedOriginal.substring(0, 120), optimized: prompt.substring(0, 120), merged: finalPrompt.substring(0, 180), forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized, }); if (textPart) { textPart.text = finalPrompt; } else { msg.parts.unshift({ type: "text", text: finalPrompt }); } break; } return cloned; } type StreamWriter = { write: (chunk: any) => void; }; function safeWriteStreamChunk(writer: StreamWriter, chunk: any, label: string): boolean { try { writer.write(chunk); return true; } catch (error) { console.warn(`[Chat API] Stream write skipped (${label})`, error); return false; } } function writeAssistantTextChunk( writer: StreamWriter, text: string, label: string, messageId = generateUUID(), ) { safeWriteStreamChunk( writer, { type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text }], }), }, label, ); return messageId; } function createSseResponse(stream: ReadableStream) { return new Response(stream.pipeThrough(new JsonToSseTransformStream()), { headers: { "Content-Type": "text/event-stream; charset=utf-8", "Cache-Control": "no-cache, no-transform", Connection: "keep-alive", "X-Accel-Buffering": "no", }, }); } function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs = 15000) { const timer = setInterval(() => { safeWriteStreamChunk( writer, { type: "data-keepalive", data: JSON.stringify({ ts: Date.now(), label }), }, `${label}:keepalive`, ); }, intervalMs); return () => clearInterval(timer); } type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; const VIDEO_PROVIDER_TYPES = new Set([ "volcano-video", "doubao-video", "qianwen-video", "fal-video", ]); function isVideoProviderType(providerType?: string | null): boolean { return Boolean(providerType && VIDEO_PROVIDER_TYPES.has(providerType)); } function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] { const normalized = modelId.toLowerCase(); if (normalized.includes("lite-t2v")) { return ["text-to-video"]; } if (normalized.includes("lite-i2v")) { return ["first-frame", "first-last-frame", "reference-images"]; } if (normalized.includes("pro-fast")) { return ["text-to-video", "first-frame"]; } if (normalized.includes("-pro-") || normalized.endsWith("-pro")) { return ["text-to-video", "first-frame", "first-last-frame"]; } return ["text-to-video", "first-frame", "first-last-frame", "reference-images"]; } function getSupportedVideoModesForModel(model: any): VideoModeType[] { const capabilityModes = (((model?.capabilities as any)?.supportedModes ?? []) as string[]) .filter((mode): mode is VideoModeType => mode === "text-to-video" || mode === "first-frame" || mode === "first-last-frame" || mode === "reference-images" ); if (capabilityModes.length > 0) { return capabilityModes; } const modelId = String(model?.modelId || ""); return inferSupportedVideoModesByModelId(modelId); } function scoreVideoSubModelForMode(modelId: string, mode: VideoModeType): number { const normalized = modelId.toLowerCase(); const isProFast = normalized.includes("pro-fast"); const isLiteT2V = normalized.includes("lite-t2v"); const isLiteI2V = normalized.includes("lite-i2v"); const isPro = (normalized.includes("-pro-") || normalized.endsWith("-pro")) && !isProFast; const isV15Pro = normalized.includes("seedance-1-5-pro"); switch (mode) { case "first-last-frame": if (isV15Pro) return 150; if (isPro) return 120; if (isLiteI2V) return 90; if (isProFast) return -1000; if (isLiteT2V) return -1000; return 10; case "first-frame": if (isV15Pro) return 140; if (isProFast) return 130; if (isPro) return 120; if (isLiteI2V) return 100; if (isLiteT2V) return 20; return 10; case "reference-images": if (isV15Pro) return 150; if (isLiteI2V) return 140; if (isPro) return 110; if (isProFast || isLiteT2V) return -1000; return 10; case "text-to-video": default: if (isV15Pro) return 140; if (isLiteT2V) return 130; if (isProFast) return 120; if (isPro) return 100; if (isLiteI2V) return 80; return 10; } } function selectBestVideoSubModelForMode( subModels: Array<{ model: any }>, mode: VideoModeType, ): { model: any } | null { const compatible = subModels.filter(({ model }) => { const supportedModes = getSupportedVideoModesForModel(model); return supportedModes.includes(mode); }); const candidates = compatible.length > 0 ? compatible : subModels; if (candidates.length === 0) return null; return [...candidates].sort((a, b) => { const scoreA = scoreVideoSubModelForMode(String(a.model?.modelId || ""), mode); const scoreB = scoreVideoSubModelForMode(String(b.model?.modelId || ""), mode); return scoreB - scoreA; })[0]; } function extractVideoUrlFromText(text: string): string | null { const normalizedText = String(text || ""); if (!normalizedText) return null; const htmlMatch = normalizedText.match(/]+src="([^"]+)"/i); if (htmlMatch?.[1]) return htmlMatch[1]; const markdownMatch = normalizedText.match( /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i, ); if (markdownMatch?.[1]) return markdownMatch[1]; const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i); return plainMatch?.[1] || null; } function replaceVideoUrlInText(text: string, oldUrl: string, newUrl: string): string { if (!text || !oldUrl || !newUrl || oldUrl === newUrl) return text; return text.split(oldUrl).join(newUrl); } function isAlreadyArchivedVideoUrl(url: string): boolean { return /\/uploads\//i.test(url); } function scheduleVideoArchive(params: { chatId: string; messageId: string; text: string; temporaryVideoUrl: string; taskId?: string; metadata?: Record | null; }) { const { chatId, messageId, text, temporaryVideoUrl, taskId, metadata } = params; if (!temporaryVideoUrl || isAlreadyArchivedVideoUrl(temporaryVideoUrl)) { return; } after(async () => { try { const permanentVideoUrl = await videoArchiver.archiveVideo({ messageId, temporaryUrl: temporaryVideoUrl, taskId: taskId || messageId, metadata: { duration: typeof metadata?.duration === "number" && Number.isFinite(metadata.duration) ? metadata.duration : undefined, resolution: typeof metadata?.resolution === "string" ? metadata.resolution : undefined, ratio: typeof metadata?.ratio === "string" ? metadata.ratio : undefined, }, }); const archivedText = replaceVideoUrlInText(text, temporaryVideoUrl, permanentVideoUrl); const nextMetadata = { ...(metadata || {}), temporaryVideoUrl, permanentVideoUrl, storageStatus: "archived", archivedAt: Date.now(), updatedAt: Date.now(), }; await updateMessageById({ messageId, chatId, parts: [{ type: "text", text: archivedText }], videoStorageStatus: "archived", temporaryVideoUrl, permanentVideoUrl, archivedAt: new Date(), videoMetadata: nextMetadata as any, }); console.log("[Chat API] ✅ 视频已归档到永久存储:", { messageId, permanentVideoUrl, }); } catch (error) { console.error("[Chat API] ❌ 视频归档失败(保留临时地址):", { messageId, temporaryVideoUrl, error, }); } }); } async function replayAssistantMessagesForDuplicateRequest({ chatId, userMessageId, writer, source, waitMs = DUPLICATE_REQUEST_WAIT_MS, }: { chatId: string; userMessageId: string; writer: StreamWriter; source: string; waitMs?: number; }) { const startedAt = Date.now(); const seenAssistantMessageIds = new Set(); while (Date.now() - startedAt < waitMs) { try { const dbMessages = await getMessagesByChatId({ id: chatId }); const userMessageIndex = dbMessages.findIndex((msg) => msg.id === userMessageId); if (userMessageIndex >= 0) { const assistantMessages = dbMessages .slice(userMessageIndex + 1) .filter((msg) => msg.role === "assistant"); if (assistantMessages.length > 0) { for (const assistantMessage of assistantMessages) { if (seenAssistantMessageIds.has(assistantMessage.id)) continue; seenAssistantMessageIds.add(assistantMessage.id); safeWriteStreamChunk( writer, { type: "data-appendMessage", data: JSON.stringify({ id: assistantMessage.id, role: "assistant", parts: assistantMessage.parts, }), }, `${source}:replay`, ); } return; } } } catch (error) { console.warn(`[Chat API] Failed duplicate replay fetch (${source})`, error); } safeWriteStreamChunk( writer, { type: "data-keepalive", data: JSON.stringify({ ts: Date.now(), source }), }, `${source}:keepalive`, ); await sleep(1000); } } type ChatImageParams = NonNullable; type AgentRecommendedImageParams = { aspectRatio?: string; imageSize?: string; imageCount?: number; negativePrompt?: string; promptExtend?: boolean; }; type ChatVideoParams = NonNullable; type ChatVideoParamsDraft = Partial; type AgentRecommendedVideoParams = { mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; resolution?: "480p" | "720p" | "1080p"; ratio?: string; duration?: number; generateAudio?: boolean; cameraMove?: string; }; type AgentVideoPlan = { script: string; storyboard: Array<{ shot: number; title: string; duration: number; description: string; prompt: string; camera?: string; }>; finalPrompt: string; negativePrompt?: string; recommendedVideoParams?: AgentRecommendedVideoParams; }; const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ "alibaba-image", "ruxa-image", "fal-image", "volcano-vision", "gemini", "huanke-gemini", ]); const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000; const imageFailoverModelCooldown = new Map(); function normalizeFailoverErrorMessage(error: unknown): string { if (!error) return ""; if (typeof error === "string") return error; if (error instanceof Error) return error.message || String(error); try { return JSON.stringify(error); } catch { return String(error); } } function shouldCooldownImageFallbackModel(errorMessage: string): boolean { const msg = errorMessage.toLowerCase(); return ( msg.includes("model not found") || msg.includes("model_not_found") || msg.includes("no available channel for model") || msg.includes("unknown model") || msg.includes("invalid model") || msg.includes("api key is missing") || msg.includes("invalid api key") || msg.includes("unauthorized") || msg.includes("cannot convert argument to a bytestring") ); } function isImageFallbackModelCoolingDown(modelId: string): boolean { const snapshot = imageFailoverModelCooldown.get(modelId); if (!snapshot) return false; if (snapshot.until <= Date.now()) { imageFailoverModelCooldown.delete(modelId); return false; } return true; } function setImageFallbackModelCooldown(modelId: string, reason: string): void { imageFailoverModelCooldown.set(modelId, { until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS, reason: reason.slice(0, 500), }); } function getImageCrossProviderCandidates( allModels: any[], options: { currentModelId: string; currentProviderId?: string; providerPriority: Record; } ) { return allModels .filter((m) => { if (!m?.model?.supportsImageGen || !m?.model?.isEnabled || m?.model?.isSubModel) { return false; } if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) { return false; } if (m.model.id === options.currentModelId) return false; if (options.currentProviderId && m.provider.id === options.currentProviderId) return false; if (isImageFallbackModelCoolingDown(m.model.id)) { const cooldown = imageFailoverModelCooldown.get(m.model.id); console.log( `[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}` ); return false; } return true; }) .sort((a, b) => { const left = options.providerPriority[a.provider.type as string] ?? 99; const right = options.providerPriority[b.provider.type as string] ?? 99; if (left !== right) return left - right; return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0); }); } function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean { if (!params) return false; const customizedAspectRatio = typeof params.aspectRatio === "string" && params.aspectRatio.trim() && params.aspectRatio !== "auto"; const customizedImageSize = typeof params.imageSize === "string" && params.imageSize.trim() && params.imageSize.toUpperCase() !== "2K"; const customizedImageCount = typeof params.imageCount === "number" && params.imageCount !== 1; const customizedPromptExtend = typeof params.promptExtend === "boolean"; const customizedAdvanced = Boolean( params.size || params.seed !== undefined || params.n || params.negativePrompt || customizedPromptExtend ); return customizedAspectRatio || customizedImageSize || customizedImageCount || customizedAdvanced; } function mergeImageParamsWithAgentRecommendation(params: { incoming?: PostRequestBody["imageParams"]; recommended?: AgentRecommendedImageParams; intent?: "image_edit" | "image_gen" | "vision" | "video_gen" | "chat"; hasUserCustomSettings: boolean; }): PostRequestBody["imageParams"] { const { incoming, recommended, intent, hasUserCustomSettings } = params; const base: ChatImageParams = { ...(incoming || {}) }; if (!recommended) { return Object.keys(base).length > 0 ? base : undefined; } const merged: ChatImageParams = { ...base, negativePrompt: typeof recommended.negativePrompt === "string" && recommended.negativePrompt.trim() ? recommended.negativePrompt.trim() : base.negativePrompt, promptExtend: typeof recommended.promptExtend === "boolean" ? recommended.promptExtend : base.promptExtend, }; if (!hasUserCustomSettings) { if (recommended.aspectRatio) merged.aspectRatio = recommended.aspectRatio; if (recommended.imageSize) merged.imageSize = recommended.imageSize; if (typeof recommended.imageCount === "number") merged.imageCount = recommended.imageCount; } if (intent === "image_edit") { merged.imageCount = 1; if (typeof base.promptExtend !== "boolean") { merged.promptExtend = false; } } return Object.keys(merged).length > 0 ? merged : undefined; } function hasUserCustomizedVideoParams(params?: PostRequestBody["videoParams"]): boolean { if (!params) return false; const defaults = { mode: "text-to-video", resolution: "720p", ratio: "16:9", duration: 5, generateAudio: true, cameraMove: "none", } as const; if (params.mode && params.mode !== defaults.mode) return true; if (params.resolution && params.resolution !== defaults.resolution) return true; if (params.ratio && params.ratio !== defaults.ratio) return true; if (typeof params.duration === "number" && params.duration !== defaults.duration) return true; if (typeof params.generateAudio === "boolean" && params.generateAudio !== defaults.generateAudio) return true; if (typeof params.cameraMove === "string" && params.cameraMove !== defaults.cameraMove) return true; if (typeof params.seed === "number") return true; return false; } function mergeVideoParamsWithAgentRecommendation(params: { incoming?: PostRequestBody["videoParams"]; recommended?: AgentRecommendedVideoParams; hasUserCustomSettings: boolean; }): PostRequestBody["videoParams"] { const { incoming, recommended, hasUserCustomSettings } = params; const base: ChatVideoParamsDraft = { ...(incoming || {}) }; if (!recommended) { return Object.keys(base).length > 0 ? (base as ChatVideoParams) : undefined; } const merged: ChatVideoParamsDraft = { ...base }; if (!hasUserCustomSettings) { if (recommended.mode) merged.mode = recommended.mode; if (recommended.resolution) merged.resolution = recommended.resolution; if (recommended.ratio) merged.ratio = recommended.ratio; if (typeof recommended.duration === "number") merged.duration = recommended.duration; if (typeof recommended.generateAudio === "boolean") { merged.generateAudio = recommended.generateAudio; } if (recommended.cameraMove) merged.cameraMove = recommended.cameraMove; } if (!merged.mode) { merged.mode = recommended.mode || (incoming?.mode as ChatVideoParams["mode"]) || "text-to-video"; } if (!merged.resolution) { merged.resolution = recommended.resolution || (incoming?.resolution as ChatVideoParams["resolution"]) || "720p"; } if (!merged.ratio) { merged.ratio = recommended.ratio || incoming?.ratio || "16:9"; } if (typeof merged.duration !== "number") { merged.duration = recommended.duration || incoming?.duration || 5; } return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined; } function sanitizeAgentOutputSettingsSection(params: { message?: string; userImageParams?: PostRequestBody["imageParams"]; hasUserCustomImageParams: boolean; }): string | undefined { const { message, userImageParams, hasUserCustomImageParams } = params; if (!message?.trim()) return message; const outputSectionRegex = /(?:###\s*输出设置|【\s*输出设置\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; if (!outputSectionRegex.test(message)) return message; if (!hasUserCustomImageParams) { return message.replace( outputSectionRegex, "【输出设置】\n- 参数:按用户面板当前设置执行(未自动改写)" ); } const lines: string[] = []; const aspectRatio = typeof userImageParams?.aspectRatio === "string" ? userImageParams.aspectRatio.trim() : ""; const imageSize = typeof userImageParams?.imageSize === "string" ? userImageParams.imageSize.trim() : ""; const imageCount = typeof userImageParams?.imageCount === "number" ? userImageParams.imageCount : undefined; if (aspectRatio && aspectRatio !== "auto") { lines.push(`- 宽高比:${aspectRatio}(来自用户面板设置)`); } if (imageSize && imageSize.toUpperCase() !== "2K") { lines.push(`- 清晰度档位:${imageSize.toUpperCase()}(来自用户面板设置)`); } if (typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount !== 1) { lines.push(`- 出图数量:${Math.max(1, Math.round(imageCount))} 张(来自用户面板设置)`); } lines.push("- 其余参数:保持用户面板当前设置"); return message.replace(outputSectionRegex, `【输出设置】\n${lines.join("\n")}`); } function normalizeAgentReportHeadingLine(line: string): string { const trimmed = line.trim(); if (!trimmed) return line; const markdownHeading = trimmed.match(/^#{1,6}\s*(.+?)\s*$/); const plainTitle = markdownHeading?.[1]?.trim() || trimmed; const normalizedTitle = plainTitle .replace(/^[^\u4e00-\u9fa5A-Za-z0-9]+/, "") .trim(); const knownTitles = [ "需求理解", "图片分析", "执行计划", "输出设置", "脚本规划", "分镜规划", "执行说明", "视频生成完成", ]; if (knownTitles.some((title) => normalizedTitle === title)) { return `【${normalizedTitle}】`; } if (markdownHeading) { return `【${normalizedTitle || plainTitle}】`; } return line; } function stripAgentFriendlyMediaEmbeds(message: string): string { if (!message) return message; const withoutInlineMedia = message .replace(/!\[[^\]]*]\((https?:\/\/[^\s)]+)\)/gi, "") .replace(//gi, "") .replace(/]*>/gi, ""); const cleanedLines = withoutInlineMedia .split("\n") .filter((line) => { const trimmed = line.trim(); if (!trimmed) return true; if (/^https?:\/\/\S+$/i.test(trimmed)) return false; if (/^\[[^\]]*]\(https?:\/\/[^\s)]+\)$/i.test(trimmed)) return false; return true; }) .join("\n"); return cleanedLines.replace(/\n{3,}/g, "\n\n").trim(); } function normalizeAgentReportForDisplay(message?: string): string | undefined { if (!message?.trim()) return message; const stripped = stripAgentFriendlyMediaEmbeds(message); if (!stripped) return undefined; return stripped .split("\n") .map((line) => normalizeAgentReportHeadingLine(line)) .join("\n"); } function guessImageMediaType(url: string): string { const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg"; if (cleanUrl.endsWith(".webp")) return "image/webp"; if (cleanUrl.endsWith(".gif")) return "image/gif"; if (cleanUrl.endsWith(".bmp")) return "image/bmp"; if (cleanUrl.endsWith(".svg")) return "image/svg+xml"; return "image/png"; } function injectHistoryImagesToLastUserMessage( messages: ChatMessage[], historyImages: string[] ): ChatMessage[] { if (!historyImages || historyImages.length === 0) return messages; let lastUserIndex = -1; for (let i = messages.length - 1; i >= 0; i--) { if (messages[i]?.role === "user") { lastUserIndex = i; break; } } if (lastUserIndex < 0) return messages; const lastUserMessage = messages[lastUserIndex]; const hasExistingImage = lastUserMessage.parts?.some( (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" ); if (hasExistingImage) return messages; const imageParts = historyImages.map((url, index) => ({ type: "file" as const, url, name: `history-image-${index + 1}`, mediaType: guessImageMediaType(url), mimeType: guessImageMediaType(url), })); const updatedMessage: ChatMessage = { ...lastUserMessage, parts: [...lastUserMessage.parts, ...imageParts], }; return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg)); } function compactAutoDemand(text: string, maxLength = 56): string { const normalized = text.replace(/\s+/g, " ").trim(); if (!normalized) return "按你刚才的描述执行"; if (normalized.length <= maxLength) return normalized; return `${normalized.slice(0, maxLength)}...`; } function buildAutoModeFallbackMessage(params: { text: string; hasImage: boolean; fromHistory: boolean; imageSummary?: string; }): string { const demand = compactAutoDemand(params.text); if (params.hasImage) { const sourceLine = params.fromHistory ? "基于上一轮图片继续处理。" : "基于当前上传图片处理。"; return [ "### 需求理解", `- ${demand}`, "### 图片分析", params.imageSummary ? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}` : `- ${sourceLine}`, "- 我会优先保持主体、构图和整体风格一致,只处理你指定的改动。", "### 执行计划", "- 先完成本轮核心改动,再检查边缘与光影一致性。", "- 若结果偏差,我会按“保留A,修改B”的方式继续迭代。", "### 输出设置", "- 默认单图输出,优先稳定命中。", ].join("\n"); } return [ "### 需求理解", `- ${demand}`, "### 执行计划", "- 先锁定主体与场景,再细化风格、材质与光影。", "### 输出设置", "- 默认单图输出,优先稳定命中。", ].join("\n"); } function hasStructuredAgentSections(message?: string): boolean { if (!message) return false; return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】)/.test( message, ); } function ensureStructuredAgentFriendlyMessage(params: { message?: string; text: string; hasImage: boolean; fromHistory: boolean; }): string { const existingMessage = String(params.message || "").trim(); if (hasStructuredAgentSections(existingMessage)) { return existingMessage; } const fallback = buildAutoModeFallbackMessage({ text: params.text, hasImage: params.hasImage, fromHistory: params.fromHistory, }); if (!existingMessage) { return fallback; } const normalizedSupplement = existingMessage.replace(/\s+/g, " ").trim(); if (!normalizedSupplement) { return fallback; } return fallback.replace( "### 执行计划\n", `### 执行计划\n- 补充说明:${normalizedSupplement}\n`, ); } function extractMessageImageUrls(parts: any[]): string[] { return (parts || []) .map((part: any) => { if (!part) return null; if (part.type === "file") return part.url || part.data || null; if (part.type === "image") return part.url || part.image || null; if (part.type === "image_url") return part.image_url?.url || null; return null; }) .filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url)); } function shouldEnrichImageAnalysisText(message?: string): boolean { if (!message) return false; if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false; return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message); } function injectImageAnalysisSummary(message: string, summary: string): string { const normalizedSummary = summary.replace(/\s+/g, " ").trim(); if (!normalizedSummary) return message; const lines = message.split("\n"); const analysisHeaderIndex = lines.findIndex((line) => /^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim()) ); if (analysisHeaderIndex < 0) return message; for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) { const current = lines[i]?.trim() || ""; if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break; if (/^- /.test(current)) { lines[i] = `- ${normalizedSummary}`; return lines.join("\n"); } } const insertIndex = analysisHeaderIndex + 1; lines.splice(insertIndex, 0, `- ${normalizedSummary}`); return lines.join("\n"); } function ensureImageAnalysisSummarySection(message: string, summary: string): string { const normalizedSummary = summary.replace(/\s+/g, " ").trim(); if (!normalizedSummary) return message; if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) { return injectImageAnalysisSummary(message, normalizedSummary); } const trimmed = message.trim(); if (!trimmed) return `【图片分析】\n- ${normalizedSummary}`; return `${trimmed}\n【图片分析】\n- ${normalizedSummary}`; } /** * 自定义消息转换函数 * 注意:gemini-image 和 huanke-gemini 都需要 URL 格式(不是 Base64) * 因为这些 provider 内部会自行处理图片上传到 S3 */ async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) { if (providerType === "ruxa-image" || providerType === "fal-image") { const convertedMessages = []; for (const message of messages) { const content: any[] = []; for (const part of message.parts) { const partAny = part as any; if (partAny.type === "text" && partAny.text) { content.push({ type: "text", text: partAny.text, }); continue; } const candidateUrl = partAny.type === "file" ? (partAny.url || partAny.data) : partAny.type === "image" ? (partAny.image || partAny.url) : partAny.type === "image_url" ? partAny.image_url?.url : undefined; if (!candidateUrl || typeof candidateUrl !== "string") continue; if (candidateUrl.startsWith("data:")) { throw new Error("当前模型不支持 Base64 图片传输,请先上传图片后再编辑"); } if (!candidateUrl.startsWith("http")) continue; content.push({ type: "image_url", image_url: { url: candidateUrl, }, }); } convertedMessages.push({ role: message.role, content: content.length === 1 && content[0].type === "text" ? content[0].text : content, }); } return convertedMessages; } // 所有 provider 都使用 URL 格式传输图片 // gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL // 这里不再进行 Base64 转换,直接返回标准格式 if (providerType !== "gemini") { return convertToModelMessages(messages); } // 🆕 检测是否有标注图片 const hasAnnotation = hasAnnotatedImage(messages); if (hasAnnotation) { console.log("[Message Convert] 🎨 检测到标注图片,将自动添加系统提示"); } // 对于 Gemini,需要特殊处理图片 const convertedMessages = []; for (const message of messages) { const content: any[] = []; // 🆕 如果是用户消息且有标注图片,在文本前添加系统提示 let hasAddedAnnotationPrompt = false; // 处理消息的各个部分 for (const part of message.parts) { if (part.type === "text" && part.text) { let textContent = part.text; // 🆕 如果有标注图片且是用户消息,在用户文本前添加系统提示 if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) { textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求:" + textContent; hasAddedAnnotationPrompt = true; console.log("[Message Convert] 🎨 已注入标注提示词"); } content.push({ type: "text", text: textContent, }); } else if (part.type === "file" && part.url) { // 直接使用 URL 格式(不再转换为 Base64) // gemini-image 和 huanke-gemini provider 内部会处理图片上传 content.push({ type: "image_url", image_url: { url: part.url, }, }); console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "..."); } } // 🆕 如果有标注图片但消息中没有文本部分,添加一个纯提示文本 if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) { content.unshift({ type: "text", text: ANNOTATION_SYSTEM_PROMPT, }); console.log("[Message Convert] 🎨 已在消息开头注入标注提示词"); } convertedMessages.push({ role: message.role, content: content.length === 1 && content[0].type === "text" ? content[0].text : content, }); } return convertedMessages; } function parseImageAltMetadata(altText: string, thumbnailUrl: string): { originalUrl: string; imageSize: string; } { let originalUrl = thumbnailUrl; let imageSize = "2K"; // 格式1: thumb|||originalUrl|||size (新格式) if (altText.includes("thumb|||")) { const content = altText.substring(altText.indexOf("thumb|||") + 8); const parts = content.split("|||"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 格式2: thumb$$originalUrl$$size (中间格式,可能被 LaTeX 解析器破坏) else if (altText.includes("thumb$$")) { const content = altText.substring(altText.indexOf("thumb$$") + 7); const parts = content.split("$$"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 格式3: thumb|originalUrl|size (旧格式) else if (altText.includes("thumb|")) { const content = altText.substring(altText.indexOf("thumb|") + 6); const parts = content.split("|"); originalUrl = parts[0] || thumbnailUrl; imageSize = parts[1] || "2K"; } // 回退策略:从缩略图 URL 推断原图 URL if (originalUrl === thumbnailUrl) { if (thumbnailUrl.includes("gemini-thumb-")) { originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-"); } else if (thumbnailUrl.includes("openai-thumb-")) { originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-"); } else if (thumbnailUrl.includes("volcano-thumb-")) { originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-"); } else if (thumbnailUrl.includes("alibaba-image-thumb-")) { originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-"); } else if (thumbnailUrl.includes("ruxa-image-thumb-")) { originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-"); } else if (thumbnailUrl.includes("fal-image-thumb-")) { originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-"); } } return { originalUrl, imageSize }; } function extractOriginalImageUrlsFromText(text: string): string[] { return extractImageUrlsFromText(text); } function mimeTypeToExtension(mimeType: string) { const normalized = mimeType.toLowerCase(); if (normalized.includes("png")) return "png"; if (normalized.includes("webp")) return "webp"; if (normalized.includes("gif")) return "gif"; if (normalized.includes("bmp")) return "bmp"; return "jpg"; } function getExtensionFromUrl(url: string): string { try { const pathname = new URL(url).pathname; const ext = pathname.split(".").pop()?.toLowerCase(); if (!ext) return "jpg"; if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext; } catch { // ignore } return "jpg"; } async function uploadRemoteImageToStorage(imageUrl: string): Promise { const response = await fetch(imageUrl, { method: "GET", signal: AbortSignal.timeout(30_000), }); if (!response.ok) { throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`); } const contentType = response.headers.get("content-type")?.toLowerCase() || ""; if (contentType && !contentType.startsWith("image/")) { throw new Error(`Remote url is not an image: ${contentType}`); } const arrayBuffer = await response.arrayBuffer(); if (!arrayBuffer || arrayBuffer.byteLength === 0) { throw new Error("Remote image content is empty"); } const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl); const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`; const uploaded = await uploadFile(fileName, arrayBuffer); return uploaded.url; } async function isRemoteImageAccessible(imageUrl: string): Promise { try { const headResponse = await fetch(imageUrl, { method: "HEAD", signal: AbortSignal.timeout(10_000), }); if (headResponse.ok) { return true; } if (headResponse.status !== 405 && headResponse.status !== 403) { return false; } const probeResponse = await fetch(imageUrl, { method: "GET", headers: { Range: "bytes=0-0", }, signal: AbortSignal.timeout(10_000), }); return probeResponse.ok; } catch { return false; } } async function ensureHostedImageUrl(imageUrl: string): Promise { const normalizedUrl = imageUrl.trim(); if (!normalizedUrl) { throw new Error("Image URL is empty"); } if (normalizedUrl.startsWith("data:image")) { return uploadDataUrlImage(normalizedUrl); } if (!/^https?:\/\//i.test(normalizedUrl)) { throw new Error(`Unsupported image URL: ${normalizedUrl}`); } if (/\/uploads\//i.test(normalizedUrl)) { const accessible = await isRemoteImageAccessible(normalizedUrl); if (accessible) { return normalizedUrl; } } return uploadRemoteImageToStorage(normalizedUrl); } async function uploadDataUrlImage(dataUrl: string): Promise { const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i); if (!match) { throw new Error("Invalid data URL image format"); } const mimeType = match[1] || "image/jpeg"; const base64Data = (match[2] || "").replace(/\s+/g, ""); if (!base64Data) { throw new Error("Empty base64 image content"); } const buffer = Buffer.from(base64Data, "base64"); const fileExt = mimeTypeToExtension(mimeType); const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`; const arrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; const uploaded = await uploadFile(fileName, arrayBuffer); return uploaded.url; } async function normalizeAssistantImageMarkdown( text: string, imageSize = "2K" ): Promise { if (!text) return text; const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim; const markdownMatches = [...text.matchAll(markdownDataUrlRegex)]; let normalizedText = text; const uploadedUrls: string[] = []; if (markdownMatches.length > 0) { let rebuilt = ""; let cursor = 0; for (const match of markdownMatches) { const matchText = match[0]; const dataUrl = match[2]; const start = match.index ?? -1; if (start < 0) continue; rebuilt += normalizedText.slice(cursor, start); try { const uploadedUrl = await uploadDataUrlImage(dataUrl); uploadedUrls.push(uploadedUrl); rebuilt += `![thumb|||${uploadedUrl}|||${imageSize}](${uploadedUrl})`; } catch (error) { console.error("[Chat API] Failed to upload inline data URL image:", error); rebuilt += matchText; } cursor = start + matchText.length; } rebuilt += normalizedText.slice(cursor); normalizedText = rebuilt; } if (normalizedText.includes("data:image")) { const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim; const rawMatches = normalizedText.match(rawDataUrlRegex) ?? []; for (const rawDataUrl of rawMatches) { try { const uploadedUrl = await uploadDataUrlImage(rawDataUrl); uploadedUrls.push(uploadedUrl); normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl); } catch (error) { console.error("[Chat API] Failed to normalize raw data URL image:", error); } } } const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim; const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)]; if (markdownHttpMatches.length > 0) { let rebuilt = ""; let cursor = 0; for (const match of markdownHttpMatches) { const matchText = match[0]; const altText = match[1] || ""; const thumbnailUrl = (match[2] || "").trim(); const start = match.index ?? -1; if (start < 0) continue; rebuilt += normalizedText.slice(cursor, start); try { const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata( altText, thumbnailUrl ); let hostedThumbnailUrl = thumbnailUrl; let hostedOriginalUrl = originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl; try { hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl); } catch (error) { console.error("[Chat API] Failed to normalize remote thumbnail URL:", error); } if (originalUrl && originalUrl !== thumbnailUrl) { try { hostedOriginalUrl = await ensureHostedImageUrl(originalUrl); } catch (error) { console.error("[Chat API] Failed to normalize remote original URL:", error); hostedOriginalUrl = hostedThumbnailUrl || originalUrl; } } else { hostedOriginalUrl = hostedThumbnailUrl; } const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl; const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl; uploadedUrls.push(finalThumbnailUrl); uploadedUrls.push(finalOriginalUrl); rebuilt += `![thumb|||${finalOriginalUrl}|||${parsedImageSize || imageSize}](${finalThumbnailUrl})`; } catch (error) { console.error("[Chat API] Failed to parse remote image markdown metadata:", error); rebuilt += matchText; } cursor = start + matchText.length; } rebuilt += normalizedText.slice(cursor); normalizedText = rebuilt; } const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText); if (!hasMarkdownImage && uploadedUrls.length > 0) { const uniqueUrls = [...new Set(uploadedUrls)]; console.log( `[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)` ); return uniqueUrls .map((url) => `![thumb|||${url}|||${imageSize}](${url})`) .join("\n"); } return normalizedText; } function prependImageModelBadge(text: string, modelDisplayName?: string | null): string { const normalizedText = String(text || "").trim(); if (!normalizedText) return normalizedText; const normalizedModelName = String(modelDisplayName || "").trim(); if (!normalizedModelName) return normalizedText; if ( /^\s*[◈◇◆🔹⬡✦•·]\s*[^\n]+/im.test(normalizedText) || /^\s*(?:Routed to|Using model|使用模型|模型)\s*[::]/im.test(normalizedText) ) { return normalizedText; } return `◈ ${normalizedModelName}\n\n${normalizedText}`; } function resolveModelDisplayNameByModelId( models: Awaited> | undefined, modelId: string | undefined, ): string { if (!modelId || !models?.length) return ""; const match = models.find( (item) => item.model.modelId === modelId || item.model.id === modelId, ); return String(match?.model?.displayName || ""); } /** * 解析生图 Markdown 并保存图片记录到数据库 * 支持三种格式:thumb|||url|||size、thumb$$url$$size、thumb|url|size */ async function saveGeneratedImages( imageMarkdown: string, userId: string, chatId: string, messageId: string, prompt: string, modelId: string ): Promise { // 匹配 Markdown 图片格式: ![alt](url) const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g; const matches = [...imageMarkdown.matchAll(imageRegex)]; let savedImageCount = 0; for (const match of matches) { const altText = match[1] || ""; const thumbnailUrl = match[2]; const { originalUrl, imageSize } = parseImageAltMetadata(altText, thumbnailUrl); try { await db.insert(generatedImage).values({ userId, chatId, messageId, thumbnailUrl, originalUrl, imageSize, prompt, modelId, }); savedImageCount += 1; console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`); } catch (error) { console.error("[Gallery] Failed to save image record:", error); } } if (savedImageCount <= 0) return; try { const pointsConfig = await getPointsSystemConfig(); if (!pointsConfig.enabled) return; const modelConfig = await resolveBillingModelByReference(modelId); const unitCost = getModelCreditCost(modelConfig, "image"); if (unitCost <= 0) return; const totalCost = unitCost * savedImageCount; const consumeResult = await consumeUserPoints({ userId, amount: totalCost, reason: "image_generation", modelId: modelConfig?.id || modelId, chatId, messageId, note: `模型(${modelConfig?.displayName || modelId}) 生图 ${savedImageCount} 张`, }); if (!consumeResult.ok) { console.warn( `[Points] Failed to consume points after generation. user=${userId} model=${modelId} cost=${totalCost}` ); } else { console.log( `[Points] Consumed ${totalCost} points (${unitCost} x ${savedImageCount}). balance=${consumeResult.balance}` ); } } catch (error) { console.error("[Points] Failed to consume points after image generation:", error); } } const MODEL_UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; function getModelCreditCost(modelConfig: any, target: "image" | "video"): number { const config = (modelConfig as any)?.imageGenConfig || {}; const sourceValue = target === "video" ? config.creditCostPerVideo ?? config.creditCostPerImage ?? 0 : config.creditCostPerImage ?? 0; const normalized = Number(sourceValue); return Number.isFinite(normalized) ? Math.max(0, Math.round(normalized)) : 0; } async function resolveBillingModelByReference(modelRef: string): Promise { if (!modelRef) return undefined; if (MODEL_UUID_REGEX.test(modelRef)) { return await getAiModelById({ id: modelRef }).catch(() => undefined); } const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []); const matched = allModels.find( ({ model }: any) => model.id === modelRef || model.modelId === modelRef ); return matched?.model; } async function consumeGeneratedVideoPoints(params: { userId: string; chatId: string; messageId: string; modelRefs: Array; }) { try { const pointsConfig = await getPointsSystemConfig(); if (!pointsConfig.enabled) return; let billingModel: any | undefined; for (const ref of params.modelRefs) { if (!ref) continue; const resolved = await resolveBillingModelByReference(ref); if (!resolved) continue; billingModel = resolved; if (getModelCreditCost(resolved, "video") > 0) break; } if (!billingModel) return; const unitCost = getModelCreditCost(billingModel, "video"); if (unitCost <= 0) return; const consumeResult = await consumeUserPoints({ userId: params.userId, amount: unitCost, reason: "video_generation", modelId: billingModel.id, chatId: params.chatId, messageId: params.messageId, note: `模型(${billingModel.displayName || billingModel.modelId || billingModel.id}) 生视频 1 条`, }); if (!consumeResult.ok) { console.warn( `[Points] Failed to consume video points. user=${params.userId} model=${billingModel.id} cost=${unitCost}` ); } else { console.log( `[Points] Consumed ${unitCost} points for video. balance=${consumeResult.balance}` ); } } catch (error) { console.error("[Points] Failed to consume points after video generation:", error); } } export const maxDuration = 600; // 10 分钟,用于支持图片编辑等耗时任务 let globalStreamContext: ResumableStreamContext | null = null; const inFlightChatRequests = new Map(); const IN_FLIGHT_REQUEST_TTL_MS = 5 * 60 * 1000; const DUPLICATE_REQUEST_WAIT_MS = 2 * 60 * 1000; const SMART_AUTO_GRAPH_TIMEOUT_MS = 120000; const SMART_AUTO_FALLBACK_TIMEOUT_MS = 120000; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); class OperationTimeoutError extends Error { constructor(operation: string, timeoutMs: number) { super(`${operation} timed out after ${timeoutMs}ms`); this.name = "OperationTimeoutError"; } } async function withOperationTimeout( operation: Promise, operationName: string, timeoutMs: number, ): Promise { let timer: ReturnType | null = null; try { return await Promise.race([ operation, new Promise((_, reject) => { timer = setTimeout( () => reject(new OperationTimeoutError(operationName, timeoutMs)), timeoutMs, ); }), ]); } finally { if (timer) { clearTimeout(timer); } } } const getTokenlensCatalog = cache( async (): Promise => { try { return await fetchModels(); } catch (err) { console.warn( "TokenLens: catalog fetch failed, using default catalog", err ); return; // tokenlens helpers will fall back to defaultCatalog } }, ["tokenlens-catalog"], { revalidate: 24 * 60 * 60 } // 24 hours ); export function getStreamContext() { if (!globalStreamContext) { try { globalStreamContext = createResumableStreamContext({ waitUntil: after, }); } catch (error: any) { if (error.message.includes("REDIS_URL")) { console.log( " > Resumable streams are disabled due to missing REDIS_URL" ); } else { console.error(error); } } } return globalStreamContext; } export async function POST(request: Request) { let requestBody: PostRequestBody; let requestDedupKey: string | null = null; let acquiredInFlightLock = false; try { const json = await request.json(); console.log("[Chat API] Received request body:", JSON.stringify(json, null, 2)); requestBody = postRequestBodySchema.parse(json); } catch (error) { console.error("[Chat API] Schema validation failed:", error); return new ChatSDKError("bad_request:api").toResponse(); } try { const { id, message, selectedChatModel, selectedVisibilityType, hiddenPrompt, autoContext, deepThinkingEnabled, }: { id: string; message: ChatMessage; selectedChatModel: ChatModel["id"]; selectedVisibilityType: VisibilityType; hiddenPrompt?: string; autoContext?: string; deepThinkingEnabled?: boolean; } = requestBody; const now = Date.now(); for (const [key, startedAt] of inFlightChatRequests.entries()) { if (now - startedAt > IN_FLIGHT_REQUEST_TTL_MS) { inFlightChatRequests.delete(key); } } requestDedupKey = `${id}:${message.id}`; if (inFlightChatRequests.has(requestDedupKey)) { console.warn( `[Chat API] Duplicate in-flight request ignored (chatId=${id}, messageId=${message.id})` ); const duplicateStream = createUIMessageStream({ execute: async ({ writer }) => { await replayAssistantMessagesForDuplicateRequest({ chatId: id, userMessageId: message.id, writer, source: "duplicate-in-flight", }); }, }); return createSseResponse(duplicateStream); } inFlightChatRequests.set(requestDedupKey, now); acquiredInFlightLock = true; let finalModelId = selectedChatModel; // 🆕 存储智能模式的相关信息 let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复 let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词(不修改用户消息,只用于API调用) let smartAutoPromptFromVideoPlan = false; let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined; let smartAutoVideoPlan: AgentVideoPlan | undefined; let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined; let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined; let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL(支持多张) let smartAutoNeedsClarification = false; // 是否需要先追问用户 let smartAutoClarificationQuestion: string | undefined; let preferredAgentModelId: string | undefined; const deepThinkingOn = deepThinkingEnabled !== false; let shouldRunAgentConversation = finalModelId === "auto"; let shouldKeepSelectedModel = false; const requestedModelId = selectedChatModel; let requestedModelProviderType: string | undefined; if (!shouldRunAgentConversation && requestedModelId !== "auto") { const selectedModelIsUUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(requestedModelId); let selectedModelConfig = selectedModelIsUUID ? await getAiModelById({ id: requestedModelId }) : null; let selectedModelProviderType: string | undefined; if (!selectedModelConfig) { const enabledModels = await getEnabledAiModelsWithProvider(false); const matchedModel = enabledModels.find( ({ model }) => model.id === requestedModelId || model.modelId === requestedModelId ); if (matchedModel) { selectedModelConfig = matchedModel.model; selectedModelProviderType = matchedModel.provider.type as string; } } if (selectedModelConfig && !selectedModelProviderType) { const selectedProvider = await getAiProviderById({ id: selectedModelConfig.providerId, }); selectedModelProviderType = selectedProvider?.type as string | undefined; } requestedModelProviderType = selectedModelProviderType; if (selectedModelConfig) { const fixedImageProviderTypes = new Set([ "gemini", "huanke-gemini", "alibaba-image", "ruxa-image", "fal-image", "volcano-vision", ]); const isFixedVisualModel = !selectedModelConfig.isSubModel && (selectedModelConfig.supportsImageGen || (selectedModelProviderType ? isVideoProviderType(selectedModelProviderType) : false) || (selectedModelProviderType ? fixedImageProviderTypes.has(selectedModelProviderType) : false)); if (deepThinkingOn) { if (isAiModelAgentAuto(selectedModelConfig)) { shouldRunAgentConversation = true; shouldKeepSelectedModel = false; console.log( `[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}` ); } else if (isFixedVisualModel) { shouldRunAgentConversation = true; shouldKeepSelectedModel = true; preferredAgentModelId = selectedModelConfig.id; console.log( `[Smart Auto Mode] 🎯 Agent conversation enabled for fixed visual model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"}), preferredAgentModelId=${preferredAgentModelId}` ); } } else if (isFixedVisualModel || isAiModelAgentAuto(selectedModelConfig)) { console.log( `[Smart Auto Mode] ⏭️ Deep thinking disabled, skip agent analysis for: ${selectedModelConfig.displayName}` ); } } } const allowAgentVideoGeneration = Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType); // ========== Smart Auto Mode Logic ========== if (shouldRunAgentConversation) { console.log("[Smart Auto Mode] 🚀 Starting intelligent analysis..."); let autoTextForFallback = ""; let autoHasImageForFallback = false; let autoFromHistoryForFallback = false; try { // 🆕 首先获取历史消息(在 Smart Auto Mode 内部单独获取,避免依赖后面的 messagesFromDb) const smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id }); console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB"); // 1. 提取用户消息内容 const lastUserMessage = message; const textContent = lastUserMessage.parts .filter((p): p is { type: "text"; text: string } => p.type === "text") .map((p) => p.text) .join(" "); autoTextForFallback = textContent; const autoContextText = typeof autoContext === "string" ? autoContext.trim() : ""; const hasSelectedImageHint = /用户已选中\s*\d+\s*张图片/.test(autoContextText); const hasExplicitTargetHint = /第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(textContent); const hasActionableEditIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test( textContent ); // @ts-ignore - 检查各种可能的图片类型 let hasImage = lastUserMessage.parts.some( (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" ); autoHasImageForFallback = hasImage; // @ts-ignore - 提取图片URL let imageUrls = lastUserMessage.parts .filter((p: any) => p.type === "file" && p.url) .map((p: any) => p.url as string); // 🆕 历史图片上下文感知:如果用户没有发送新图片,自动使用历史中最近的图片(支持多张) // 逻辑:只要用户在对话中没有附上新图片,就默认继续处理之前的图片 if (!hasImage && smartAutoHistoryMessagesFromDb.length > 0) { console.log("[Smart Auto Mode] 🔍 No image in current message, auto-searching history..."); const explicitFollowLatestResult = /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果)/i.test( textContent ); const latestAssistantImages: string[] = []; const latestUserImages: string[] = []; const pushUniqueUrl = (bucket: string[], maybeUrl?: string) => { if (!maybeUrl || typeof maybeUrl !== "string") return; const normalized = maybeUrl.trim(); if (!/^https?:\/\//i.test(normalized)) return; if (!bucket.includes(normalized)) { bucket.push(normalized); } }; // 从历史消息中倒序查找最近的“AI生成图”和“用户上传图” for (let i = smartAutoHistoryMessagesFromDb.length - 1; i >= 0; i--) { const msg = smartAutoHistoryMessagesFromDb[i]; const parts = msg.parts as any[]; if (!parts) continue; if (msg.role === "assistant" && latestAssistantImages.length === 0) { for (const part of parts) { if (part.type === "text" && part.text) { const parsedImageUrls = extractOriginalImageUrlsFromText(part.text); for (const parsedUrl of parsedImageUrls) { pushUniqueUrl(latestAssistantImages, parsedUrl); } } } } if (msg.role === "user" && latestUserImages.length === 0) { for (const part of parts) { if ((part.type === "file" || part.type === "image") && part.url) { pushUniqueUrl(latestUserImages, part.url); } if (part.type === "image_url" && part.image_url?.url) { pushUniqueUrl(latestUserImages, part.image_url.url); } if (part.type === "text" && part.text) { const parsedImageUrls = extractOriginalImageUrlsFromText(part.text); for (const parsedUrl of parsedImageUrls) { pushUniqueUrl(latestUserImages, parsedUrl); } } } } if (latestAssistantImages.length > 0 && latestUserImages.length > 0) { break; } } if (explicitFollowLatestResult && latestAssistantImages.length > 0) { smartAutoHistoryImages = latestAssistantImages.slice(0, 4); console.log( "[Smart Auto Mode] 🖼️ Using latest generated image(s) by explicit instruction:", smartAutoHistoryImages.length ); } else if (latestUserImages.length > 0) { smartAutoHistoryImages = latestUserImages.slice(0, 4); console.log( "[Smart Auto Mode] 🖼️ Using latest user-uploaded image(s) to avoid iterative blur:", smartAutoHistoryImages.length ); } else if (latestAssistantImages.length > 0) { smartAutoHistoryImages = latestAssistantImages.slice(0, 4); console.log( "[Smart Auto Mode] 🖼️ Fallback to latest generated image(s):", smartAutoHistoryImages.length ); } if (smartAutoHistoryImages.length > 0) { hasImage = true; imageUrls = smartAutoHistoryImages; autoHasImageForFallback = true; autoFromHistoryForFallback = true; console.log("[Smart Auto Mode] ✅ Auto-using", smartAutoHistoryImages.length, "history image(s) for context continuity"); } else { console.log("[Smart Auto Mode] ℹ️ No image found in history, proceeding as pure text"); } } console.log("[Smart Auto Mode] Input:", { textContent: textContent.substring(0, 100), hasImage, imageCount: imageUrls.length, fromHistory: smartAutoHistoryImages.length > 0 }); // 🆕 从历史消息中提取对话文本(用于上下文理解) const historyMessages: { role: "user" | "assistant"; content: string }[] = []; for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条 const parts = msg.parts as any[]; if (parts && parts.length > 0) { // 提取文本内容 const textParts = parts.filter((p: any) => p.type === 'text'); const content = textParts.map((p: any) => p.text || '').join(' ').trim(); if (content) { historyMessages.push({ role: msg.role as "user" | "assistant", content: content.substring(0, 300), // 限制长度 }); } } } console.log("[Smart Auto Mode] History context:", historyMessages.length, "messages"); // 2. 使用 LangGraph 驱动的自动模式工作流 const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph"); const autoGraphResult = await withOperationTimeout( runAutoAgentGraph({ text: textContent, hasImage, imageUrls, imageFromHistory: smartAutoHistoryImages.length > 0, historyMessages, context: autoContextText || undefined, preferredModelId: preferredAgentModelId, }), "runAutoAgentGraph", SMART_AUTO_GRAPH_TIMEOUT_MS, ); if (autoGraphResult.analysis) { console.log("[Smart Auto Mode] Analysis result:", { intent: autoGraphResult.analysis.intent, subIntent: autoGraphResult.analysis.subIntent, confidence: autoGraphResult.analysis.confidence, needsClarification: autoGraphResult.analysis.needsClarification, }); smartAutoDetectedIntent = autoGraphResult.analysis.intent; smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification); smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0]; if ( smartAutoNeedsClarification && hasImage && hasActionableEditIntent && (hasSelectedImageHint || hasExplicitTargetHint) ) { smartAutoNeedsClarification = false; smartAutoClarificationQuestion = undefined; if (!smartAutoOptimizedPrompt && textContent.trim()) { smartAutoOptimizedPrompt = textContent.trim(); } if ( !smartAutoFriendlyMessage || /[??]$/.test(smartAutoFriendlyMessage.trim()) || /哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage) ) { smartAutoFriendlyMessage = "收到,我将按你指定的目标图直接执行。"; } console.log( "[Smart Auto Mode] ✅ Skip clarification due to explicit target/selection context" ); } } if (autoGraphResult.friendlyMessage) { smartAutoFriendlyMessage = autoGraphResult.friendlyMessage; console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage); } if ( !smartAutoNeedsClarification && hasImage && hasActionableEditIntent && (hasSelectedImageHint || hasExplicitTargetHint) && smartAutoFriendlyMessage && (/[??]$/.test(smartAutoFriendlyMessage.trim()) || /哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage)) ) { smartAutoFriendlyMessage = "收到,我将按你指定的目标图直接执行。"; console.log("[Smart Auto Mode] ✅ Replace clarification-like friendly reply"); } const autoIntent = autoGraphResult.analysis?.intent; const isImageTask = autoIntent === "image_gen" || autoIntent === "image_edit" || hasImage; const needsRicherAutoReply = isImageTask && !smartAutoNeedsClarification && !smartAutoFriendlyMessage; if (needsRicherAutoReply) { if (isImageTask) { smartAutoFriendlyMessage = autoGraphResult.analysis?.conversationalReply || "我先分析你这轮图片里的主体、风格和构图,再给你执行方案。"; } else { smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage, fromHistory: smartAutoHistoryImages.length > 0, }); } console.log("[Smart Auto Mode] 🧩 Filled missing conversational reply"); } if (autoGraphResult.optimizedPrompt) { smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt; console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", { original: textContent.substring(0, 50), optimized: autoGraphResult.optimizedPrompt.substring(0, 100), }); } if (autoGraphResult.videoPlan) { smartAutoVideoPlan = autoGraphResult.videoPlan; smartAutoRecommendedVideoParams = autoGraphResult.videoPlan.recommendedVideoParams; if (!smartAutoOptimizedPrompt && autoGraphResult.videoPlan.finalPrompt?.trim()) { smartAutoOptimizedPrompt = autoGraphResult.videoPlan.finalPrompt.trim(); smartAutoPromptFromVideoPlan = true; } console.log("[Smart Auto Mode] 🎬 Agent video plan ready:", { hasScript: Boolean(autoGraphResult.videoPlan.script), storyboardCount: autoGraphResult.videoPlan.storyboard?.length || 0, hasFinalPrompt: Boolean(autoGraphResult.videoPlan.finalPrompt), recommendedVideoParams: smartAutoRecommendedVideoParams, }); } if (autoGraphResult.recommendedImageParams) { smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams; console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams); } if (autoGraphResult.route && !shouldKeepSelectedModel) { const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType); if (routeIsVideo && !allowAgentVideoGeneration) { const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen"; smartAutoDetectedIntent = fallbackIntent; smartAutoVideoPlan = undefined; smartAutoRecommendedVideoParams = undefined; if (smartAutoPromptFromVideoPlan) { smartAutoOptimizedPrompt = textContent.trim() || undefined; smartAutoPromptFromVideoPlan = false; } smartAutoFriendlyMessage = smartAutoFriendlyMessage && !/视频/.test(smartAutoFriendlyMessage) ? smartAutoFriendlyMessage : "当前 Agent 默认按图片任务执行;如需生成视频,请先手动切换到“视频”模式。"; console.log( `[Smart Auto Mode] 🚫 Blocked auto video route (${autoGraphResult.route.displayName}), fallback to ${fallbackIntent}`, ); try { const { routeToModel } = await import("@/lib/ai/model-router"); const fallbackRoute = await routeToModel({ intent: fallbackIntent, subIntent: autoGraphResult.analysis?.subIntent, hasImage, }); if (!isVideoProviderType(fallbackRoute.providerType)) { finalModelId = fallbackRoute.id; console.log( `[Smart Auto Mode] ✅ Fallback routed to: ${fallbackRoute.displayName} (${fallbackRoute.reason})`, ); } else { console.warn( `[Smart Auto Mode] ⚠️ Fallback route still video (${fallbackRoute.displayName}), keep previous model`, ); } } catch (fallbackRouteError) { console.error("[Smart Auto Mode] Failed to resolve non-video fallback route:", fallbackRouteError); } } else { finalModelId = autoGraphResult.route.id; console.log( `[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`, ); } } else if (autoGraphResult.route && shouldKeepSelectedModel) { finalModelId = requestedModelId; console.log("[Smart Auto Mode] 🔒 Keep user-selected model, skip routing override"); } } catch (autoError) { // 智能分析失败时,回退到简单逻辑 console.error("[Smart Auto Mode] ❌ Error, falling back:", autoError); const isAutoTimeout = autoError instanceof OperationTimeoutError; if (isAutoTimeout) { console.warn( `[Smart Auto Mode] ⏱️ Auto-agent timed out after ${SMART_AUTO_GRAPH_TIMEOUT_MS}ms, using local fallback`, ); } try { const { analyzeIntent } = await import("@/lib/ai/smart-analyzer"); const rescueAnalysis = await withOperationTimeout( analyzeIntent({ text: autoTextForFallback, hasImage: autoHasImageForFallback || smartAutoHistoryImages.length > 0, imageUrls: smartAutoHistoryImages, imageFromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, historyMessages: [], context: undefined, preferredModelId: preferredAgentModelId, }), "fallbackAnalyzeIntent", SMART_AUTO_FALLBACK_TIMEOUT_MS, ); if (rescueAnalysis.conversationalReply?.trim()) { smartAutoFriendlyMessage = rescueAnalysis.conversationalReply.trim(); } } catch (rescueError) { console.error("[Smart Auto Mode] Rescue analysis failed:", rescueError); } const allModels = await getEnabledAiModelsWithProvider(); // @ts-ignore - 检查各种可能的图片类型 const hasImage = autoHasImageForFallback || message.parts.some( (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" ); let fallbackVisualSummary: string | undefined; if (isAutoTimeout && hasImage) { try { const extractCurrentImageUrls = (parts: any[]) => parts .map((part: any) => { if (!part) return null; if (part.type === "file") return part.url || part.data || null; if (part.type === "image") return part.url || part.image || null; if (part.type === "image_url") return part.image_url?.url || null; return null; }) .filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url)); const currentImageUrls = extractCurrentImageUrls(message.parts as any[]); const summaryImageUrls = currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages; if (summaryImageUrls.length > 0) { const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer"); const summary = await withOperationTimeout( summarizeImageForExecutionReport({ text: autoTextForFallback, imageUrls: summaryImageUrls, context: typeof autoContext === "string" ? autoContext : undefined, preferredModelId: preferredAgentModelId, }), "fallbackVisualSummary", SMART_AUTO_FALLBACK_TIMEOUT_MS, ); if (summary?.trim()) { fallbackVisualSummary = summary.trim(); } } } catch (summaryError) { console.warn("[Smart Auto Mode] Fallback visual summary failed:", summaryError); } } // 简单回退:有图选图片模型,无图选默认模型 let bestModel = null; if (hasImage) { bestModel = allModels.find( (m) => m.model.supportsImageGen && !m.model.isSubModel && !isVideoProviderType(m.provider?.type), ); if (isAutoTimeout) { smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage: true, fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, imageSummary: fallbackVisualSummary, }); } else { smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage: true, fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, imageSummary: fallbackVisualSummary, }); } } else { smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({ text: autoTextForFallback, hasImage: false, fromHistory: false, }); } if (!bestModel) { bestModel = allModels.find( (m) => m.model.isDefault && !m.model.isSubModel && !isVideoProviderType(m.provider?.type), ); } if (!bestModel && allModels.length > 0) { bestModel = allModels.find( (m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type), ) || allModels.find((m) => !m.model.isSubModel) || allModels[0]; } if (shouldKeepSelectedModel) { finalModelId = requestedModelId; console.log("[Smart Auto Mode] 🔒 Keep user-selected model in fallback mode"); } else if (bestModel) { finalModelId = bestModel.model.id; console.log(`[Smart Auto Mode] Fallback selected: ${bestModel.model.displayName}`); } else { return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse(); } } } // ========== End Smart Auto Mode Logic ========== if (shouldRunAgentConversation && !allowAgentVideoGeneration) { const finalModelIsUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(finalModelId); let finalProviderType: string | undefined; if (finalModelIsUuid) { const finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null); if (finalModelConfig) { finalProviderType = ( await getAiProviderById({ id: finalModelConfig.providerId }).catch(() => null) )?.type as string | undefined; } } if (isVideoProviderType(finalProviderType)) { const allModels = await getEnabledAiModelsWithProvider(false); const fallbackModel = allModels.find( (m) => !m.model.isSubModel && m.model.supportsImageGen && !isVideoProviderType(m.provider?.type), ) || allModels.find( (m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type), ); if (fallbackModel) { const hasImageInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); smartAutoDetectedIntent = hasImageInput ? "image_edit" : "image_gen"; finalModelId = fallbackModel.model.id; smartAutoVideoPlan = undefined; smartAutoRecommendedVideoParams = undefined; console.log( `[Smart Auto Mode] 🚫 Safety override: replace video model (${finalProviderType}) with ${fallbackModel.model.displayName}`, ); } } } if ( shouldRunAgentConversation && !smartAutoNeedsClarification && !smartAutoFriendlyMessage?.trim() ) { const latestUserText = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" "); const hasCurrentImageInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); const hasHistoryImageInput = smartAutoHistoryImages.length > 0; smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ text: latestUserText, hasImage: hasCurrentImageInput || hasHistoryImageInput, fromHistory: !hasCurrentImageInput && hasHistoryImageInput, }); console.log("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback"); } if (shouldRunAgentConversation && !smartAutoNeedsClarification) { const latestUserText = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" "); const hasCurrentImageInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); const hasHistoryImageInput = smartAutoHistoryImages.length > 0; smartAutoFriendlyMessage = ensureStructuredAgentFriendlyMessage({ message: smartAutoFriendlyMessage, text: latestUserText, hasImage: hasCurrentImageInput || hasHistoryImageInput, fromHistory: !hasCurrentImageInput && hasHistoryImageInput, }); } const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams); const hasUserCustomVideoParams = hasUserCustomizedVideoParams(requestBody.videoParams); const shouldApplyAgentImageParams = shouldRunAgentConversation && !smartAutoNeedsClarification && (smartAutoDetectedIntent === "image_gen" || smartAutoDetectedIntent === "image_edit"); const shouldApplyAgentVideoParams = shouldRunAgentConversation && !smartAutoNeedsClarification && allowAgentVideoGeneration && smartAutoDetectedIntent === "video_gen"; const effectiveImageParams = shouldApplyAgentImageParams ? mergeImageParamsWithAgentRecommendation({ incoming: requestBody.imageParams, recommended: smartAutoRecommendedImageParams, intent: smartAutoDetectedIntent, hasUserCustomSettings: hasUserCustomImageParams, }) : requestBody.imageParams; const effectiveVideoParams = shouldApplyAgentVideoParams ? mergeVideoParamsWithAgentRecommendation({ incoming: requestBody.videoParams, recommended: smartAutoRecommendedVideoParams, hasUserCustomSettings: hasUserCustomVideoParams, }) : requestBody.videoParams; const defaultImageQualityParams = { aspectRatio: "auto", imageSize: smartAutoDetectedIntent === "image_edit" ? "4K" : "2K", imageCount: 1, } as const; if (shouldApplyAgentImageParams) { console.log("[Smart Auto Mode] 🎯 Effective image params:", { incoming: requestBody.imageParams, recommended: smartAutoRecommendedImageParams, merged: effectiveImageParams, hasUserCustomImageParams, }); } if (shouldApplyAgentVideoParams) { console.log("[Smart Auto Mode] 🎯 Effective video params:", { incoming: requestBody.videoParams, recommended: smartAutoRecommendedVideoParams, merged: effectiveVideoParams, hasUserCustomVideoParams, hasVideoPlan: Boolean(smartAutoVideoPlan), }); } if (shouldRunAgentConversation && smartAutoFriendlyMessage) { smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({ message: smartAutoFriendlyMessage, userImageParams: requestBody.imageParams, hasUserCustomImageParams, }); } if (shouldRunAgentConversation && !smartAutoNeedsClarification) { const currentImageUrls = extractMessageImageUrls(message.parts as any[]); const summaryImageUrls = currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages; const shouldRequireImageAnalysis = summaryImageUrls.length > 0; if (shouldRequireImageAnalysis) { try { const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer"); const latestUserText = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" "); let summary: string | null = null; const summaryTimeouts = [120000]; for (const timeoutMs of summaryTimeouts) { summary = await withOperationTimeout( summarizeImageForExecutionReport({ text: latestUserText, imageUrls: summaryImageUrls, context: typeof autoContext === "string" ? autoContext : undefined, preferredModelId: preferredAgentModelId, }), "enrichImageAnalysisSummary", timeoutMs, ); if (summary?.trim()) break; } if (summary?.trim()) { const baseMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({ text: latestUserText, hasImage: true, fromHistory: currentImageUrls.length === 0 && summaryImageUrls.length > 0, imageSummary: summary.trim(), }); smartAutoFriendlyMessage = ensureImageAnalysisSummarySection(baseMessage, summary.trim()); console.log( "[Smart Auto Mode] ✅ Image analysis summary ready:", summary.trim().slice(0, 160), ); } else if (smartAutoFriendlyMessage && shouldEnrichImageAnalysisText(smartAutoFriendlyMessage)) { console.warn("[Smart Auto Mode] ⚠️ Image summary unavailable, keep original analysis text"); } } catch (enrichError) { console.warn("[Smart Auto Mode] Skip image analysis enrichment:", enrichError); } } } if (shouldRunAgentConversation && smartAutoFriendlyMessage) { smartAutoFriendlyMessage = normalizeAgentReportForDisplay(smartAutoFriendlyMessage); } const session = await auth(); if (!session?.user) { return new ChatSDKError("unauthorized:chat").toResponse(); } const userType: UserType = session.user.type; const messageCount = await getMessageCountByUserId({ id: session.user.id, differenceInHours: 24, }); if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) { return new ChatSDKError("rate_limit:chat").toResponse(); } // ========== 使用时间限制验证(后端安全检查) ========== try { // 从数据库获取网站设置 const { siteSettings } = await import("@/lib/db/schema"); const [settings] = await db.select().from(siteSettings).limit(1); if (settings?.usageTimeEnabled) { const now = new Date(); const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`; const startTime = settings.usageTimeStart || "08:00"; const endTime = settings.usageTimeEnd || "20:00"; const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime; if (!isWithinUsageTime) { console.log(`[Chat API] ⛔ 访问被拒绝:当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`); return NextResponse.json( { error: "usage_time_restricted", message: settings.usageTimeMessage || "当前不在系统使用时间段内", currentTime, allowedTime: `${startTime} - ${endTime}` }, { status: 403 } ); } console.log(`[Chat API] ✅ 时间验证通过:${currentTime} 在允许范围 ${startTime}-${endTime}`); } } catch (timeCheckError) { console.error("[Chat API] ⚠️ 时间验证失败(继续执行):", timeCheckError); // 如果时间检查失败,继续执行(避免因检查逻辑错误导致系统不可用) } let chat = await getChatById({ id }); let messagesFromDb: DBMessage[] = []; if (chat) { if (chat.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } // Only fetch messages if chat already exists messagesFromDb = await getMessagesByChatId({ id }); } else { const title = await generateTitleFromUserMessage({ message, }); try { await saveChat({ id, userId: session.user.id, title, visibility: selectedVisibilityType, }); } catch (error) { const racedChat = await getChatById({ id }); if (!racedChat) { throw error; } if (racedChat.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } console.warn("[Chat API] saveChat conflicted, reused existing chat:", { chatId: id, userId: session.user.id, }); } chat = await getChatById({ id }); if (!chat) { throw new ChatSDKError("bad_request:database", "Failed to save chat"); } if (chat.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } // In a race scenario another request may have already written messages. messagesFromDb = await getMessagesByChatId({ id }); } const uiMessages = [...convertToUIMessages(messagesFromDb), message]; const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : ""; const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : ""; const uiMessagesForModel = hiddenPromptText ? applyHiddenPromptToMessages(uiMessages, hiddenPromptText) : uiMessages; const { longitude, latitude, city, country } = geolocation(request); const requestHints: RequestHints = { longitude, latitude, city, country, }; const insertedUserMessage = await db .insert(messageTable) .values({ chatId: id, id: message.id, role: "user", parts: message.parts as any, attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }) .onConflictDoNothing({ target: [messageTable.id], }) .returning({ id: messageTable.id }); if (insertedUserMessage.length === 0) { console.warn( `[Chat API] Duplicate user message ignored (chatId=${id}, messageId=${message.id})` ); const duplicateStream = createUIMessageStream({ execute: async ({ writer }) => { await replayAssistantMessagesForDuplicateRequest({ chatId: id, userMessageId: message.id, writer, source: "duplicate-user-message", }); }, }); return createSseResponse(duplicateStream); } const streamId = generateUUID(); await createStreamId({ streamId, chatId: id }); if (shouldRunAgentConversation && smartAutoNeedsClarification) { const hasCurrentImageInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); const hasHistoryImageInput = smartAutoHistoryImages.length > 0; const latestUserText = message.parts .filter((part): part is { type: "text"; text: string } => part.type === "text") .map((part) => part.text) .join(" ") .trim(); if (hasCurrentImageInput || hasHistoryImageInput || latestUserText.length > 0) { smartAutoNeedsClarification = false; smartAutoClarificationQuestion = undefined; if ( !smartAutoFriendlyMessage || /[??]$/.test(smartAutoFriendlyMessage.trim()) || /哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage) ) { smartAutoFriendlyMessage = "收到,我将按你的当前需求直接执行。"; } console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)"); } } if (shouldRunAgentConversation && smartAutoNeedsClarification) { const clarificationMessage = smartAutoClarificationQuestion || smartAutoFriendlyMessage || "我先确认一下,你希望我优先处理哪一部分?"; const clarificationStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: clarificationMessage }], }), }); await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: clarificationMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); }, }); return createSseResponse(clarificationStream); } let finalMergedUsage: AppUsage | undefined; // 获取启用的工具列表 const enabledTools = await getEnabledTools(); // 检查是否是 UUID 格式(数据库中的模型 ID) const isUUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( finalModelId ); // 获取语言模型实例 let languageModel; let actualModelId: string; let resolvedModelSupportsImageGen = false; let resolvedProviderType: string | undefined; if (isUUID) { // 从数据库获取动态配置的模型 let targetModelId = finalModelId; // 🆕 检查是否是视频主模型,如果是,根据 videoParams.mode 选择子模型 const aiModelData = await getAiModelById({ id: finalModelId }); const aiModelProvider = aiModelData ? await getAiProviderById({ id: aiModelData.providerId }).catch(() => null) : null; console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel }); if (aiModelData && !aiModelData.isSubModel && aiModelProvider?.type === "volcano-video") { // 检查是否有子模型 const subModels = await getSubModelsByParentModelId(finalModelId); console.log("[Chat API] Found sub-models:", subModels.length); if (subModels.length > 0) { // 这是一个主模型,需要根据 videoParams 选择子模型 const videoParams = effectiveVideoParams; console.log("[Chat API] Video params:", videoParams); // 如果没有 videoParams 或没有 mode,使用默认模式 const mode = (videoParams?.mode || "text-to-video") as VideoModeType; console.log("[Chat API] Using mode:", mode); const matchedSubModel = selectBestVideoSubModelForMode(subModels as any, mode); if (matchedSubModel) { console.log( `[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName} (${matchedSubModel.model.modelId})` ); targetModelId = matchedSubModel.model.id; } else { console.warn(`[Chat API] 未找到可用子模型,继续使用主模型`); } } } const dynamicModel = await getLanguageModelById(targetModelId); if (!dynamicModel) { return new ChatSDKError( "bad_request:api", "Model not found or disabled" ).toResponse(); } languageModel = dynamicModel; // 对于动态模型,我们需要从数据库获取 modelId const finalModelData = await getAiModelById({ id: targetModelId }); actualModelId = finalModelData?.modelId ?? targetModelId; resolvedModelSupportsImageGen = Boolean(finalModelData?.supportsImageGen); if (finalModelData) { resolvedProviderType = ( await getAiProviderById({ id: finalModelData.providerId }).catch(() => null) )?.type; } } else { // 使用预定义的模型 languageModel = myProvider.languageModel(finalModelId); actualModelId = languageModel.modelId; resolvedProviderType = (languageModel as any)?.provider; } // ========== 图片生成配额检查 ========== const isImageGenerationModel = (languageModel as any).provider === "gemini-image" || (languageModel as any).provider === "volcano-vision" || (languageModel as any).provider === "alibaba-image" || (languageModel as any).provider === "ruxa-image" || (languageModel as any).provider === "fal-image" || (languageModel as any).provider === "huanke-gemini"; const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( finalModelId ); const currentModelConfig = isUuidModelId ? await getAiModelById({ id: finalModelId }).catch(() => undefined) : undefined; const modelCreditCostPerImage = Number( (currentModelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0 ); const modelCreditCostPerVideo = getModelCreditCost(currentModelConfig, "video"); if (isImageGenerationModel) { const quota = await checkUserImageQuota(session.user.id); if (!quota.allowed) { console.log(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`); return NextResponse.json( { error: "quota_exceeded", message: `本月图片生成配额已用完(已使用 ${quota.used}/${quota.limit} 张),配额将于下月1日重置,如需更多配额请联系管理员。`, used: quota.used, limit: quota.limit, }, { status: 429 } ); } console.log(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`); const pointsConfig = await getPointsSystemConfig(); const normalizedUnitCost = Number.isFinite(modelCreditCostPerImage) ? Math.max(0, Math.round(modelCreditCostPerImage)) : 0; if (pointsConfig.enabled && normalizedUnitCost > 0 && pointsConfig.blockOnInsufficient !== false) { const requestedImageCount = Math.max( 1, Math.round( Number( effectiveImageParams?.imageCount || defaultImageQualityParams.imageCount || 1 ) ) ); const expectedCost = normalizedUnitCost * requestedImageCount; const userData = await getUserById({ id: session.user.id }); const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0)); if (currentBalance < expectedCost) { return NextResponse.json( { error: "insufficient_points", message: `积分不足,当前余额 ${currentBalance},本次预计消耗 ${expectedCost}(${normalizedUnitCost} x ${requestedImageCount} 张)。`, balance: currentBalance, expectedCost, unitCost: normalizedUnitCost, }, { status: 402 } ); } } } const isVideoGenerationModel = [ "qianwen-video", "fal-video", "volcano-video", "doubao-video", ].includes(String(resolvedProviderType || (languageModel as any).provider || "")); if (isVideoGenerationModel) { const pointsConfig = await getPointsSystemConfig(); const normalizedVideoUnitCost = Number.isFinite(modelCreditCostPerVideo) ? Math.max(0, Math.round(modelCreditCostPerVideo)) : 0; if ( pointsConfig.enabled && normalizedVideoUnitCost > 0 && pointsConfig.blockOnInsufficient !== false ) { const expectedCost = normalizedVideoUnitCost; const userData = await getUserById({ id: session.user.id }); const currentBalance = Math.max(0, Number(userData?.pointsBalance || 0)); if (currentBalance < expectedCost) { return NextResponse.json( { error: "insufficient_points", message: `积分不足,当前余额 ${currentBalance},本次预计消耗 ${expectedCost}(${normalizedVideoUnitCost} x 1 条视频)。`, balance: currentBalance, expectedCost, unitCost: normalizedVideoUnitCost, }, { status: 402 } ); } } } // ========== Gemini 图片生成独立处理 ========== // 支持 gemini-image 和 huanke-gemini 两种图片生成 provider const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider); if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", effectiveImageParams); // 🆕 获取备用模型配置 const currentModelConfig = await getAiModelById({ id: finalModelId }); const configuredFallbacks = [ currentModelConfig?.fallbackModelId1, currentModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const sameProviderFallbacks = currentModelConfig ? allModels .filter(m => m.provider.id === currentModelConfig.providerId && m.model.supportsImageGen && m.model.isEnabled && !m.model.isSubModel && m.model.modelId !== actualModelId ) .map(m => m.model.modelId) : []; const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks])); console.log("[Chat API] Fallback model IDs:", fallbackModelIds); const geminiStream = createUIMessageStream({ execute: async ({ writer }) => { // 🆕 如果是智能模式且有对话回复,先发送对话消息让用户看到 // 在消息末尾加上特殊标记,告诉前端"图片正在生成中" const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writeAssistantTextChunk( writer, conversationMessage, "gemini:conversation", chatMessageId, ); // 保存对话消息到数据库(不带标记,纯对话) await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 🆕 尝试生成图片的函数(支持备用模型切换) const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise => { console.log(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`); // 只传递最后一条用户消息,避免上下文历史干扰生图效果 const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop(); let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel; messagesForGemini = applyOptimizedPromptToLatestUserMessage( messagesForGemini, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages); console.log("[Chat API] Gemini: Using only last user message (no history context)"); // 转换消息:gemini-image 需要 base64 格式,huanke-gemini 需要 URL 格式 const providerForConvert = (languageModel as any).provider === "huanke-gemini" ? "huanke-gemini" : "gemini"; const modelMessages = await convertMessagesWithImages(messagesForGemini, providerForConvert); console.log("[Chat API] Message conversion provider:", providerForConvert); // 调用 doGenerate 获取图片(如果是备用模型,传入 overrideModelId) const result = await (languageModel as any).doGenerate({ messages: modelMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, }, overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID }); return result.text || "图片生成失败"; }; let stopKeepalive: (() => void) | null = null; try { let imageMarkdown = ""; // 初始化为空字符串 let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记(用于前端显示占位图) writeAssistantTextChunk( writer, "", "gemini:image-start", ); stopKeepalive = startStreamKeepalive(writer, "gemini-image"); try { // 首先尝试主模型 imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Primary model failed:", primaryError); // 🆕 尝试备用模型(使用同一个 provider,但切换 modelId) let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Trying fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Fallback model ${fallbackModelId} failed:`, fallbackError); } } // 如果所有备用模型都失败,抛出原始错误触发火山引擎故障转移 if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Gemini generated, length:", imageMarkdown.length); console.log("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200)); console.log("[Chat API] Used model ID:", usedModelId); const usedModelDisplayName = resolveModelDisplayNameByModelId(allModels, usedModelId) || currentModelConfig?.displayName || actualModelId; const imageMarkdownWithModel = prependImageModelBadge( imageMarkdown, usedModelDisplayName, ); // 生成一个统一的消息 ID const messageId = generateUUID(); // 写入消息到流 - 使用正确的 parts 格式 writeAssistantTextChunk( writer, imageMarkdownWithModel, "gemini:image-result", messageId, ); // 🆕 图片生成结束标记(成功) writeAssistantTextChunk( writer, "", "gemini:image-end-success", ); // 保存消息到数据库(现在是 S3 URL,可以完整保存) try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdownWithModel }], attachments: [], // 必须提供 attachments 字段(数据库 notNull) createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Gemini message saved successfully!"); // 提取用户 prompt(最后一条用户消息的文本) const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdownWithModel, session.user.id, id, messageId, userPrompt, finalModelId ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError); console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300)); } } catch (error) { console.error("[Chat API] Gemini generation error:", error); console.log("[Chat API] Attempting failover to Volcano Vision..."); // 故障转移:尝试使用火山引擎模型 try { // 查找火山引擎模型 const allModels = await getEnabledAiModelsWithProvider(); console.log("[Chat API] Total enabled models:", allModels.length); const volcanoModel = allModels.find( (m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled ); if (volcanoModel) { console.log("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id); // 获取火山引擎模型实例 const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id); if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") { console.log("[Chat API] Executing Volcano Vision failover..."); // 转换消息(火山引擎会自动处理 Base64) const volcanoMessages = await convertMessagesWithImages( injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages), "volcano" ); // 调用火山引擎生成(故障转移时只生成 1 张,保持与 Gemini 一致) const volcanoResult = await (volcanoLanguageModel as any).doGenerate({ messages: volcanoMessages, maxImages: 1, // 故障转移时只生成 1 张 imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, }); const imageMarkdown = volcanoResult.text || "图片生成失败"; const imageMarkdownWithModel = prependImageModelBadge( imageMarkdown, volcanoModel.model.displayName || volcanoModel.model.modelId, ); console.log("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length); // 生成消息 ID const messageId = generateUUID(); // 写入消息到流 writeAssistantTextChunk( writer, imageMarkdownWithModel, "gemini:volcano-failover-result", messageId, ); // 🆕 图片生成结束标记(故障转移成功) writeAssistantTextChunk( writer, "", "gemini:volcano-failover-end", ); // 保存消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdownWithModel }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Volcano Vision failover message saved!"); // 提取用户 prompt const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdownWithModel, session.user.id, id, messageId, userPrompt, volcanoModel.model.id ); } catch (saveError) { console.error("[Chat API] Failed to save failover message:", saveError); } return; // 成功,直接返回 } } console.log("[Chat API] No Volcano Vision model available for failover"); } catch (failoverError) { console.error("[Chat API] Failover to Volcano Vision also failed:", failoverError); } // 如果故障转移也失败,返回错误消息 const errorMessageId = generateUUID(); writeAssistantTextChunk( writer, "图片生成失败,请稍后重试", "gemini:error", errorMessageId, ); // 🆕 图片生成结束标记(失败) writeAssistantTextChunk( writer, "", "gemini:image-end-failed", ); // 尝试保存错误消息 try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: "图片生成失败,请稍后重试" }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } finally { stopKeepalive?.(); } }, }); return createSseResponse(geminiStream); } // ========== Gemini 处理结束 ========== // ========== 阿里云千问图片生成独立处理 ========== const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image"].includes((languageModel as any).provider); if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using image generation provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", effectiveImageParams); // 🆕 获取模型显示名称,用于错误提示 const alibabaModelConfig = await getAiModelById({ id: finalModelId }); const modelDisplayName = alibabaModelConfig?.displayName || actualModelId; const configuredFallbacks = [ alibabaModelConfig?.fallbackModelId1, alibabaModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const sameProviderFallbacks = alibabaModelConfig ? allModels .filter(m => m.provider.id === alibabaModelConfig.providerId && m.model.supportsImageGen && m.model.isEnabled && !m.model.isSubModel && m.model.modelId !== actualModelId ) .map(m => m.model.modelId) : []; const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks])); console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds); const alibabaStream = createUIMessageStream({ execute: async ({ writer }) => { let stopKeepalive: (() => void) | null = null; try { // 🆕 如果是智能模式且有对话回复,先发送对话消息让用户看到 const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writeAssistantTextChunk( writer, conversationMessage, "alibaba:conversation", chatMessageId, ); // 保存对话消息到数据库(不带标记,纯对话) await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 只传递最后一条用户消息 const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop(); let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessagesForModel; console.log("[Chat API] Alibaba: Using only last user message"); // 🆕 如果有智能模式优化后的提示词,替换消息中的文本(仅用于API调用) if (smartAutoOptimizedPrompt) { console.log("[Chat API] Using optimized prompt for API call"); messagesForAlibaba = applyOptimizedPromptToLatestUserMessage( messagesForAlibaba, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); } messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages); const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => { // 转换消息(图片编辑模型统一使用 URL 输入) const providerForConvert = (languageModel as any).provider === "ruxa-image" ? "ruxa-image" : (languageModel as any).provider === "fal-image" ? "fal-image" : "alibaba-image"; const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert); // 调用 doGenerate 获取图片,传递模型显示名称 const result = await (languageModel as any).doGenerate({ messages: modelMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, modelDisplayName, // 🆕 传递模型显示名称 overrideModelId: isRetry ? modelIdToUse : undefined, }); if (result?.finishReason === "error") { throw new Error(result.text || "图片生成失败"); } return result.text || "图片生成失败"; }; let imageMarkdown = ""; let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记(用于前端显示占位图) writeAssistantTextChunk( writer, "", "alibaba:image-start", ); stopKeepalive = startStreamKeepalive(writer, "alibaba-image"); try { imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Alibaba primary model failed:", primaryError); let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Alibaba fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Alibaba fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Alibaba fallback model ${fallbackModelId} failed:`, fallbackError); } } if (!fallbackSuccess) { const providerPriority: Record = { "volcano-vision": 0, "alibaba-image": 1, "ruxa-image": 2, "fal-image": 3, gemini: 4, "huanke-gemini": 5, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, currentProviderId: alibabaModelConfig?.providerId, providerPriority, }); for (const candidate of crossProviderCandidates) { try { console.log( `[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})` ); const candidateLanguageModel = await getLanguageModelById(candidate.model.id); if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") { console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`); continue; } const convertProvider = candidate.provider.type === "ruxa-image" ? "ruxa-image" : candidate.provider.type === "fal-image" ? "fal-image" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" ? "huanke-gemini" : candidate.provider.type === "volcano-vision" ? "volcano" : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForAlibaba, convertProvider ); const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, modelDisplayName: candidate.model.displayName, overrideModelId: candidate.model.modelId, }); if (candidateResult?.finishReason === "error") { throw new Error(candidateResult.text || "图片生成失败"); } imageMarkdown = candidateResult.text || "图片生成失败"; usedModelId = candidate.model.modelId; fallbackSuccess = true; console.log( `[Chat API] ✅ Cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})` ); break; } catch (crossError) { const crossErrorMessage = normalizeFailoverErrorMessage(crossError); if (shouldCooldownImageFallbackModel(crossErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage); } console.error( `[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError ); } } } if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Alibaba generated, length:", imageMarkdown.length); console.log("[Chat API] Alibaba used model:", usedModelId); const usedModelDisplayName = resolveModelDisplayNameByModelId(allModels, usedModelId) || alibabaModelConfig?.displayName || actualModelId; const imageMarkdownWithModel = prependImageModelBadge( imageMarkdown, usedModelDisplayName, ); // 生成消息 ID(图片消息) const messageId = generateUUID(); // 写入图片消息到流 writeAssistantTextChunk( writer, imageMarkdownWithModel, "alibaba:image-result", messageId, ); // 🆕 图片生成结束标记(成功) writeAssistantTextChunk( writer, "", "alibaba:image-end-success", ); // 保存图片消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdownWithModel }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Alibaba message saved successfully!"); // 提取用户 prompt const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdownWithModel, session.user.id, id, messageId, userPrompt, finalModelId ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Alibaba message:", saveError); } } catch (error) { console.error("[Chat API] Alibaba generation error:", error); // 返回错误消息 const errorMessageId = generateUUID(); const errorText = error instanceof Error ? error.message : "图片生成失败,请稍后重试"; writeAssistantTextChunk( writer, errorText, "alibaba:error", errorMessageId, ); // 🆕 图片生成结束标记(失败) writeAssistantTextChunk( writer, "", "alibaba:image-end-failed", ); try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorText }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } finally { stopKeepalive?.(); } }, }); return createSseResponse(alibabaStream); } // ========== 阿里云图片生成处理结束 ========== // ========== 火山引擎视觉图片生成独立处理 ========== if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Volcano Vision image generation"); // 🆕 检测是否有标注图片,如果有则在消息中注入系统提示 const hasAnnotation = hasAnnotatedImage(uiMessages); let messagesForVolcano = applyOptimizedPromptToLatestUserMessage( uiMessagesForModel, smartAutoOptimizedPrompt, hiddenPromptPrefix, ); const volcanoModelConfig = await getAiModelById({ id: finalModelId }); const configuredFallbacks = [ volcanoModelConfig?.fallbackModelId1, volcanoModelConfig?.fallbackModelId2, ].filter(Boolean) as string[]; const allModels = await getEnabledAiModelsWithProvider(); const sameProviderFallbacks = volcanoModelConfig ? allModels .filter(m => m.provider.id === volcanoModelConfig.providerId && m.model.supportsImageGen && m.model.isEnabled && !m.model.isSubModel && m.model.modelId !== actualModelId ) .map(m => m.model.modelId) : []; const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks])); console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds); if (hasAnnotation) { console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片,将注入系统提示"); // 创建消息副本,修改最后一条用户消息的文本 messagesForVolcano = messagesForVolcano.map((msg, idx) => { if (msg.role === "user" && idx === messagesForVolcano.length - 1) { // 修改最后一条用户消息 const modifiedParts = msg.parts.map((part) => { if (part.type === "text" && part.text) { return { ...part, text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求:" + part.text, }; } return part; }); return { ...msg, parts: modifiedParts }; } return msg; }); } messagesForVolcano = injectHistoryImagesToLastUserMessage(messagesForVolcano, smartAutoHistoryImages); const volcanoStream = createUIMessageStream({ execute: async ({ writer }) => { let stopKeepalive: (() => void) | null = null; try { // 🆕 如果是智能模式且有对话回复,先发送对话消息让用户看到 const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); console.log("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 writeAssistantTextChunk( writer, conversationMessage, "volcano:conversation", chatMessageId, ); // 保存对话消息到数据库(不带标记,纯对话) await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: smartAutoFriendlyMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } // 转换消息,火山引擎支持 URL 和 base64 const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano"); const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => { const result = await (languageModel as any).doGenerate({ messages: modelMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, overrideModelId: isRetry ? modelIdToUse : undefined, }); if (result?.finishReason === "error") { throw new Error(result.text || "图片生成失败"); } return result.text || "图片生成失败"; }; let imageMarkdown = ""; let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记(用于前端显示占位图) writeAssistantTextChunk( writer, "", "volcano:image-start", ); stopKeepalive = startStreamKeepalive(writer, "volcano-image"); try { imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { console.error("[Chat API] Volcano primary model failed:", primaryError); let fallbackSuccess = false; for (const fallbackModelId of fallbackModelIds) { try { console.log(`[Chat API] 🔄 Volcano fallback model: ${fallbackModelId}`); imageMarkdown = await tryGenerateImage(fallbackModelId, true); usedModelId = fallbackModelId; fallbackSuccess = true; console.log(`[Chat API] ✅ Volcano fallback model ${fallbackModelId} succeeded!`); break; } catch (fallbackError) { console.error(`[Chat API] ❌ Volcano fallback model ${fallbackModelId} failed:`, fallbackError); } } if (!fallbackSuccess) { const providerPriority: Record = { "alibaba-image": 0, "ruxa-image": 1, "fal-image": 2, gemini: 3, "huanke-gemini": 4, "volcano-vision": 5, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, currentProviderId: volcanoModelConfig?.providerId, providerPriority, }); for (const candidate of crossProviderCandidates) { try { console.log( `[Chat API] 🔁 Volcano cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})` ); const candidateLanguageModel = await getLanguageModelById(candidate.model.id); if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") { console.log(`[Chat API] Skip candidate without doGenerate: ${candidate.model.id}`); continue; } const convertProvider = candidate.provider.type === "ruxa-image" ? "ruxa-image" : candidate.provider.type === "fal-image" ? "fal-image" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" ? "huanke-gemini" : candidate.provider.type === "volcano-vision" ? "volcano" : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForVolcano, convertProvider ); const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, imageParams: effectiveImageParams || { aspectRatio: defaultImageQualityParams.aspectRatio, imageSize: defaultImageQualityParams.imageSize, imageCount: defaultImageQualityParams.imageCount, }, modelDisplayName: candidate.model.displayName, overrideModelId: candidate.model.modelId, }); if (candidateResult?.finishReason === "error") { throw new Error(candidateResult.text || "图片生成失败"); } imageMarkdown = candidateResult.text || "图片生成失败"; usedModelId = candidate.model.modelId; fallbackSuccess = true; console.log( `[Chat API] ✅ Volcano cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})` ); break; } catch (crossError) { const crossErrorMessage = normalizeFailoverErrorMessage(crossError); if (shouldCooldownImageFallbackModel(crossErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage); } console.error( `[Chat API] ❌ Volcano cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError ); } } } if (!fallbackSuccess) { throw primaryError; } } console.log("[Chat API] Volcano Vision generated, length:", imageMarkdown.length); console.log("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200)); const usedModelDisplayName = resolveModelDisplayNameByModelId(allModels, usedModelId) || volcanoModelConfig?.displayName || actualModelId; const imageMarkdownWithModel = prependImageModelBadge( imageMarkdown, usedModelDisplayName, ); // 生成消息 ID(图片消息) const messageId = generateUUID(); // 写入图片消息到流 writeAssistantTextChunk( writer, imageMarkdownWithModel, "volcano:image-result", messageId, ); // 🆕 图片生成结束标记(成功) writeAssistantTextChunk( writer, "", "volcano:image-end-success", ); // 保存图片消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: imageMarkdownWithModel }], attachments: [], // 必须提供 attachments 字段(数据库 notNull) createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); console.log("[Chat API] ✅ Volcano Vision message saved successfully!"); // 提取用户 prompt const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || ""; // 保存图片记录到 GeneratedImage 表 await saveGeneratedImages( imageMarkdownWithModel, session.user.id, id, messageId, userPrompt, finalModelId ); } catch (saveError) { console.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError); console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300)); } } catch (error) { console.error("[Chat API] Volcano Vision generation error:", error); const errorMessageId = generateUUID(); writeAssistantTextChunk( writer, "图片生成失败,请稍后重试", "volcano:error", errorMessageId, ); // 🆕 图片生成结束标记(失败) writeAssistantTextChunk( writer, "", "volcano:image-end-failed", ); // 尝试保存错误消息 try { await saveMessages({ messages: [ { id: errorMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: "图片生成失败,请稍后重试" }], attachments: [], // 必须提供 attachments 字段 createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } } finally { stopKeepalive?.(); } }, }); return createSseResponse(volcanoStream); } // ========== 火山引擎视觉处理结束 ========== // ========== 千问/FAL 视频生成独立处理 ========== if ( ["qianwen-video", "fal-video"].includes((languageModel as any).provider) && typeof (languageModel as any).doGenerate === "function" ) { const currentVideoProvider = (languageModel as any).provider as "qianwen-video" | "fal-video"; console.log(`[Chat API] Using ${currentVideoProvider} video generation`); const qianwenVideoStream = createUIMessageStream({ execute: async ({ writer }) => { try { const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: chatMessageId, role: "assistant", parts: [{ type: "text", text: conversationMessage }], }), }); await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: conversationMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } const messagesForVideo = injectHistoryImagesToLastUserMessage( applyOptimizedPromptToLatestUserMessage( uiMessagesForModel, smartAutoOptimizedPrompt, hiddenPromptPrefix, ), smartAutoHistoryImages, ); // 千问/FAL 视频直接使用原始 URL,不转换为 base64 const modelMessages = convertToModelMessages(messagesForVideo); console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length); // 调用 doGenerate 获取视频 const result = await (languageModel as any).doGenerate({ messages: modelMessages, videoParams: effectiveVideoParams, }); const videoMarkdown = result.text || "视频生成失败"; console.log(`[Chat API] ${currentVideoProvider} video generated, length:`, videoMarkdown.length); // 提取视频 URL(后续会自动转存到永久存储) const tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || ""; // 生成一个统一的消息 ID const messageId = generateUUID(); const videoMetadata = { provider: currentVideoProvider, model: finalModelId, mode: effectiveVideoParams?.mode, resolution: effectiveVideoParams?.resolution, ratio: effectiveVideoParams?.ratio, duration: effectiveVideoParams?.duration, status: "succeeded", updatedAt: Date.now(), }; // 写入消息到流 writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: videoMarkdown }], attachments: [], }), }); // 保存消息到数据库 try { await saveMessages({ messages: [ { id: messageId, chatId: id, role: "assistant", parts: [{ type: "text", text: videoMarkdown }], attachments: [], createdAt: new Date(), videoStorageStatus: tempVideoUrl ? "temporary" : null, temporaryVideoUrl: tempVideoUrl || null, permanentVideoUrl: null, archivedAt: null, videoMetadata: videoMetadata as any, }, ], }); console.log(`[Chat API] ✅ ${currentVideoProvider} video message saved successfully!`); await consumeGeneratedVideoPoints({ userId: session.user.id, chatId: id, messageId, modelRefs: [finalModelId], }); if (tempVideoUrl) { scheduleVideoArchive({ chatId: id, messageId, text: videoMarkdown, temporaryVideoUrl: tempVideoUrl, taskId: messageId, metadata: videoMetadata, }); } } catch (saveError) { console.error(`[Chat API] ❌ Failed to save ${currentVideoProvider} video message:`, saveError); } } catch (error) { console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error); const errorMessageId = generateUUID(); writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: "视频生成失败,请稍后重试" }], attachments: [], }), }); } }, }); return createSseResponse(qianwenVideoStream); } // ========== 千问/FAL 视频处理结束 ========== const stream = createUIMessageStream({ execute: async ({ writer: dataStream }) => { // 构建工具对象,只包含启用的工具 const tools: any = {}; const activeTools: string[] = []; if (enabledTools.includes("getWeather")) { tools.getWeather = getWeather; activeTools.push("getWeather"); } if (enabledTools.includes("createDocument")) { tools.createDocument = createDocument({ session, dataStream }); activeTools.push("createDocument"); } if (enabledTools.includes("updateDocument")) { tools.updateDocument = updateDocument({ session, dataStream }); activeTools.push("updateDocument"); } if (enabledTools.includes("requestSuggestions")) { tools.requestSuggestions = requestSuggestions({ session, dataStream, }); activeTools.push("requestSuggestions"); } const shouldEnableGenerateVideoTool = enabledTools.includes("generateVideo") && (!shouldRunAgentConversation || allowAgentVideoGeneration); if (shouldEnableGenerateVideoTool) { tools.generateVideo = generateVideo({ dataStream }); activeTools.push("generateVideo"); } else if (enabledTools.includes("generateVideo") && shouldRunAgentConversation) { console.log("[Smart Auto Mode] 🚫 generateVideo tool disabled (manual video mode not selected)"); } console.log("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志 console.log("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志 // 转换消息,对于 Gemini 和千问视频将图片 URL 转换为 base64 const convertedMessages = await convertMessagesWithImages( injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages), (languageModel as any).provider || "" ); console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志 // 🆕 检测是否是视频模型 const providerType = (languageModel as any).provider; const effectiveProviderType = resolvedProviderType || providerType; console.log("[Chat API] Provider type for tool detection:", { runtime: providerType, resolved: resolvedProviderType, effective: effectiveProviderType, }); const isVolcanoVideoModel = effectiveProviderType === "volcano-video" || effectiveProviderType === "doubao-video" || actualModelId.includes("seedance"); const isOpenAICompatImageModel = resolvedModelSupportsImageGen && providerType === "openai.chat" && !isVolcanoVideoModel; console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel); // 🆕 处理 OpenAI 兼容的生图模型输出(防止返回 data:image;base64 原文) if (isOpenAICompatImageModel) { console.log("[Chat API] Using generateText for OpenAI-compatible image model"); // 生成开始标记 dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); try { const imageRequestTimeoutMs = 180_000; const generated = await generateText({ model: languageModel, system: systemPrompt({ selectedChatModel: finalModelId, requestHints }), messages: convertedMessages, abortSignal: AbortSignal.timeout(imageRequestTimeoutMs), }); const rawText = generated.text?.trim() || "图片生成失败,请稍后重试"; const normalizedText = await normalizeAssistantImageMarkdown( rawText, effectiveImageParams?.imageSize || "2K" ); const normalizedTextWithModel = prependImageModelBadge( normalizedText, currentModelConfig?.displayName || actualModelId, ); const messageId = generateUUID(); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: messageId, role: "assistant", parts: [{ type: "text", text: normalizedTextWithModel }], }), }); // 生成结束标记 dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); await saveMessages({ messages: [ { id: messageId, role: "assistant", parts: [{ type: "text", text: normalizedTextWithModel }], createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); const latestUserMessage = uiMessages.filter((m) => m.role === "user").pop(); const userPrompt = latestUserMessage?.parts?.find( (p): p is { type: "text"; text: string } => p.type === "text" )?.text || ""; await saveGeneratedImages( normalizedTextWithModel, session.user.id, id, messageId, userPrompt, finalModelId ); return; } catch (error) { console.error("[Chat API] OpenAI-compatible image model generation error:", error); const errorMessageId = generateUUID(); const errorText = error instanceof Error ? error.message : "图片生成失败,请稍后重试"; dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorText }], }), }); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: generateUUID(), role: "assistant", parts: [{ type: "text", text: "" }], }), }); await saveMessages({ messages: [ { id: errorMessageId, role: "assistant", parts: [{ type: "text", text: errorText }], createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); return; } } // 🎬 视频模型使用专门的生成流程 if (isVolcanoVideoModel) { console.log("[Chat API] Using video generation API"); // 导入视频生成函数 const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video"); const conversationMessage = smartAutoFriendlyMessage || null; if (conversationMessage) { const chatMessageId = generateUUID(); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: chatMessageId, role: "assistant", parts: [{ type: "text", text: conversationMessage }], attachments: [], }), }); await saveMessages({ messages: [ { id: chatMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: conversationMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); } const messagesForVideo = injectHistoryImagesToLastUserMessage( applyOptimizedPromptToLatestUserMessage( uiMessagesForModel, smartAutoOptimizedPrompt, hiddenPromptPrefix, ), smartAutoHistoryImages, ); // 从最后一条用户消息中提取提示词和图片 const lastUserMessage = messagesForVideo.filter(m => m.role === "user").pop(); let prompt = ""; const imageUrls: string[] = []; const summarizeImageRef = (rawUrl: string) => { const trimmed = String(rawUrl || "").trim(); if (!trimmed) return ""; if (trimmed.startsWith("data:image/")) { return `data:image(base64,${trimmed.length} chars)`; } try { const parsed = new URL(trimmed); return `${parsed.origin}${parsed.pathname}`; } catch { return trimmed.slice(0, 120); } }; if (lastUserMessage) { // 🆕 处理 parts 数组(AI SDK v4 格式) if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) { console.log("[Chat API] Video input parts:", (lastUserMessage as any).parts.length); for (const part of (lastUserMessage as any).parts) { if (part.type === "text" && part.text) { // 提取文本 prompt += part.text; } else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) { // 提取图片 imageUrls.push(part.url); } } } // 备用方案1:content 字符串/数组 const legacyContent = (lastUserMessage as any).content; if (typeof legacyContent === "string") { prompt = legacyContent; } else if (Array.isArray(legacyContent)) { for (const part of legacyContent) { if (typeof part === "string") { prompt += part; } else if (part.type === "text") { prompt += part.text; } else if (part.type === "image" && (part as any).image) { imageUrls.push((part as any).image); } } } else if ((lastUserMessage as any).experimental_attachments) { // 备用方案2:experimental_attachments for (const attachment of (lastUserMessage as any).experimental_attachments) { if (attachment.url && attachment.contentType?.startsWith("image/")) { imageUrls.push(attachment.url); } } } } console.log("[Chat API] Video input extracted:", { promptLength: prompt.trim().length, imageCount: imageUrls.length, imageRefs: imageUrls.slice(0, 4).map(summarizeImageRef), }); const videoMessageId = generateUUID(); const videoStartAt = Date.now(); const baseVideoMetadata: Record = { provider: effectiveProviderType || "volcano-video", mode: effectiveVideoParams?.mode || "text-to-video", resolution: effectiveVideoParams?.resolution || "720p", ratio: effectiveVideoParams?.ratio || "16:9", duration: effectiveVideoParams?.duration || 5, status: "queued", startedAt: videoStartAt, updatedAt: videoStartAt, }; try { console.log("[Chat API] Video generation params:", { promptLength: prompt.trim().length, imageCount: imageUrls.length, videoParams: effectiveVideoParams, }); const videoMode = (effectiveVideoParams?.mode || "text-to-video") as VideoModeType; // 🆕 重新执行子模型选择逻辑,确定要使用的模型 UUID let videoModelId = finalModelId; // 默认使用选择的模型 ID // 检查是否是主模型,需要选择子模型(仅 volcano-video 支持子模型) const selectedModelData = await getAiModelById({ id: finalModelId }); const selectedModelProvider = selectedModelData ? await getAiProviderById({ id: selectedModelData.providerId }).catch(() => null) : null; if ( selectedModelData && !selectedModelData.isSubModel && selectedModelProvider?.type === "volcano-video" ) { const videoSubModels = await getSubModelsByParentModelId(finalModelId); console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`); if (videoSubModels.length > 0) { const matchedModel = selectBestVideoSubModelForMode(videoSubModels as any, videoMode); if (matchedModel) { videoModelId = matchedModel.model.id; console.log( `[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} (${matchedModel.model.modelId}) for mode ${videoMode}` ); } else { console.log( `[Chat API] Video: No compatible sub-model found for mode ${videoMode}, keep current model` ); } } } console.log("[Chat API] Getting model config for:", videoModelId); const aiModelData = await getAiModelById({ id: videoModelId }); if (!aiModelData) { throw new Error("视频模型配置未找到"); } // 若用户手动选了不兼容模式的子模型,自动回退到同父模型兼容子模型 const selectedSupportsMode = getSupportedVideoModesForModel(aiModelData as any).includes(videoMode); if (!selectedSupportsMode) { let fallbackModel: { model: any } | null = null; if (aiModelData.parentModelId) { const siblingSubModels = await getSubModelsByParentModelId(aiModelData.parentModelId); fallbackModel = selectBestVideoSubModelForMode(siblingSubModels as any, videoMode); } else if (selectedModelProvider?.type === "volcano-video") { const enabledModels = await getEnabledAiModelsWithProvider(); const sameProviderModels = enabledModels .filter( ({ model, provider }) => provider.type === "volcano-video" && model.isEnabled && model.id !== aiModelData.id, ) .map(({ model }) => ({ model })); fallbackModel = selectBestVideoSubModelForMode(sameProviderModels as any, videoMode); } if (fallbackModel) { console.warn( `[Chat API] Video: Model ${aiModelData.modelId} does not support ${videoMode}, fallback to ${fallbackModel.model.modelId}` ); videoModelId = fallbackModel.model.id; } } const finalVideoModelData = videoModelId === aiModelData.id ? aiModelData : await getAiModelById({ id: videoModelId }); if (!finalVideoModelData) { throw new Error("视频模型配置未找到"); } // 如果是子模型,获取父模型的 Provider 配置 let providerModelData = finalVideoModelData; if (finalVideoModelData.isSubModel && finalVideoModelData.parentModelId) { console.log( "[Chat API] This is a sub-model, getting parent model config:", finalVideoModelData.parentModelId ); const parentModel = await getAiModelById({ id: finalVideoModelData.parentModelId }); if (parentModel) { providerModelData = parentModel; } } console.log("[Chat API] Getting provider config for:", providerModelData.providerId); const providerData = await getAiProviderById({ id: providerModelData.providerId }); if (!providerData) { throw new Error("Provider 配置未找到"); } const actualVideoModelId = finalVideoModelData.modelId; // 构建视频生成选项 const videoOptions: any = { model: actualVideoModelId, prompt: prompt || "生成视频", mode: videoMode, resolution: effectiveVideoParams?.resolution || "720p", ratio: effectiveVideoParams?.ratio || "16:9", duration: effectiveVideoParams?.duration || 5, fps: effectiveVideoParams?.fps || 24, cameraMove: effectiveVideoParams?.cameraMove || "none", cameraFixed: effectiveVideoParams?.cameraFixed || false, watermark: effectiveVideoParams?.watermark || false, seed: effectiveVideoParams?.seed, }; // 根据模式添加图片 if (videoOptions.mode === "first-frame" && imageUrls[0]) { videoOptions.firstFrameImage = imageUrls[0]; } else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) { videoOptions.firstFrameImage = imageUrls[0]; videoOptions.lastFrameImage = imageUrls[1]; } else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) { videoOptions.referenceImages = imageUrls; } console.log("[Chat API] Video options constructed:", { model: videoOptions.model, mode: videoOptions.mode, resolution: videoOptions.resolution, ratio: videoOptions.ratio, duration: videoOptions.duration, imageCount: videoOptions.referenceImages?.length || Number(Boolean(videoOptions.firstFrameImage)) + Number(Boolean(videoOptions.lastFrameImage)), }); // 先写入占位消息,避免刷新后消息丢失 const pendingText = "🎬 视频任务已创建,正在处理中,请稍候..."; await saveMessages({ messages: [ { id: videoMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: pendingText }], attachments: [], createdAt: new Date(), videoStorageStatus: "temporary", temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: { ...baseVideoMetadata, model: actualVideoModelId, status: "running", updatedAt: Date.now(), } as any, }, ], }); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: videoMessageId, role: "assistant", parts: [{ type: "text", text: pendingText }], attachments: [], }), }); const apiKey = providerData.apiKey; const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3"; console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey }); // 调用视频生成流 console.log("[Chat API] Starting video generation stream..."); const videoStream = doGenerateStream(apiKey, baseURL, videoOptions); // 收集所有流输出 console.log("[Chat API] Processing video stream..."); let fullText = ""; let chunkCount = 0; for await (const chunk of videoStream) { chunkCount++; fullText += chunk; console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`); } console.log( `[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}` ); // 提取流中的元数据并清理输出文案 const metadataMatch = fullText.match(//s); let streamMetadata: Record = {}; if (metadataMatch?.[1]) { try { streamMetadata = JSON.parse(metadataMatch[1]); } catch (parseError) { console.warn("[Chat API] Failed to parse VIDEO_METADATA from stream", parseError); } } const cleanedText = fullText .replace(//gs, "") .trim(); const videoUrlMatch = cleanedText.match(/]+src="([^"]+)"/); const temporaryVideoUrl = (typeof streamMetadata.temporaryVideoUrl === "string" && streamMetadata.temporaryVideoUrl) || (videoUrlMatch ? videoUrlMatch[1] : null); const successMetadata = { ...baseVideoMetadata, ...streamMetadata, model: actualVideoModelId, taskId: typeof streamMetadata.taskId === "string" && streamMetadata.taskId ? streamMetadata.taskId : undefined, status: "succeeded", completedAt: Date.now(), updatedAt: Date.now(), }; await updateMessageById({ messageId: videoMessageId, chatId: id, parts: [{ type: "text", text: cleanedText }], videoStorageStatus: temporaryVideoUrl ? "temporary" : null, temporaryVideoUrl: temporaryVideoUrl, permanentVideoUrl: null, archivedAt: null, videoMetadata: successMetadata as any, }); await consumeGeneratedVideoPoints({ userId: session.user.id, chatId: id, messageId: videoMessageId, modelRefs: [finalVideoModelData.id, providerModelData.id, finalModelId], }); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: videoMessageId, role: "assistant", parts: [{ type: "text", text: cleanedText }], attachments: [], }), }); console.log("[Chat API] ✅ Video message updated successfully"); if (temporaryVideoUrl) { console.log("[Chat API] 📹 临时视频URL已保存,开始异步归档到S3"); scheduleVideoArchive({ chatId: id, messageId: videoMessageId, text: cleanedText, temporaryVideoUrl, taskId: typeof successMetadata.taskId === "string" && successMetadata.taskId ? successMetadata.taskId : videoMessageId, metadata: successMetadata, }); } console.log("[Chat API] Video generation finished successfully"); return; } catch (error: any) { console.error("[Chat API] ❌ Video generation error:", error); console.error("[Chat API] Error stack:", error.stack); const errorMessage = `❌ 视频生成失败:${error.message}\n\n请检查配置或稍后重试。`; const failedMetadata = { ...baseVideoMetadata, status: "failed", error: error?.message || "未知错误", updatedAt: Date.now(), }; try { await updateMessageById({ messageId: videoMessageId, chatId: id, parts: [{ type: "text", text: errorMessage }], videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: failedMetadata as any, }); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: videoMessageId, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], }), }); } catch (updateError) { console.error("[Chat API] Failed to update placeholder video message:", updateError); const fallbackMessageId = generateUUID(); dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ id: fallbackMessageId, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], }), }); await saveMessages({ messages: [ { id: fallbackMessageId, chatId: id, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: failedMetadata as any, }, ], }); } return; } } // 常规文本模型使用 streamText const streamRequestTimeoutMs = resolvedModelSupportsImageGen ? 180_000 : 90_000; const shouldDisableSmoothStream = resolvedModelSupportsImageGen || isVolcanoVideoModel; const result = streamText({ model: languageModel, system: systemPrompt({ selectedChatModel: finalModelId, requestHints }), messages: convertedMessages, stopWhen: stepCountIs(5), experimental_activeTools: finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools, experimental_transform: shouldDisableSmoothStream ? undefined : smoothStream({ chunking: "word" }), tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具 abortSignal: AbortSignal.timeout(streamRequestTimeoutMs), experimental_telemetry: { isEnabled: isProductionEnvironment, functionId: "stream-text", }, onFinish: async ({ usage }) => { try { const providers = await getTokenlensCatalog(); if (!actualModelId) { finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage, }); return; } if (!providers) { finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage, }); return; } const summary = getUsage({ modelId: actualModelId, usage, providers, }); finalMergedUsage = { ...usage, ...summary, modelId: actualModelId, } as AppUsage; dataStream.write({ type: "data-usage", data: finalMergedUsage }); } catch (err) { console.warn("TokenLens enrichment failed", err); finalMergedUsage = usage; dataStream.write({ type: "data-usage", data: finalMergedUsage }); } }, }); result.consumeStream(); dataStream.merge( result.toUIMessageStream({ sendReasoning: true, }) ); }, generateId: generateUUID, onFinish: async ({ messages }) => { await saveMessages({ messages: messages.map((currentMessage) => ({ id: currentMessage.id, role: currentMessage.role, parts: currentMessage.parts, createdAt: new Date(), attachments: [], chatId: id, videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, })), }); // 检测助手消息中是否包含视频,如果有则立即标记并归档到永久存储 for (const msg of messages) { if (msg.role === "assistant" && msg.parts) { const textContent = msg.parts .filter((part: any) => part && part.type === "text" && part.text) .map((part: any) => part.text) .join(""); const tempVideoUrl = extractVideoUrlFromText(textContent); if (tempVideoUrl) { const genericVideoMetadata = { provider: "stream", model: actualModelId, status: "succeeded", updatedAt: Date.now(), }; await updateMessageById({ messageId: msg.id, chatId: id, videoStorageStatus: "temporary", temporaryVideoUrl: tempVideoUrl, permanentVideoUrl: null, archivedAt: null, videoMetadata: genericVideoMetadata as any, }); scheduleVideoArchive({ chatId: id, messageId: msg.id, text: textContent, temporaryVideoUrl: tempVideoUrl, taskId: msg.id, metadata: genericVideoMetadata, }); } } } if (finalMergedUsage) { try { await updateChatLastContextById({ chatId: id, context: finalMergedUsage, }); } catch (err) { console.warn("Unable to persist last usage for chat", id, err); } } }, onError: () => { return "Oops, an error occurred!"; }, }); // const streamContext = getStreamContext(); // if (streamContext) { // return new Response( // await streamContext.resumableStream(streamId, () => // stream.pipeThrough(new JsonToSseTransformStream()) // ) // ); // } return createSseResponse(stream); } catch (error) { const vercelId = request.headers.get("x-vercel-id"); if (error instanceof ChatSDKError) { return error.toResponse(); } // Check for Vercel AI Gateway credit card error if ( error instanceof Error && error.message?.includes( "AI Gateway requires a valid credit card on file to service requests" ) ) { return new ChatSDKError("bad_request:activate_gateway").toResponse(); } console.error("Unhandled error in chat API:", error, { vercelId }); return new ChatSDKError("offline:chat").toResponse(); } finally { if (requestDedupKey && acquiredInFlightLock) { inFlightChatRequests.delete(requestDedupKey); } } } export async function DELETE(request: Request) { const { searchParams } = new URL(request.url); const id = searchParams.get("id"); if (!id) { return new ChatSDKError("bad_request:api").toResponse(); } const session = await auth(); if (!session?.user) { return new ChatSDKError("unauthorized:chat").toResponse(); } const chat = await getChatById({ id }); if (chat?.userId !== session.user.id) { return new ChatSDKError("forbidden:chat").toResponse(); } const deletedChat = await deleteChatById({ id }); return Response.json(deletedChat, { status: 200 }); }