2319 lines
93 KiB
TypeScript
2319 lines
93 KiB
TypeScript
import { geolocation } from "@vercel/functions";
|
||
import {
|
||
convertToModelMessages,
|
||
createUIMessageStream,
|
||
JsonToSseTransformStream,
|
||
smoothStream,
|
||
stepCountIs,
|
||
streamText,
|
||
} from "ai";
|
||
import { unstable_cache as cache } from "next/cache";
|
||
import { after, NextResponse } from "next/server";
|
||
import {
|
||
createResumableStreamContext,
|
||
type ResumableStreamContext,
|
||
} from "resumable-stream";
|
||
import type { ModelCatalog } from "tokenlens/core";
|
||
import { fetchModels } from "tokenlens/fetch";
|
||
import { getUsage } from "tokenlens/helpers";
|
||
import { auth, type UserType } from "@/app/(auth)/auth";
|
||
import type { VisibilityType } from "@/components/visibility-selector";
|
||
import { entitlementsByUserType } from "@/lib/ai/entitlements";
|
||
import type { ChatModel } from "@/lib/ai/models";
|
||
import { type RequestHints, systemPrompt } from "@/lib/ai/prompts";
|
||
import { myProvider } from "@/lib/ai/providers";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { createDocument } from "@/lib/ai/tools/create-document";
|
||
import { getWeather } from "@/lib/ai/tools/get-weather";
|
||
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
|
||
import { updateDocument } from "@/lib/ai/tools/update-document";
|
||
import { generateVideo } from "@/lib/ai/tools/generate-video";
|
||
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
||
import { isProductionEnvironment } from "@/lib/constants";
|
||
import {
|
||
createStreamId,
|
||
deleteChatById,
|
||
getChatById,
|
||
getMessageCountByUserId,
|
||
getMessagesByChatId,
|
||
saveChat,
|
||
saveMessages,
|
||
updateChatLastContextById,
|
||
getAiModelById,
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId,
|
||
getAiProviderById,
|
||
checkUserImageQuota,
|
||
} from "@/lib/db/queries";
|
||
import { db } from "@/lib/db";
|
||
import type { DBMessage } from "@/lib/db/schema";
|
||
import { generatedImage } from "@/lib/db/schema";
|
||
import { ChatSDKError } from "@/lib/errors";
|
||
import type { ChatMessage } from "@/lib/types";
|
||
import type { AppUsage } from "@/lib/usage";
|
||
import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
||
import { generateTitleFromUserMessage } from "../../actions";
|
||
import { type PostRequestBody, postRequestBodySchema } from "./schema";
|
||
|
||
/**
|
||
* 将图片 URL 转换为 base64 格式
|
||
* 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API
|
||
*/
|
||
async function convertImageUrlToBase64(url: string): Promise<string> {
|
||
try {
|
||
console.log("[Image Convert] Fetching image from:", url);
|
||
const response = await fetch(url);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to fetch image: ${response.statusText}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
const base64 = buffer.toString("base64");
|
||
|
||
// 获取 MIME 类型
|
||
const contentType = response.headers.get("content-type") || "image/jpeg";
|
||
const dataUrl = `data:${contentType};base64,${base64}`;
|
||
|
||
console.log("[Image Convert] Converted to base64, size:", base64.length);
|
||
return dataUrl;
|
||
} catch (error) {
|
||
console.error("[Image Convert] Failed:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头）
|
||
*/
|
||
function hasAnnotatedImage(messages: ChatMessage[]): boolean {
|
||
for (const message of messages) {
|
||
for (const part of message.parts) {
|
||
if (part.type === "file" && "name" in part && typeof part.name === "string") {
|
||
if (part.name.startsWith("annotated-")) {
|
||
return true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* 标注图片的系统提示词（用户不可见，后台自动添加）
|
||
*/
|
||
const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。";
|
||
|
||
function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: string): ChatMessage[] {
|
||
if (!hiddenPrompt || !hiddenPrompt.trim()) return messages;
|
||
const cloned = messages.map((msg) => ({
|
||
...msg,
|
||
parts: msg.parts.map((part) => ({ ...part })),
|
||
}));
|
||
for (let i = cloned.length - 1; i >= 0; i--) {
|
||
const msg = cloned[i];
|
||
if (msg.role !== "user") continue;
|
||
const textPart = msg.parts.find((part) => part.type === "text") as { type: "text"; text: string } | undefined;
|
||
if (textPart) {
|
||
textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim();
|
||
} else {
|
||
msg.parts.unshift({ type: "text", text: hiddenPrompt });
|
||
}
|
||
break;
|
||
}
|
||
return cloned;
|
||
}
|
||
|
||
function guessImageMediaType(url: string): string {
|
||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg";
|
||
if (cleanUrl.endsWith(".webp")) return "image/webp";
|
||
if (cleanUrl.endsWith(".gif")) return "image/gif";
|
||
if (cleanUrl.endsWith(".bmp")) return "image/bmp";
|
||
if (cleanUrl.endsWith(".svg")) return "image/svg+xml";
|
||
return "image/png";
|
||
}
|
||
|
||
function injectHistoryImagesToLastUserMessage(
|
||
messages: ChatMessage[],
|
||
historyImages: string[]
|
||
): ChatMessage[] {
|
||
if (!historyImages || historyImages.length === 0) return messages;
|
||
|
||
let lastUserIndex = -1;
|
||
for (let i = messages.length - 1; i >= 0; i--) {
|
||
if (messages[i]?.role === "user") {
|
||
lastUserIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (lastUserIndex < 0) return messages;
|
||
|
||
const lastUserMessage = messages[lastUserIndex];
|
||
const hasExistingImage = lastUserMessage.parts?.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
if (hasExistingImage) return messages;
|
||
|
||
const imageParts = historyImages.map((url, index) => ({
|
||
type: "file" as const,
|
||
url,
|
||
name: `history-image-${index + 1}`,
|
||
mediaType: guessImageMediaType(url),
|
||
mimeType: guessImageMediaType(url),
|
||
}));
|
||
|
||
const updatedMessage: ChatMessage = {
|
||
...lastUserMessage,
|
||
parts: [...lastUserMessage.parts, ...imageParts],
|
||
};
|
||
|
||
return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
|
||
}
|
||
|
||
/**
|
||
* 自定义消息转换函数
|
||
* 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64）
|
||
* 因为这些 provider 内部会自行处理图片上传到 S3
|
||
*/
|
||
async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) {
|
||
if (providerType === "ruxa-image") {
|
||
const convertedMessages = [];
|
||
|
||
for (const message of messages) {
|
||
const content: any[] = [];
|
||
|
||
for (const part of message.parts) {
|
||
const partAny = part as any;
|
||
|
||
if (partAny.type === "text" && partAny.text) {
|
||
content.push({
|
||
type: "text",
|
||
text: partAny.text,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const candidateUrl =
|
||
partAny.type === "file"
|
||
? (partAny.url || partAny.data)
|
||
: partAny.type === "image"
|
||
? (partAny.image || partAny.url)
|
||
: partAny.type === "image_url"
|
||
? partAny.image_url?.url
|
||
: undefined;
|
||
|
||
if (!candidateUrl || typeof candidateUrl !== "string") continue;
|
||
if (candidateUrl.startsWith("data:")) {
|
||
throw new Error("RUXA 不支持 Base64 图片传输，请先上传图片后再编辑");
|
||
}
|
||
if (!candidateUrl.startsWith("http")) continue;
|
||
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: candidateUrl,
|
||
},
|
||
});
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text" ? content[0].text : content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
// 所有 provider 都使用 URL 格式传输图片
|
||
// gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL
|
||
// 这里不再进行 Base64 转换，直接返回标准格式
|
||
if (providerType !== "gemini") {
|
||
return convertToModelMessages(messages);
|
||
}
|
||
|
||
// 🆕 检测是否有标注图片
|
||
const hasAnnotation = hasAnnotatedImage(messages);
|
||
if (hasAnnotation) {
|
||
console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示");
|
||
}
|
||
|
||
// 对于 Gemini，需要特殊处理图片
|
||
const convertedMessages = [];
|
||
|
||
for (const message of messages) {
|
||
const content: any[] = [];
|
||
|
||
// 🆕 如果是用户消息且有标注图片，在文本前添加系统提示
|
||
let hasAddedAnnotationPrompt = false;
|
||
|
||
// 处理消息的各个部分
|
||
for (const part of message.parts) {
|
||
if (part.type === "text" && part.text) {
|
||
let textContent = part.text;
|
||
|
||
// 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) {
|
||
textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent;
|
||
hasAddedAnnotationPrompt = true;
|
||
console.log("[Message Convert] 🎨 已注入标注提示词");
|
||
}
|
||
|
||
content.push({
|
||
type: "text",
|
||
text: textContent,
|
||
});
|
||
} else if (part.type === "file" && part.url) {
|
||
// 直接使用 URL 格式（不再转换为 Base64）
|
||
// gemini-image 和 huanke-gemini provider 内部会处理图片上传
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: part.url,
|
||
},
|
||
});
|
||
console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "...");
|
||
}
|
||
}
|
||
|
||
// 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) {
|
||
content.unshift({
|
||
type: "text",
|
||
text: ANNOTATION_SYSTEM_PROMPT,
|
||
});
|
||
console.log("[Message Convert] 🎨 已在消息开头注入标注提示词");
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text"
|
||
? content[0].text
|
||
: content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
/**
|
||
* 解析生图 Markdown 并保存图片记录到数据库
|
||
* 支持三种格式：thumb|||url|||size、thumb$$url$$size、thumb|url|size
|
||
*/
|
||
async function saveGeneratedImages(
|
||
imageMarkdown: string,
|
||
userId: string,
|
||
chatId: string,
|
||
messageId: string,
|
||
prompt: string,
|
||
modelId: string
|
||
): Promise<void> {
|
||
// 匹配 Markdown 图片格式: 
|
||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||
const matches = [...imageMarkdown.matchAll(imageRegex)];
|
||
|
||
for (const match of matches) {
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = match[2];
|
||
let originalUrl = thumbnailUrl;
|
||
let imageSize = "2K";
|
||
|
||
// 解析 alt 中的原图 URL 和尺寸
|
||
// 格式1: thumb|||originalUrl|||size (新格式)
|
||
if (altText.includes("thumb|||")) {
|
||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||
const parts = content.split("|||");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式2: thumb$$originalUrl$$size (中间格式，可能被 LaTeX 解析器破坏)
|
||
else if (altText.includes("thumb$$")) {
|
||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||
const parts = content.split("$$");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式3: thumb|originalUrl|size (旧格式)
|
||
else if (altText.includes("thumb|")) {
|
||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||
const parts = content.split("|");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
|
||
// 回退策略：从缩略图 URL 推断原图 URL
|
||
if (originalUrl === thumbnailUrl) {
|
||
if (thumbnailUrl.includes("gemini-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-");
|
||
} else if (thumbnailUrl.includes("openai-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-");
|
||
} else if (thumbnailUrl.includes("volcano-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-");
|
||
} else if (thumbnailUrl.includes("alibaba-image-thumb-")) {
|
||
// 🆕 阿里生图模型
|
||
originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-");
|
||
} else if (thumbnailUrl.includes("ruxa-image-thumb-")) {
|
||
// 🆕 RUXA 生图模型
|
||
originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-");
|
||
}
|
||
}
|
||
|
||
try {
|
||
await db.insert(generatedImage).values({
|
||
userId,
|
||
chatId,
|
||
messageId,
|
||
thumbnailUrl,
|
||
originalUrl,
|
||
imageSize,
|
||
prompt,
|
||
modelId,
|
||
});
|
||
console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`);
|
||
} catch (error) {
|
||
console.error("[Gallery] Failed to save image record:", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务
|
||
|
||
let globalStreamContext: ResumableStreamContext | null = null;
|
||
|
||
const getTokenlensCatalog = cache(
|
||
async (): Promise<ModelCatalog | undefined> => {
|
||
try {
|
||
return await fetchModels();
|
||
} catch (err) {
|
||
console.warn(
|
||
"TokenLens: catalog fetch failed, using default catalog",
|
||
err
|
||
);
|
||
return; // tokenlens helpers will fall back to defaultCatalog
|
||
}
|
||
},
|
||
["tokenlens-catalog"],
|
||
{ revalidate: 24 * 60 * 60 } // 24 hours
|
||
);
|
||
|
||
export function getStreamContext() {
|
||
if (!globalStreamContext) {
|
||
try {
|
||
globalStreamContext = createResumableStreamContext({
|
||
waitUntil: after,
|
||
});
|
||
} catch (error: any) {
|
||
if (error.message.includes("REDIS_URL")) {
|
||
console.log(
|
||
" > Resumable streams are disabled due to missing REDIS_URL"
|
||
);
|
||
} else {
|
||
console.error(error);
|
||
}
|
||
}
|
||
}
|
||
|
||
return globalStreamContext;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
let requestBody: PostRequestBody;
|
||
|
||
try {
|
||
const json = await request.json();
|
||
console.log("[Chat API] Received request body:", JSON.stringify(json, null, 2));
|
||
requestBody = postRequestBodySchema.parse(json);
|
||
} catch (error) {
|
||
console.error("[Chat API] Schema validation failed:", error);
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
try {
|
||
const {
|
||
id,
|
||
message,
|
||
selectedChatModel,
|
||
selectedVisibilityType,
|
||
hiddenPrompt,
|
||
autoContext,
|
||
}: {
|
||
id: string;
|
||
message: ChatMessage;
|
||
selectedChatModel: ChatModel["id"];
|
||
selectedVisibilityType: VisibilityType;
|
||
hiddenPrompt?: string;
|
||
autoContext?: string;
|
||
} = requestBody;
|
||
|
||
let finalModelId = selectedChatModel;
|
||
|
||
// 🆕 存储智能模式的相关信息
|
||
let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复
|
||
let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用）
|
||
let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张）
|
||
|
||
// ========== Smart Auto Mode Logic ==========
|
||
if (finalModelId === "auto") {
|
||
console.log("[Smart Auto Mode] 🚀 Starting intelligent analysis...");
|
||
|
||
try {
|
||
// 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb）
|
||
const smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id });
|
||
console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB");
|
||
|
||
// 1. 提取用户消息内容
|
||
const lastUserMessage = message;
|
||
const textContent = lastUserMessage.parts
|
||
.filter((p): p is { type: "text"; text: string } => p.type === "text")
|
||
.map((p) => p.text)
|
||
.join(" ");
|
||
const autoContextText = typeof autoContext === "string" ? autoContext.trim() : "";
|
||
|
||
// @ts-ignore - 检查各种可能的图片类型
|
||
let hasImage = lastUserMessage.parts.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
|
||
// @ts-ignore - 提取图片URL
|
||
let imageUrls = lastUserMessage.parts
|
||
.filter((p: any) => p.type === "file" && p.url)
|
||
.map((p: any) => p.url as string);
|
||
|
||
// 🆕 历史图片上下文感知：如果用户没有发送新图片，自动使用历史中最近的图片（支持多张）
|
||
// 逻辑：只要用户在对话中没有附上新图片，就默认继续处理之前的图片
|
||
if (!hasImage && smartAutoHistoryMessagesFromDb.length > 0) {
|
||
console.log("[Smart Auto Mode] 🔍 No image in current message, auto-searching history...");
|
||
|
||
// 从历史消息中倒序查找最近一轮包含图片的消息
|
||
// 收集该消息中的所有图片（支持多图融合编辑）
|
||
for (let i = smartAutoHistoryMessagesFromDb.length - 1; i >= 0; i--) {
|
||
const msg = smartAutoHistoryMessagesFromDb[i];
|
||
const parts = msg.parts as any[];
|
||
|
||
if (!parts) continue;
|
||
|
||
// 查找AI生成的图片（从Markdown中提取所有图片）
|
||
if (msg.role === "assistant") {
|
||
for (const part of parts) {
|
||
if (part.type === "text" && part.text) {
|
||
// 匹配所有Markdown图片格式: 
|
||
const imgMatches = part.text.matchAll(/!\[.*?\]\((https?:\/\/[^\s)]+)\)/g);
|
||
for (const match of imgMatches) {
|
||
if (match[1] && !smartAutoHistoryImages.includes(match[1])) {
|
||
smartAutoHistoryImages.push(match[1]);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// 如果在这条assistant消息中找到了图片，就停止搜索（使用最近一轮的结果）
|
||
if (smartAutoHistoryImages.length > 0) {
|
||
console.log("[Smart Auto Mode] 🖼️ Found AI-generated images:", smartAutoHistoryImages.length);
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 查找用户上传的图片（收集该消息中的所有图片）
|
||
if (msg.role === "user" && smartAutoHistoryImages.length === 0) {
|
||
for (const part of parts) {
|
||
if ((part.type === "file" || part.type === "image") && part.url) {
|
||
if (!smartAutoHistoryImages.includes(part.url)) {
|
||
smartAutoHistoryImages.push(part.url);
|
||
}
|
||
}
|
||
}
|
||
// 如果在这条user消息中找到了图片，就停止搜索
|
||
if (smartAutoHistoryImages.length > 0) {
|
||
console.log("[Smart Auto Mode] 🖼️ Found user-uploaded images:", smartAutoHistoryImages.length);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
// 如果找到了历史图片，标记为有图片
|
||
if (smartAutoHistoryImages.length > 0) {
|
||
hasImage = true;
|
||
imageUrls = smartAutoHistoryImages;
|
||
console.log("[Smart Auto Mode] ✅ Auto-using", smartAutoHistoryImages.length, "history image(s) for context continuity");
|
||
} else {
|
||
console.log("[Smart Auto Mode] ℹ️ No image found in history, proceeding as pure text");
|
||
}
|
||
}
|
||
|
||
console.log("[Smart Auto Mode] Input:", { textContent: textContent.substring(0, 100), hasImage, imageCount: imageUrls.length, fromHistory: smartAutoHistoryImages.length > 0 });
|
||
|
||
// 🆕 从历史消息中提取对话文本（用于上下文理解）
|
||
const historyMessages: { role: "user" | "assistant"; content: string }[] = [];
|
||
for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条
|
||
const parts = msg.parts as any[];
|
||
if (parts && parts.length > 0) {
|
||
// 提取文本内容
|
||
const textParts = parts.filter((p: any) => p.type === 'text');
|
||
const content = textParts.map((p: any) => p.text || '').join(' ').trim();
|
||
if (content) {
|
||
historyMessages.push({
|
||
role: msg.role as "user" | "assistant",
|
||
content: content.substring(0, 300), // 限制长度
|
||
});
|
||
}
|
||
}
|
||
}
|
||
console.log("[Smart Auto Mode] History context:", historyMessages.length, "messages");
|
||
|
||
// 2. 使用 LangGraph 驱动的自动模式工作流
|
||
const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph");
|
||
const autoGraphResult = await runAutoAgentGraph({
|
||
text: textContent,
|
||
hasImage,
|
||
imageUrls,
|
||
imageFromHistory: smartAutoHistoryImages.length > 0,
|
||
historyMessages,
|
||
context: autoContextText || undefined,
|
||
});
|
||
|
||
if (autoGraphResult.analysis) {
|
||
console.log("[Smart Auto Mode] Analysis result:", {
|
||
intent: autoGraphResult.analysis.intent,
|
||
subIntent: autoGraphResult.analysis.subIntent,
|
||
confidence: autoGraphResult.analysis.confidence,
|
||
needsClarification: autoGraphResult.analysis.needsClarification,
|
||
});
|
||
}
|
||
|
||
if (autoGraphResult.friendlyMessage) {
|
||
smartAutoFriendlyMessage = autoGraphResult.friendlyMessage;
|
||
console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage);
|
||
}
|
||
|
||
if (autoGraphResult.optimizedPrompt) {
|
||
smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt;
|
||
console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", {
|
||
original: textContent.substring(0, 50),
|
||
optimized: autoGraphResult.optimizedPrompt.substring(0, 100),
|
||
});
|
||
}
|
||
|
||
if (autoGraphResult.route) {
|
||
finalModelId = autoGraphResult.route.id;
|
||
console.log(`[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`);
|
||
}
|
||
|
||
} catch (autoError) {
|
||
// 智能分析失败时，回退到简单逻辑
|
||
console.error("[Smart Auto Mode] ❌ Error, falling back:", autoError);
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
// @ts-ignore - 检查各种可能的图片类型
|
||
const hasImage = message.parts.some(
|
||
(p: any) => p.type === "file" || p.type === "image" || p.type === "image_url"
|
||
);
|
||
|
||
// 简单回退：有图选图片模型，无图选默认模型
|
||
let bestModel = null;
|
||
if (hasImage) {
|
||
bestModel = allModels.find(m => m.model.supportsImageGen && !m.model.isSubModel);
|
||
// 🆕 回退时也生成一个友好消息
|
||
smartAutoFriendlyMessage = "好的，我来帮你处理这张图片~";
|
||
}
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(m => m.model.isDefault && !m.model.isSubModel);
|
||
}
|
||
if (!bestModel && allModels.length > 0) {
|
||
bestModel = allModels.find(m => !m.model.isSubModel) || allModels[0];
|
||
}
|
||
|
||
if (bestModel) {
|
||
finalModelId = bestModel.model.id;
|
||
console.log(`[Smart Auto Mode] Fallback selected: ${bestModel.model.displayName}`);
|
||
} else {
|
||
return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse();
|
||
}
|
||
}
|
||
}
|
||
// ========== End Smart Auto Mode Logic ==========
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const userType: UserType = session.user.type;
|
||
|
||
const messageCount = await getMessageCountByUserId({
|
||
id: session.user.id,
|
||
differenceInHours: 24,
|
||
});
|
||
|
||
if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) {
|
||
return new ChatSDKError("rate_limit:chat").toResponse();
|
||
}
|
||
|
||
// ========== 使用时间限制验证（后端安全检查） ==========
|
||
try {
|
||
// 从数据库获取网站设置
|
||
const { siteSettings } = await import("@/lib/db/schema");
|
||
const [settings] = await db.select().from(siteSettings).limit(1);
|
||
|
||
if (settings?.usageTimeEnabled) {
|
||
const now = new Date();
|
||
const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
|
||
const startTime = settings.usageTimeStart || "08:00";
|
||
const endTime = settings.usageTimeEnd || "20:00";
|
||
|
||
const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime;
|
||
|
||
if (!isWithinUsageTime) {
|
||
console.log(`[Chat API] ⛔ 访问被拒绝：当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "usage_time_restricted",
|
||
message: settings.usageTimeMessage || "当前不在系统使用时间段内",
|
||
currentTime,
|
||
allowedTime: `${startTime} - ${endTime}`
|
||
},
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 时间验证通过：${currentTime} 在允许范围 ${startTime}-${endTime}`);
|
||
}
|
||
} catch (timeCheckError) {
|
||
console.error("[Chat API] ⚠️ 时间验证失败（继续执行）:", timeCheckError);
|
||
// 如果时间检查失败，继续执行（避免因检查逻辑错误导致系统不可用）
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
let messagesFromDb: DBMessage[] = [];
|
||
|
||
if (chat) {
|
||
if (chat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
// Only fetch messages if chat already exists
|
||
messagesFromDb = await getMessagesByChatId({ id });
|
||
} else {
|
||
const title = await generateTitleFromUserMessage({
|
||
message,
|
||
});
|
||
|
||
await saveChat({
|
||
id,
|
||
userId: session.user.id,
|
||
title,
|
||
visibility: selectedVisibilityType,
|
||
});
|
||
// New chat - no need to fetch messages, it's empty
|
||
}
|
||
|
||
const uiMessages = [...convertToUIMessages(messagesFromDb), message];
|
||
const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : "";
|
||
const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : "";
|
||
const uiMessagesForModel = hiddenPromptText
|
||
? applyHiddenPromptToMessages(uiMessages, hiddenPromptText)
|
||
: uiMessages;
|
||
|
||
const { longitude, latitude, city, country } = geolocation(request);
|
||
|
||
const requestHints: RequestHints = {
|
||
longitude,
|
||
latitude,
|
||
city,
|
||
country,
|
||
};
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
chatId: id,
|
||
id: message.id,
|
||
role: "user",
|
||
parts: message.parts,
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
const streamId = generateUUID();
|
||
await createStreamId({ streamId, chatId: id });
|
||
|
||
let finalMergedUsage: AppUsage | undefined;
|
||
|
||
// 获取启用的工具列表
|
||
const enabledTools = await getEnabledTools();
|
||
|
||
// 检查是否是 UUID 格式（数据库中的模型 ID）
|
||
const isUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
|
||
// 获取语言模型实例
|
||
let languageModel;
|
||
let actualModelId: string;
|
||
|
||
if (isUUID) {
|
||
// 从数据库获取动态配置的模型
|
||
let targetModelId = finalModelId;
|
||
|
||
// 🆕 检查是否是视频主模型，如果是，根据 videoParams.mode 选择子模型
|
||
const aiModelData = await getAiModelById({ id: finalModelId });
|
||
console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel });
|
||
|
||
if (aiModelData && !aiModelData.isSubModel) {
|
||
// 检查是否有子模型
|
||
const subModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log("[Chat API] Found sub-models:", subModels.length);
|
||
|
||
if (subModels.length > 0) {
|
||
// 这是一个主模型，需要根据 videoParams 选择子模型
|
||
const videoParams = requestBody.videoParams;
|
||
console.log("[Chat API] Video params:", videoParams);
|
||
|
||
// 如果没有 videoParams 或没有 mode，使用默认模式
|
||
const mode = videoParams?.mode || "text-to-video";
|
||
console.log("[Chat API] Using mode:", mode);
|
||
|
||
// 根据模式选择对应的子模型
|
||
const modeToModelMap: Record<string, string> = {
|
||
"text-to-video": "lite-t2v", // 文生视频 → Lite T2V
|
||
"first-last-frame": "pro", // 首尾帧 → Pro
|
||
"first-frame": "pro-fast", // 智能多帧 → Pro Fast
|
||
"reference-images": "lite-i2v", // 主体参考 → Lite I2V
|
||
};
|
||
|
||
const targetSubModelKey = modeToModelMap[mode];
|
||
if (targetSubModelKey) {
|
||
// 查找匹配的子模型（通过 modelId 中包含的关键字）
|
||
const matchedSubModel = subModels.find(({ model }) =>
|
||
model.modelId.toLowerCase().includes(targetSubModelKey)
|
||
);
|
||
|
||
if (matchedSubModel) {
|
||
console.log(`[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName}`);
|
||
targetModelId = matchedSubModel.model.id;
|
||
} else {
|
||
console.warn(`[Chat API] 未找到匹配的子模型: ${targetSubModelKey}，将使用第一个可用子模型`);
|
||
// 如果没找到匹配的，使用第一个子模型作为后备
|
||
targetModelId = subModels[0].model.id;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const dynamicModel = await getLanguageModelById(targetModelId);
|
||
if (!dynamicModel) {
|
||
return new ChatSDKError(
|
||
"bad_request:api",
|
||
"Model not found or disabled"
|
||
).toResponse();
|
||
}
|
||
languageModel = dynamicModel;
|
||
// 对于动态模型，我们需要从数据库获取 modelId
|
||
const finalModelData = await getAiModelById({ id: targetModelId });
|
||
actualModelId = finalModelData?.modelId ?? targetModelId;
|
||
} else {
|
||
// 使用预定义的模型
|
||
languageModel = myProvider.languageModel(finalModelId);
|
||
actualModelId = languageModel.modelId;
|
||
}
|
||
|
||
// ========== 图片生成配额检查 ==========
|
||
const isImageGenerationModel =
|
||
(languageModel as any).provider === "gemini-image" ||
|
||
(languageModel as any).provider === "volcano-vision" ||
|
||
(languageModel as any).provider === "alibaba-image" ||
|
||
(languageModel as any).provider === "ruxa-image" ||
|
||
(languageModel as any).provider === "huanke-gemini";
|
||
|
||
if (isImageGenerationModel) {
|
||
const quota = await checkUserImageQuota(session.user.id);
|
||
|
||
if (!quota.allowed) {
|
||
console.log(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "quota_exceeded",
|
||
message: `本月图片生成配额已用完（已使用 ${quota.used}/${quota.limit} 张），配额将于下月1日重置，如需更多配额请联系管理员。`,
|
||
used: quota.used,
|
||
limit: quota.limit,
|
||
},
|
||
{ status: 429 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`);
|
||
}
|
||
|
||
// ========== Gemini 图片生成独立处理 ==========
|
||
// 支持 gemini-image 和 huanke-gemini 两种图片生成 provider
|
||
const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider);
|
||
if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider);
|
||
console.log("[Chat API] Image params:", requestBody.imageParams);
|
||
|
||
// 🆕 获取备用模型配置
|
||
const currentModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
currentModelConfig?.fallbackModelId1,
|
||
currentModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = currentModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === currentModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Fallback model IDs:", fallbackModelIds);
|
||
|
||
const geminiStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
// 在消息末尾加上特殊标记，告诉前端"图片正在生成中"
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 🆕 尝试生成图片的函数（支持备用模型切换）
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise<string> => {
|
||
console.log(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`);
|
||
|
||
// 只传递最后一条用户消息，避免上下文历史干扰生图效果
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages);
|
||
console.log("[Chat API] Gemini: Using only last user message (no history context)");
|
||
|
||
// 转换消息：gemini-image 需要 base64 格式，huanke-gemini 需要 URL 格式
|
||
const providerForConvert = (languageModel as any).provider === "huanke-gemini" ? "huanke-gemini" : "gemini";
|
||
const modelMessages = await convertMessagesWithImages(messagesForGemini, providerForConvert);
|
||
console.log("[Chat API] Message conversion provider:", providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId）
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K" },
|
||
overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID
|
||
});
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
try {
|
||
let imageMarkdown = ""; // 初始化为空字符串
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
|
||
try {
|
||
// 首先尝试主模型
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Primary model failed:", primaryError);
|
||
|
||
// 🆕 尝试备用模型（使用同一个 provider，但切换 modelId）
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Trying fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
// 如果所有备用模型都失败，抛出原始错误触发火山引擎故障转移
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Gemini generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200));
|
||
console.log("[Chat API] Used model ID:", usedModelId);
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流 - 使用正确的 parts 格式
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库（现在是 S3 URL，可以完整保存）
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Gemini message saved successfully!");
|
||
|
||
// 提取用户 prompt（最后一条用户消息的文本）
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
actualModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Gemini generation error:", error);
|
||
console.log("[Chat API] Attempting failover to Volcano Vision...");
|
||
|
||
// 故障转移：尝试使用火山引擎模型
|
||
try {
|
||
// 查找火山引擎模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
console.log("[Chat API] Total enabled models:", allModels.length);
|
||
|
||
const volcanoModel = allModels.find(
|
||
(m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled
|
||
);
|
||
|
||
if (volcanoModel) {
|
||
console.log("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id);
|
||
|
||
// 获取火山引擎模型实例
|
||
const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id);
|
||
|
||
if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Executing Volcano Vision failover...");
|
||
|
||
// 转换消息（火山引擎会自动处理 Base64）
|
||
const volcanoMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
"volcano"
|
||
);
|
||
|
||
// 调用火山引擎生成（故障转移时只生成 1 张，保持与 Gemini 一致）
|
||
const volcanoResult = await (volcanoLanguageModel as any).doGenerate({
|
||
messages: volcanoMessages,
|
||
maxImages: 1, // 故障转移时只生成 1 张
|
||
});
|
||
|
||
const imageMarkdown = volcanoResult.text || "图片生成失败";
|
||
console.log("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（故障转移成功）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision failover message saved!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
volcanoModel.model.modelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save failover message:", saveError);
|
||
}
|
||
|
||
return; // 成功，直接返回
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] No Volcano Vision model available for failover");
|
||
} catch (failoverError) {
|
||
console.error("[Chat API] Failover to Volcano Vision also failed:", failoverError);
|
||
}
|
||
|
||
// 如果故障转移也失败，返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(geminiStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== Gemini 处理结束 ==========
|
||
|
||
// ========== 阿里云千问图片生成独立处理 ==========
|
||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image"].includes((languageModel as any).provider);
|
||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||
console.log("[Chat API] Image params:", requestBody.imageParams);
|
||
|
||
// 🆕 获取模型显示名称，用于错误提示
|
||
const alibabaModelConfig = await getAiModelById({ id: finalModelId });
|
||
const modelDisplayName = alibabaModelConfig?.displayName || actualModelId;
|
||
const configuredFallbacks = [
|
||
alibabaModelConfig?.fallbackModelId1,
|
||
alibabaModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = alibabaModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === alibabaModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds);
|
||
|
||
const alibabaStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 只传递最后一条用户消息
|
||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||
let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||
console.log("[Chat API] Alibaba: Using only last user message");
|
||
|
||
// 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用）
|
||
if (smartAutoOptimizedPrompt && messagesForAlibaba.length > 0) {
|
||
console.log("[Chat API] Using optimized prompt for API call");
|
||
messagesForAlibaba = messagesForAlibaba.map((msg) => ({
|
||
...msg,
|
||
parts: msg.parts.map((part) => {
|
||
if (part.type === "text") {
|
||
return { ...part, text: `${hiddenPromptPrefix}${smartAutoOptimizedPrompt!}`.trim() };
|
||
}
|
||
return part;
|
||
}),
|
||
}));
|
||
}
|
||
|
||
messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages);
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
// 转换消息（阿里生图支持 URL，会自动上传 base64）
|
||
const providerForConvert =
|
||
(languageModel as any).provider === "ruxa-image" ? "ruxa-image" : "alibaba-image";
|
||
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
|
||
|
||
// 调用 doGenerate 获取图片，传递模型显示名称
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 },
|
||
modelDisplayName, // 🆕 传递模型显示名称
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Alibaba primary model failed:", primaryError);
|
||
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Alibaba fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Alibaba fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Alibaba fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Alibaba generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Alibaba used model:", usedModelId);
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Alibaba message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
usedModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Alibaba message:", saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Alibaba generation error:", error);
|
||
|
||
// 返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
const errorText = error instanceof Error ? error.message : "图片生成失败，请稍后重试";
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(alibabaStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== 阿里云图片生成处理结束 ==========
|
||
|
||
// ========== 火山引擎视觉图片生成独立处理 ==========
|
||
if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Volcano Vision image generation");
|
||
|
||
// 🆕 检测是否有标注图片，如果有则在消息中注入系统提示
|
||
const hasAnnotation = hasAnnotatedImage(uiMessages);
|
||
let messagesForVolcano = uiMessagesForModel;
|
||
const volcanoModelConfig = await getAiModelById({ id: finalModelId });
|
||
const configuredFallbacks = [
|
||
volcanoModelConfig?.fallbackModelId1,
|
||
volcanoModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const sameProviderFallbacks = volcanoModelConfig
|
||
? allModels
|
||
.filter(m =>
|
||
m.provider.id === volcanoModelConfig.providerId &&
|
||
m.model.supportsImageGen &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
m.model.modelId !== actualModelId
|
||
)
|
||
.map(m => m.model.modelId)
|
||
: [];
|
||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||
console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds);
|
||
|
||
if (hasAnnotation) {
|
||
console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片，将注入系统提示");
|
||
// 创建消息副本，修改最后一条用户消息的文本
|
||
messagesForVolcano = messagesForVolcano.map((msg, idx) => {
|
||
if (msg.role === "user" && idx === messagesForVolcano.length - 1) {
|
||
// 修改最后一条用户消息
|
||
const modifiedParts = msg.parts.map((part) => {
|
||
if (part.type === "text" && part.text) {
|
||
return {
|
||
...part,
|
||
text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + part.text,
|
||
};
|
||
}
|
||
return part;
|
||
});
|
||
return { ...msg, parts: modifiedParts };
|
||
}
|
||
return msg;
|
||
});
|
||
}
|
||
|
||
messagesForVolcano = injectHistoryImagesToLastUserMessage(messagesForVolcano, smartAutoHistoryImages);
|
||
|
||
const volcanoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
// 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到
|
||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||
|
||
if (conversationMessage) {
|
||
const chatMessageId = generateUUID();
|
||
console.log("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage);
|
||
|
||
// 发送带标记的对话消息
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: chatMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: conversationMessage }],
|
||
}),
|
||
});
|
||
|
||
// 保存对话消息到数据库（不带标记，纯对话）
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: chatMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: smartAutoFriendlyMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
}
|
||
|
||
// 转换消息，火山引擎支持 URL 和 base64
|
||
const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano");
|
||
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => {
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
overrideModelId: isRetry ? modelIdToUse : undefined,
|
||
});
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
let imageMarkdown = "";
|
||
let usedModelId = actualModelId;
|
||
|
||
// 🆕 图片生成真正开始时发出标记（用于前端显示占位图）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_START -->" }],
|
||
}),
|
||
});
|
||
try {
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Volcano primary model failed:", primaryError);
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Volcano fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Volcano fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Volcano fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
console.log("[Chat API] Volcano Vision generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200));
|
||
|
||
// 生成消息 ID（图片消息）
|
||
const messageId = generateUUID();
|
||
|
||
// 写入图片消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（成功）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 保存图片消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
usedModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Volcano Vision generation error:", error);
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
}),
|
||
});
|
||
|
||
// 🆕 图片生成结束标记（失败）
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: generateUUID(),
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "<!-- IMAGE_GENERATION_END -->" }],
|
||
}),
|
||
});
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [], // 必须提供 attachments 字段
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(volcanoStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== 火山引擎视觉处理结束 ==========
|
||
|
||
// ========== 千问视频生成独立处理 ==========
|
||
if ((languageModel as any).provider === "qianwen-video" && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Qianwen video generation");
|
||
|
||
const { scheduleVideoUrlReplacement } = await import("@/lib/ai/providers/qianwen-video");
|
||
|
||
const qianwenVideoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
// 千问视频直接使用原始 URL，不转换为 base64
|
||
const modelMessages = convertToModelMessages(uiMessagesForModel);
|
||
console.log("[Chat API] Qianwen video messages prepared, count:", modelMessages.length);
|
||
|
||
// 调用 doGenerate 获取视频
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
});
|
||
|
||
const videoMarkdown = result.text || "视频生成失败";
|
||
console.log("[Chat API] Qianwen video generated, length:", videoMarkdown.length);
|
||
|
||
// 提取临时视频 URL 和提示词（用于后续替换）
|
||
const videoUrlMatch = videoMarkdown.match(/<video\s+src="([^"]+)"/);
|
||
const tempVideoUrl = videoUrlMatch ? videoUrlMatch[1] : "";
|
||
const promptMatch = videoMarkdown.match(/\*\*提示词\*\*:\s*(.+)/);
|
||
const prompt = promptMatch ? promptMatch[1].trim() : "生成视频";
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Qianwen video message saved successfully!");
|
||
|
||
// 启动异步视频替换任务（5分钟后执行）
|
||
if (tempVideoUrl) {
|
||
console.log("[Chat API] 🕐 Scheduling video URL replacement in 5 minutes...");
|
||
scheduleVideoUrlReplacement(id, messageId, tempVideoUrl, prompt);
|
||
}
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Qianwen video message:", saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Qianwen video generation error:", error);
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "视频生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(qianwenVideoStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== 千问视频处理结束 ==========
|
||
|
||
const stream = createUIMessageStream({
|
||
execute: async ({ writer: dataStream }) => {
|
||
// 构建工具对象，只包含启用的工具
|
||
const tools: any = {};
|
||
const activeTools: string[] = [];
|
||
|
||
if (enabledTools.includes("getWeather")) {
|
||
tools.getWeather = getWeather;
|
||
activeTools.push("getWeather");
|
||
}
|
||
if (enabledTools.includes("createDocument")) {
|
||
tools.createDocument = createDocument({ session, dataStream });
|
||
activeTools.push("createDocument");
|
||
}
|
||
if (enabledTools.includes("updateDocument")) {
|
||
tools.updateDocument = updateDocument({ session, dataStream });
|
||
activeTools.push("updateDocument");
|
||
}
|
||
if (enabledTools.includes("requestSuggestions")) {
|
||
tools.requestSuggestions = requestSuggestions({
|
||
session,
|
||
dataStream,
|
||
});
|
||
activeTools.push("requestSuggestions");
|
||
}
|
||
if (enabledTools.includes("generateVideo")) {
|
||
tools.generateVideo = generateVideo({ dataStream });
|
||
activeTools.push("generateVideo");
|
||
}
|
||
|
||
console.log("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志
|
||
console.log("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志
|
||
|
||
// 转换消息，对于 Gemini 和千问视频将图片 URL 转换为 base64
|
||
const convertedMessages = await convertMessagesWithImages(
|
||
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
|
||
(languageModel as any).provider || ""
|
||
);
|
||
console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志
|
||
|
||
// 🆕 检测是否是视频模型
|
||
const providerType = (languageModel as any).provider;
|
||
console.log("[Chat API] Provider type for tool detection:", providerType);
|
||
const isVolcanoVideoModel = providerType === "volcano-video" || actualModelId.includes("seedance");
|
||
console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel);
|
||
|
||
// 🎬 视频模型使用专门的生成流程
|
||
if (isVolcanoVideoModel) {
|
||
console.log("[Chat API] Using video generation API");
|
||
|
||
// 导入视频生成函数
|
||
const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video");
|
||
|
||
// 从最后一条用户消息中提取提示词和图片
|
||
const lastUserMessage = uiMessages.filter(m => m.role === "user").pop();
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
|
||
// 🐛 调试：打印原始消息结构
|
||
console.log("[Chat API] 🔍 原始 lastUserMessage:", JSON.stringify(lastUserMessage, null, 2));
|
||
|
||
if (lastUserMessage) {
|
||
// 🆕 处理 parts 数组（AI SDK v4 格式）
|
||
if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) {
|
||
console.log("[Chat API] 🔍 Found parts array, length:", (lastUserMessage as any).parts.length);
|
||
|
||
for (const part of (lastUserMessage as any).parts) {
|
||
console.log("[Chat API] 🔍 Processing part:", JSON.stringify(part, null, 2));
|
||
|
||
if (part.type === "text" && part.text) {
|
||
// 提取文本
|
||
prompt += part.text;
|
||
console.log("[Chat API] 🔍 ✅ Extracted text:", part.text);
|
||
} else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) {
|
||
// 提取图片
|
||
imageUrls.push(part.url);
|
||
console.log("[Chat API] 🔍 ✅ Extracted image URL:", part.url);
|
||
}
|
||
}
|
||
}
|
||
// 备用方案1：content 字符串/数组
|
||
const legacyContent = (lastUserMessage as any).content;
|
||
if (typeof legacyContent === "string") {
|
||
prompt = legacyContent;
|
||
} else if (Array.isArray(legacyContent)) {
|
||
for (const part of legacyContent) {
|
||
if (typeof part === "string") {
|
||
prompt += part;
|
||
} else if (part.type === "text") {
|
||
prompt += part.text;
|
||
} else if (part.type === "image" && (part as any).image) {
|
||
imageUrls.push((part as any).image);
|
||
}
|
||
}
|
||
} else if ((lastUserMessage as any).experimental_attachments) {
|
||
// 备用方案2：experimental_attachments
|
||
for (const attachment of (lastUserMessage as any).experimental_attachments) {
|
||
if (attachment.url && attachment.contentType?.startsWith("image/")) {
|
||
imageUrls.push(attachment.url);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] 🔍 最终提取结果 - prompt:", prompt, "imageCount:", imageUrls.length);
|
||
console.log("[Chat API] 🔍 imageUrls:", imageUrls);
|
||
|
||
try {
|
||
console.log("[Chat API] Video generation params:", {
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoParams: requestBody.videoParams,
|
||
});
|
||
|
||
// 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID
|
||
let videoModelId = finalModelId; // 默认使用选择的模型 ID
|
||
|
||
// 检查是否是主模型，需要选择子模型
|
||
const selectedModelData = await getAiModelById({ id: finalModelId });
|
||
if (selectedModelData && !selectedModelData.isSubModel) {
|
||
// 是主模型，根据 mode 选择子模型
|
||
const videoSubModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`);
|
||
|
||
if (videoSubModels.length > 0) {
|
||
const videoMode = requestBody.videoParams?.mode || "text-to-video";
|
||
const modeToModelMap: Record<string, string> = {
|
||
"text-to-video": "lite-t2v",
|
||
"first-last-frame": "pro",
|
||
"first-frame": "pro-fast",
|
||
"reference-images": "lite-i2v",
|
||
};
|
||
|
||
const targetKey = modeToModelMap[videoMode];
|
||
const matchedModel = videoSubModels.find(({ model }) =>
|
||
model.modelId.toLowerCase().includes(targetKey)
|
||
);
|
||
|
||
if (matchedModel) {
|
||
videoModelId = matchedModel.model.id;
|
||
console.log(`[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} for mode ${videoMode}`);
|
||
} else {
|
||
videoModelId = videoSubModels[0].model.id;
|
||
console.log(`[Chat API] Video: Using fallback sub-model ${videoSubModels[0].model.displayName}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 构建视频生成选项
|
||
const videoOptions: any = {
|
||
model: actualModelId,
|
||
prompt: prompt || "生成视频",
|
||
mode: requestBody.videoParams?.mode || "text-to-video",
|
||
resolution: requestBody.videoParams?.resolution || "720p",
|
||
ratio: requestBody.videoParams?.ratio || "16:9",
|
||
duration: requestBody.videoParams?.duration || 5,
|
||
fps: requestBody.videoParams?.fps || 24,
|
||
cameraMove: requestBody.videoParams?.cameraMove || "none",
|
||
cameraFixed: requestBody.videoParams?.cameraFixed || false,
|
||
watermark: requestBody.videoParams?.watermark || false,
|
||
seed: requestBody.videoParams?.seed,
|
||
};
|
||
|
||
console.log("[Chat API] Video options constructed:", videoOptions);
|
||
|
||
// 根据模式添加图片
|
||
if (videoOptions.mode === "first-frame" && imageUrls[0]) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
} else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
videoOptions.lastFrameImage = imageUrls[1];
|
||
} else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) {
|
||
videoOptions.referenceImages = imageUrls;
|
||
}
|
||
|
||
// 获取 provider 配置（使用确定的视频模型 UUID）
|
||
console.log("[Chat API] Getting model config for:", videoModelId);
|
||
const aiModelData = await getAiModelById({ id: videoModelId });
|
||
if (!aiModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 如果是子模型，获取父模型的 Provider 配置
|
||
let providerModelData = aiModelData;
|
||
if (aiModelData.isSubModel && aiModelData.parentModelId) {
|
||
console.log("[Chat API] This is a sub-model, getting parent model config:", aiModelData.parentModelId);
|
||
const parentModel = await getAiModelById({ id: aiModelData.parentModelId });
|
||
if (parentModel) {
|
||
providerModelData = parentModel;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Getting provider config for:", providerModelData.providerId);
|
||
const providerData = await getAiProviderById({ id: providerModelData.providerId });
|
||
if (!providerData) {
|
||
throw new Error("Provider 配置未找到");
|
||
}
|
||
|
||
const apiKey = providerData.apiKey;
|
||
const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3";
|
||
|
||
console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey });
|
||
|
||
// 调用视频生成流
|
||
console.log("[Chat API] Starting video generation stream...");
|
||
const videoStream = doGenerateStream(apiKey, baseURL, videoOptions);
|
||
|
||
// 收集所有流输出
|
||
console.log("[Chat API] Processing video stream...");
|
||
let fullText = "";
|
||
let chunkCount = 0;
|
||
|
||
for await (const chunk of videoStream) {
|
||
chunkCount++;
|
||
fullText += chunk;
|
||
console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`);
|
||
}
|
||
|
||
console.log(`[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}`);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 一次性写入完整消息（与千问视频相同）
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: fullText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库（提取视频URL用于归档）
|
||
try {
|
||
// 从 fullText 中提取视频URL
|
||
const videoUrlMatch = fullText.match(/<video[^>]+src="([^"]+)"/);
|
||
const temporaryVideoUrl = videoUrlMatch ? videoUrlMatch[1] : null;
|
||
|
||
// 构建视频元数据
|
||
const videoMetadata = temporaryVideoUrl ? {
|
||
provider: "volcano-video",
|
||
mode: requestBody.videoParams?.mode,
|
||
resolution: requestBody.videoParams?.resolution,
|
||
ratio: requestBody.videoParams?.ratio,
|
||
duration: requestBody.videoParams?.duration,
|
||
createdAt: Date.now(),
|
||
} : null;
|
||
|
||
await saveMessages({
|
||
messages: [{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: fullText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: temporaryVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: temporaryVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: videoMetadata,
|
||
}],
|
||
});
|
||
console.log("[Chat API] ✅ Video message saved successfully!");
|
||
if (temporaryVideoUrl) {
|
||
console.log("[Chat API] 📹 临时视频URL已保存，将在8小时后自动归档到S3");
|
||
}
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save video message:", saveError);
|
||
}
|
||
|
||
console.log("[Chat API] Video generation finished successfully");
|
||
return;
|
||
} catch (error: any) {
|
||
console.error("[Chat API] ❌ Video generation error:", error);
|
||
console.error("[Chat API] Error stack:", error.stack);
|
||
|
||
// 生成错误消息 ID
|
||
const errorMessageId = generateUUID();
|
||
const errorMessage = `❌ 视频生成失败：${error.message}\n\n请检查配置或稍后重试。`;
|
||
|
||
// 使用正确的格式发送错误消息
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存错误消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
}],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 常规文本模型使用 streamText
|
||
const result = streamText({
|
||
model: languageModel,
|
||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||
messages: convertedMessages,
|
||
stopWhen: stepCountIs(5),
|
||
experimental_activeTools:
|
||
finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools,
|
||
experimental_transform: smoothStream({ chunking: "word" }),
|
||
tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具
|
||
experimental_telemetry: {
|
||
isEnabled: isProductionEnvironment,
|
||
functionId: "stream-text",
|
||
},
|
||
onFinish: async ({ usage }) => {
|
||
try {
|
||
const providers = await getTokenlensCatalog();
|
||
if (!actualModelId) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
if (!providers) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
const summary = getUsage({
|
||
modelId: actualModelId,
|
||
usage,
|
||
providers,
|
||
});
|
||
finalMergedUsage = {
|
||
...usage,
|
||
...summary,
|
||
modelId: actualModelId,
|
||
} as AppUsage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
} catch (err) {
|
||
console.warn("TokenLens enrichment failed", err);
|
||
finalMergedUsage = usage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
}
|
||
},
|
||
});
|
||
|
||
result.consumeStream();
|
||
|
||
dataStream.merge(
|
||
result.toUIMessageStream({
|
||
sendReasoning: true,
|
||
})
|
||
);
|
||
},
|
||
generateId: generateUUID,
|
||
onFinish: async ({ messages }) => {
|
||
await saveMessages({
|
||
messages: messages.map((currentMessage) => ({
|
||
id: currentMessage.id,
|
||
role: currentMessage.role,
|
||
parts: currentMessage.parts,
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
})),
|
||
});
|
||
|
||
// 检测助手消息中是否包含视频，如果有则启动异步替换任务
|
||
for (const msg of messages) {
|
||
if (msg.role === "assistant" && msg.parts) {
|
||
const textContent = msg.parts
|
||
.filter((part: any) => part && part.type === "text" && part.text)
|
||
.map((part: any) => part.text)
|
||
.join("");
|
||
|
||
// 检测是否包含视频标签
|
||
const videoUrlMatch = textContent?.match(/<video[^>]+src="([^"]+)"/);
|
||
if (videoUrlMatch) {
|
||
const tempVideoUrl = videoUrlMatch[1];
|
||
const promptMatch = textContent?.match(/\*\*提示词\*\*:\s*(.+)/);
|
||
const prompt = promptMatch ? promptMatch[1].trim() : "生成视频";
|
||
|
||
console.log("[Chat API] 检测到视频消息，启动URL替换任务");
|
||
console.log("[Chat API] 临时视频URL:", tempVideoUrl);
|
||
|
||
// 动态导入并启动替换任务（5分钟后执行）
|
||
import("@/lib/ai/providers/qianwen-video").then(({ scheduleVideoUrlReplacement }) => {
|
||
scheduleVideoUrlReplacement(id, msg.id, tempVideoUrl, prompt);
|
||
}).catch((err) => {
|
||
console.error("[Chat API] 启动视频替换任务失败:", err);
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
if (finalMergedUsage) {
|
||
try {
|
||
await updateChatLastContextById({
|
||
chatId: id,
|
||
context: finalMergedUsage,
|
||
});
|
||
} catch (err) {
|
||
console.warn("Unable to persist last usage for chat", id, err);
|
||
}
|
||
}
|
||
},
|
||
onError: () => {
|
||
return "Oops, an error occurred!";
|
||
},
|
||
});
|
||
|
||
// const streamContext = getStreamContext();
|
||
|
||
// if (streamContext) {
|
||
// return new Response(
|
||
// await streamContext.resumableStream(streamId, () =>
|
||
// stream.pipeThrough(new JsonToSseTransformStream())
|
||
// )
|
||
// );
|
||
// }
|
||
|
||
return new Response(stream.pipeThrough(new JsonToSseTransformStream()));
|
||
} catch (error) {
|
||
const vercelId = request.headers.get("x-vercel-id");
|
||
|
||
if (error instanceof ChatSDKError) {
|
||
return error.toResponse();
|
||
}
|
||
|
||
// Check for Vercel AI Gateway credit card error
|
||
if (
|
||
error instanceof Error &&
|
||
error.message?.includes(
|
||
"AI Gateway requires a valid credit card on file to service requests"
|
||
)
|
||
) {
|
||
return new ChatSDKError("bad_request:activate_gateway").toResponse();
|
||
}
|
||
|
||
console.error("Unhandled error in chat API:", error, { vercelId });
|
||
return new ChatSDKError("offline:chat").toResponse();
|
||
}
|
||
}
|
||
|
||
export async function DELETE(request: Request) {
|
||
const { searchParams } = new URL(request.url);
|
||
const id = searchParams.get("id");
|
||
|
||
if (!id) {
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
|
||
if (chat?.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
const deletedChat = await deleteChatById({ id });
|
||
|
||
return Response.json(deletedChat, { status: 200 });
|
||
}
|