1618 lines
67 KiB
TypeScript
1618 lines
67 KiB
TypeScript
import { geolocation } from "@vercel/functions";
|
||
import {
|
||
convertToModelMessages,
|
||
createUIMessageStream,
|
||
JsonToSseTransformStream,
|
||
smoothStream,
|
||
stepCountIs,
|
||
streamText,
|
||
} from "ai";
|
||
import { unstable_cache as cache } from "next/cache";
|
||
import { after, NextResponse } from "next/server";
|
||
import {
|
||
createResumableStreamContext,
|
||
type ResumableStreamContext,
|
||
} from "resumable-stream";
|
||
import type { ModelCatalog } from "tokenlens/core";
|
||
import { fetchModels } from "tokenlens/fetch";
|
||
import { getUsage } from "tokenlens/helpers";
|
||
import { auth, type UserType } from "@/app/(auth)/auth";
|
||
import type { VisibilityType } from "@/components/visibility-selector";
|
||
import { entitlementsByUserType } from "@/lib/ai/entitlements";
|
||
import type { ChatModel } from "@/lib/ai/models";
|
||
import { type RequestHints, systemPrompt } from "@/lib/ai/prompts";
|
||
import { myProvider } from "@/lib/ai/providers";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { createDocument } from "@/lib/ai/tools/create-document";
|
||
import { getWeather } from "@/lib/ai/tools/get-weather";
|
||
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
|
||
import { updateDocument } from "@/lib/ai/tools/update-document";
|
||
import { generateVideo } from "@/lib/ai/tools/generate-video";
|
||
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
||
import { isProductionEnvironment } from "@/lib/constants";
|
||
import {
|
||
createStreamId,
|
||
deleteChatById,
|
||
getChatById,
|
||
getMessageCountByUserId,
|
||
getMessagesByChatId,
|
||
saveChat,
|
||
saveMessages,
|
||
updateChatLastContextById,
|
||
getAiModelById,
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId,
|
||
getAiProviderById,
|
||
checkUserImageQuota,
|
||
} from "@/lib/db/queries";
|
||
import { db } from "@/lib/db";
|
||
import type { DBMessage } from "@/lib/db/schema";
|
||
import { generatedImage } from "@/lib/db/schema";
|
||
import { ChatSDKError } from "@/lib/errors";
|
||
import type { ChatMessage } from "@/lib/types";
|
||
import type { AppUsage } from "@/lib/usage";
|
||
import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
||
import { generateTitleFromUserMessage } from "../../actions";
|
||
import { type PostRequestBody, postRequestBodySchema } from "./schema";
|
||
|
||
/**
|
||
* 将图片 URL 转换为 base64 格式
|
||
* 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API
|
||
*/
|
||
async function convertImageUrlToBase64(url: string): Promise<string> {
|
||
try {
|
||
console.log("[Image Convert] Fetching image from:", url);
|
||
const response = await fetch(url);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to fetch image: ${response.statusText}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
const base64 = buffer.toString("base64");
|
||
|
||
// 获取 MIME 类型
|
||
const contentType = response.headers.get("content-type") || "image/jpeg";
|
||
const dataUrl = `data:${contentType};base64,${base64}`;
|
||
|
||
console.log("[Image Convert] Converted to base64, size:", base64.length);
|
||
return dataUrl;
|
||
} catch (error) {
|
||
console.error("[Image Convert] Failed:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头）
|
||
*/
|
||
function hasAnnotatedImage(messages: ChatMessage[]): boolean {
|
||
for (const message of messages) {
|
||
for (const part of message.parts) {
|
||
if (part.type === "file" && "name" in part && typeof part.name === "string") {
|
||
if (part.name.startsWith("annotated-")) {
|
||
return true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* 标注图片的系统提示词（用户不可见，后台自动添加）
|
||
*/
|
||
const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。";
|
||
|
||
/**
|
||
* 自定义消息转换函数，支持将图片 URL 转换为 base64（仅用于 Gemini）
|
||
*/
|
||
async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) {
|
||
// 千问视频需要真实的 HTTP URL，不能用 base64
|
||
// 只有 Gemini 需要 base64
|
||
if (providerType !== "gemini") {
|
||
return convertToModelMessages(messages);
|
||
}
|
||
|
||
// 🆕 检测是否有标注图片
|
||
const hasAnnotation = hasAnnotatedImage(messages);
|
||
if (hasAnnotation) {
|
||
console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示");
|
||
}
|
||
|
||
// 对于 Gemini，需要特殊处理图片
|
||
const convertedMessages = [];
|
||
|
||
for (const message of messages) {
|
||
const content: any[] = [];
|
||
|
||
// 🆕 如果是用户消息且有标注图片，在文本前添加系统提示
|
||
let hasAddedAnnotationPrompt = false;
|
||
|
||
// 处理消息的各个部分
|
||
for (const part of message.parts) {
|
||
if (part.type === "text" && part.text) {
|
||
let textContent = part.text;
|
||
|
||
// 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) {
|
||
textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent;
|
||
hasAddedAnnotationPrompt = true;
|
||
console.log("[Message Convert] 🎨 已注入标注提示词");
|
||
}
|
||
|
||
content.push({
|
||
type: "text",
|
||
text: textContent,
|
||
});
|
||
} else if (part.type === "file" && part.url) {
|
||
// 将图片 URL 转换为 base64
|
||
try {
|
||
const base64Data = await convertImageUrlToBase64(part.url);
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: base64Data,
|
||
},
|
||
});
|
||
console.log("[Message Convert] Image converted to base64 and formatted as image_url");
|
||
} catch (error) {
|
||
console.error("[Message Convert] Failed to convert image, using URL:", error);
|
||
// 失败时仍然尝试使用 URL
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: part.url,
|
||
},
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
// 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本
|
||
if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) {
|
||
content.unshift({
|
||
type: "text",
|
||
text: ANNOTATION_SYSTEM_PROMPT,
|
||
});
|
||
console.log("[Message Convert] 🎨 已在消息开头注入标注提示词");
|
||
}
|
||
|
||
convertedMessages.push({
|
||
role: message.role,
|
||
content: content.length === 1 && content[0].type === "text"
|
||
? content[0].text
|
||
: content,
|
||
});
|
||
}
|
||
|
||
return convertedMessages;
|
||
}
|
||
|
||
/**
|
||
* 解析生图 Markdown 并保存图片记录到数据库
|
||
* 支持三种格式：thumb|||url|||size、thumb$$url$$size、thumb|url|size
|
||
*/
|
||
async function saveGeneratedImages(
|
||
imageMarkdown: string,
|
||
userId: string,
|
||
chatId: string,
|
||
messageId: string,
|
||
prompt: string,
|
||
modelId: string
|
||
): Promise<void> {
|
||
// 匹配 Markdown 图片格式: 
|
||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||
const matches = [...imageMarkdown.matchAll(imageRegex)];
|
||
|
||
for (const match of matches) {
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = match[2];
|
||
let originalUrl = thumbnailUrl;
|
||
let imageSize = "2K";
|
||
|
||
// 解析 alt 中的原图 URL 和尺寸
|
||
// 格式1: thumb|||originalUrl|||size (新格式)
|
||
if (altText.includes("thumb|||")) {
|
||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||
const parts = content.split("|||");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式2: thumb$$originalUrl$$size (中间格式，可能被 LaTeX 解析器破坏)
|
||
else if (altText.includes("thumb$$")) {
|
||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||
const parts = content.split("$$");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
// 格式3: thumb|originalUrl|size (旧格式)
|
||
else if (altText.includes("thumb|")) {
|
||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||
const parts = content.split("|");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
imageSize = parts[1] || "2K";
|
||
}
|
||
|
||
// 回退策略：从缩略图 URL 推断原图 URL
|
||
if (originalUrl === thumbnailUrl) {
|
||
if (thumbnailUrl.includes("gemini-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("gemini-thumb-", "gemini-orig-");
|
||
} else if (thumbnailUrl.includes("openai-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("openai-thumb-", "openai-orig-");
|
||
} else if (thumbnailUrl.includes("volcano-thumb-")) {
|
||
originalUrl = thumbnailUrl.replace("volcano-thumb-", "volcano-orig-");
|
||
}
|
||
}
|
||
|
||
try {
|
||
await db.insert(generatedImage).values({
|
||
userId,
|
||
chatId,
|
||
messageId,
|
||
thumbnailUrl,
|
||
originalUrl,
|
||
imageSize,
|
||
prompt,
|
||
modelId,
|
||
});
|
||
console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`);
|
||
} catch (error) {
|
||
console.error("[Gallery] Failed to save image record:", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务
|
||
|
||
let globalStreamContext: ResumableStreamContext | null = null;
|
||
|
||
const getTokenlensCatalog = cache(
|
||
async (): Promise<ModelCatalog | undefined> => {
|
||
try {
|
||
return await fetchModels();
|
||
} catch (err) {
|
||
console.warn(
|
||
"TokenLens: catalog fetch failed, using default catalog",
|
||
err
|
||
);
|
||
return; // tokenlens helpers will fall back to defaultCatalog
|
||
}
|
||
},
|
||
["tokenlens-catalog"],
|
||
{ revalidate: 24 * 60 * 60 } // 24 hours
|
||
);
|
||
|
||
export function getStreamContext() {
|
||
if (!globalStreamContext) {
|
||
try {
|
||
globalStreamContext = createResumableStreamContext({
|
||
waitUntil: after,
|
||
});
|
||
} catch (error: any) {
|
||
if (error.message.includes("REDIS_URL")) {
|
||
console.log(
|
||
" > Resumable streams are disabled due to missing REDIS_URL"
|
||
);
|
||
} else {
|
||
console.error(error);
|
||
}
|
||
}
|
||
}
|
||
|
||
return globalStreamContext;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
let requestBody: PostRequestBody;
|
||
|
||
try {
|
||
const json = await request.json();
|
||
requestBody = postRequestBodySchema.parse(json);
|
||
} catch (_) {
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
try {
|
||
const {
|
||
id,
|
||
message,
|
||
selectedChatModel,
|
||
selectedVisibilityType,
|
||
}: {
|
||
id: string;
|
||
message: ChatMessage;
|
||
selectedChatModel: ChatModel["id"];
|
||
selectedVisibilityType: VisibilityType;
|
||
} = requestBody;
|
||
|
||
let finalModelId = selectedChatModel;
|
||
|
||
// ========== Auto Mode Logic ==========
|
||
if (finalModelId === "auto") {
|
||
console.log("[Auto Mode] Detecting intent...");
|
||
const lastUserMessage = message;
|
||
const textContent = lastUserMessage.parts
|
||
.filter((p) => p.type === 'text')
|
||
.map((p) => p.text)
|
||
.join(' ');
|
||
// @ts-ignore
|
||
const hasImage = lastUserMessage.parts.some((p) => p.type === 'file' || p.type === 'image' || p.type === 'image_url');
|
||
|
||
// 获取所有可用模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
console.log("[Auto Mode] Available models:", allModels.map(m => `${m.model.displayName} (视频:${m.model.displayName.includes('视频')}, 图片:${m.model.supportsImageGen}, 视觉:${m.model.supportsVision})`));
|
||
|
||
let bestModel = null;
|
||
|
||
// 智能意图识别 - 基于关键词匹配
|
||
// 定义各类意图的关键词库
|
||
const videoKeywords = /(视频|video|movie|film|motion|mp4|avi|mov|wmv|flv|mkv|webm|m4v|生成视频|制作视频|创建视频|做视频|拍视频|录视频|录制|视频剪辑|剪辑视频|剪辑|编辑视频|sora|动起来|动画|animation|animate|animated|动态|动效|gif|clip|clips|短视频|长视频|影片|电影|微电影|宣传片|广告片|产品视频|演示视频|教程视频|mv|music video|片段|镜头|shot|场景|scene|特效|vfx|effects|转场|transition|字幕|subtitle|配音|dubbing|慢动作|快进|timelapse|延时)/i;
|
||
|
||
const imageEditKeywords = /(修图|P图|改图|调图|优化图片|美化图片|处理图片|编辑图片|编辑照片|编辑这张|编辑|edit photo|edit image|修改图片|调整图片|图片处理|照片处理|处理这张|处理照片|背景|background|去背景|换背景|改背景|背景色|去背|抠图|扣图|抠出|扣出|去除|移除|删除|擦除|消除|抹掉|清除|拿掉|弄掉|去掉|干掉|隐藏|遮挡|remove|delete|erase|wipe|clear|eliminate|change|modify|修改|变清晰|清晰化|锐化|sharpen|sharpening|模糊|blur|blurry|虚化|bokeh|散景|修复|repair|restoration|fix|还原|恢复|recover|处理|process|processing|换|改|替换|更换|转换|transform|replace|swap|exchange|调色|调整颜色|color grade|grading|配色|滤镜|filter|filters|effect|特效|effects|裁剪|剪裁|crop|cropping|cut|cutting|旋转|rotate|rotation|翻转|flip|flipping|mirror|镜像|mirroring|缩放|resize|resizing|scale|scaling|放大|enlarge|enlargement|zoom|zoom in|缩小|shrink|zoom out|增强|enhance|enhancement|improve|improvement|提升|升级|质量|quality|降噪|去噪|denoise|denoising|美颜|beauty|磨皮|smooth|smoothing|瘦脸|face slim|美白|whitening|lighten|lighting|亮度|brightness|对比度|contrast|饱和度|saturation|曝光|exposure|高光|highlight|阴影|shadow|锐度|sharpness|加水印|去水印|watermark|unwatermark|合成|composite|compositing|拼接|拼图|merge|merging|叠加|overlay|blend|blending|透明|transparency|alpha|蒙版|mask|masking)/i;
|
||
|
||
const imageGenKeywords = /(画|绘制|绘|绘画|画出|画个|画一个|画一张|画面|帮我画|给我画|画张|绘图|生成图片|生成图像|生成照片|生成一个|生成张|创建图片|创建图像|做图|制图|作图|画图|画画|想要.*图|需要.*图|来张|来个|draw|paint|sketch|素描|速写|render|rendering|渲染|picture|photo|image|图像|图片|照片|生图|出图|生成一张|生成|create image|generate image|make image|produce|design|designing|设计|设计一个|插画|illustration|illustrate|漫画|comic|卡通|cartoon|anime|动漫|二次元|海报|poster|logo|标志|商标|图标|icon|banner|横幅|背景图|背景|壁纸|wallpaper|桌面|配图|封面|cover|缩略图|thumbnail|头像|profile|avatar|artwork|艺术作品|art|artistic|美术|fine art|素材|material|asset|概念图|concept art|平面|graphic|graphics|视觉|visual|可视化|visualization|infographic|信息图)/i;
|
||
|
||
const visionKeywords = /(看|看看|看一下|看下|识别|认出|辨认|recognize|identify|这是什么|是什么|what is|what\'s|分析|analyze|analysis|描述|describe|description|解释|explain|讲解|tell me|说说|介绍|理解|understand|检测|detect|detection|找|找出|find|发现|observe|观察|examine|检查|读取|read|阅读|翻译图片|ocr|文字识别|提取文字|图片.*内容|照片.*内容|image.*content)/i;
|
||
|
||
if (videoKeywords.test(textContent)) {
|
||
console.log("[Auto Mode] Intent: VIDEO - 视频生成");
|
||
bestModel = allModels.find(m => m.model.displayName.includes("视频"));
|
||
}
|
||
// 2. 图片编辑：用户上传了图片并要求编辑
|
||
else if (hasImage && imageEditKeywords.test(textContent)) {
|
||
console.log("[Auto Mode] Intent: IMAGE_EDIT - 图片编辑");
|
||
bestModel = allModels.find(m => m.model.displayName.includes("图片编辑") || m.model.displayName.includes("编辑"));
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(m => m.model.supportsVision && !m.model.displayName.includes("视频"));
|
||
}
|
||
}
|
||
// 3. 图片生成：用户要求画图/生成图片（不需要上传图片）
|
||
else if (imageGenKeywords.test(textContent)) {
|
||
console.log("[Auto Mode] Intent: IMAGE_GEN - 图片生成");
|
||
bestModel = allModels.find(m => m.model.displayName.includes("图片编辑") || m.model.supportsImageGen);
|
||
// 排除视频模型
|
||
if (bestModel && bestModel.model.displayName.includes("视频")) {
|
||
bestModel = allModels.find(m => (m.model.displayName.includes("图片编辑") || m.model.supportsImageGen) && !m.model.displayName.includes("视频"));
|
||
}
|
||
}
|
||
// 4. 多模态理解：用户上传了图片，想让AI看图理解内容
|
||
else if (hasImage && visionKeywords.test(textContent)) {
|
||
console.log("[Auto Mode] Intent: VISION - 多模态理解");
|
||
bestModel = allModels.find(m => m.model.displayName.includes("多模态") || m.model.supportsVision);
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(m => m.model.supportsVision && !m.model.displayName.includes("视频"));
|
||
}
|
||
}
|
||
// 5. 多模态理解：用户上传了图片但没有明确意图（默认理解图片）
|
||
else if (hasImage) {
|
||
console.log("[Auto Mode] Intent: VISION - 多模态理解（图片已上传）");
|
||
bestModel = allModels.find(m => m.model.displayName.includes("多模态") || m.model.supportsVision);
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(m => m.model.supportsVision);
|
||
}
|
||
}
|
||
// 6. 默认：普通文本对话
|
||
else {
|
||
console.log("[Auto Mode] Intent: CHAT - 普通对话");
|
||
// 优先使用多模态模型或默认模型
|
||
bestModel = allModels.find(m => m.model.displayName.includes("多模态"));
|
||
if (!bestModel) {
|
||
bestModel = allModels.find(m => m.model.isDefault);
|
||
}
|
||
if (!bestModel && allModels.length > 0) {
|
||
bestModel = allModels[0];
|
||
}
|
||
}
|
||
|
||
// 最终兜底
|
||
if (!bestModel && allModels.length > 0) {
|
||
bestModel = allModels.find(m => m.model.isDefault) || allModels[0];
|
||
}
|
||
|
||
if (bestModel) {
|
||
console.log(`[Auto Mode] ✅ Selected: ${bestModel.model.displayName} (${bestModel.model.id})`);
|
||
finalModelId = bestModel.model.id;
|
||
} else {
|
||
return new ChatSDKError("bad_request:api", "No available models for auto mode").toResponse();
|
||
}
|
||
}
|
||
// ========== End Auto Mode Logic ==========
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const userType: UserType = session.user.type;
|
||
|
||
const messageCount = await getMessageCountByUserId({
|
||
id: session.user.id,
|
||
differenceInHours: 24,
|
||
});
|
||
|
||
if (messageCount > entitlementsByUserType[userType].maxMessagesPerDay) {
|
||
return new ChatSDKError("rate_limit:chat").toResponse();
|
||
}
|
||
|
||
// ========== 使用时间限制验证（后端安全检查） ==========
|
||
try {
|
||
// 从数据库获取网站设置
|
||
const { siteSettings } = await import("@/lib/db/schema");
|
||
const [settings] = await db.select().from(siteSettings).limit(1);
|
||
|
||
if (settings?.usageTimeEnabled) {
|
||
const now = new Date();
|
||
const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
|
||
const startTime = settings.usageTimeStart || "08:00";
|
||
const endTime = settings.usageTimeEnd || "20:00";
|
||
|
||
const isWithinUsageTime = currentTime >= startTime && currentTime <= endTime;
|
||
|
||
if (!isWithinUsageTime) {
|
||
console.log(`[Chat API] ⛔ 访问被拒绝：当前时间 ${currentTime} 不在允许范围 ${startTime}-${endTime}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "usage_time_restricted",
|
||
message: settings.usageTimeMessage || "当前不在系统使用时间段内",
|
||
currentTime,
|
||
allowedTime: `${startTime} - ${endTime}`
|
||
},
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 时间验证通过：${currentTime} 在允许范围 ${startTime}-${endTime}`);
|
||
}
|
||
} catch (timeCheckError) {
|
||
console.error("[Chat API] ⚠️ 时间验证失败（继续执行）:", timeCheckError);
|
||
// 如果时间检查失败，继续执行（避免因检查逻辑错误导致系统不可用）
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
let messagesFromDb: DBMessage[] = [];
|
||
|
||
if (chat) {
|
||
if (chat.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
// Only fetch messages if chat already exists
|
||
messagesFromDb = await getMessagesByChatId({ id });
|
||
} else {
|
||
const title = await generateTitleFromUserMessage({
|
||
message,
|
||
});
|
||
|
||
await saveChat({
|
||
id,
|
||
userId: session.user.id,
|
||
title,
|
||
visibility: selectedVisibilityType,
|
||
});
|
||
// New chat - no need to fetch messages, it's empty
|
||
}
|
||
|
||
const uiMessages = [...convertToUIMessages(messagesFromDb), message];
|
||
|
||
const { longitude, latitude, city, country } = geolocation(request);
|
||
|
||
const requestHints: RequestHints = {
|
||
longitude,
|
||
latitude,
|
||
city,
|
||
country,
|
||
};
|
||
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
chatId: id,
|
||
id: message.id,
|
||
role: "user",
|
||
parts: message.parts,
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
|
||
const streamId = generateUUID();
|
||
await createStreamId({ streamId, chatId: id });
|
||
|
||
let finalMergedUsage: AppUsage | undefined;
|
||
|
||
// 获取启用的工具列表
|
||
const enabledTools = await getEnabledTools();
|
||
|
||
// 检查是否是 UUID 格式（数据库中的模型 ID）
|
||
const isUUID =
|
||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||
finalModelId
|
||
);
|
||
|
||
// 获取语言模型实例
|
||
let languageModel;
|
||
let actualModelId: string;
|
||
|
||
if (isUUID) {
|
||
// 从数据库获取动态配置的模型
|
||
let targetModelId = finalModelId;
|
||
|
||
// 🆕 检查是否是视频主模型，如果是，根据 videoParams.mode 选择子模型
|
||
const aiModelData = await getAiModelById({ id: finalModelId });
|
||
console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel });
|
||
|
||
if (aiModelData && !aiModelData.isSubModel) {
|
||
// 检查是否有子模型
|
||
const subModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log("[Chat API] Found sub-models:", subModels.length);
|
||
|
||
if (subModels.length > 0) {
|
||
// 这是一个主模型，需要根据 videoParams 选择子模型
|
||
const videoParams = requestBody.videoParams;
|
||
console.log("[Chat API] Video params:", videoParams);
|
||
|
||
// 如果没有 videoParams 或没有 mode，使用默认模式
|
||
const mode = videoParams?.mode || "text-to-video";
|
||
console.log("[Chat API] Using mode:", mode);
|
||
|
||
// 根据模式选择对应的子模型
|
||
const modeToModelMap: Record<string, string> = {
|
||
"text-to-video": "lite-t2v", // 文生视频 → Lite T2V
|
||
"first-last-frame": "pro", // 首尾帧 → Pro
|
||
"first-frame": "pro-fast", // 智能多帧 → Pro Fast
|
||
"reference-images": "lite-i2v", // 主体参考 → Lite I2V
|
||
};
|
||
|
||
const targetSubModelKey = modeToModelMap[mode];
|
||
if (targetSubModelKey) {
|
||
// 查找匹配的子模型（通过 modelId 中包含的关键字）
|
||
const matchedSubModel = subModels.find(({ model }) =>
|
||
model.modelId.toLowerCase().includes(targetSubModelKey)
|
||
);
|
||
|
||
if (matchedSubModel) {
|
||
console.log(`[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName}`);
|
||
targetModelId = matchedSubModel.model.id;
|
||
} else {
|
||
console.warn(`[Chat API] 未找到匹配的子模型: ${targetSubModelKey}，将使用第一个可用子模型`);
|
||
// 如果没找到匹配的，使用第一个子模型作为后备
|
||
targetModelId = subModels[0].model.id;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const dynamicModel = await getLanguageModelById(targetModelId);
|
||
if (!dynamicModel) {
|
||
return new ChatSDKError(
|
||
"bad_request:api",
|
||
"Model not found or disabled"
|
||
).toResponse();
|
||
}
|
||
languageModel = dynamicModel;
|
||
// 对于动态模型，我们需要从数据库获取 modelId
|
||
const finalModelData = await getAiModelById({ id: targetModelId });
|
||
actualModelId = finalModelData?.modelId ?? targetModelId;
|
||
} else {
|
||
// 使用预定义的模型
|
||
languageModel = myProvider.languageModel(finalModelId);
|
||
actualModelId = languageModel.modelId;
|
||
}
|
||
|
||
// ========== 图片生成配额检查 ==========
|
||
const isImageGenerationModel =
|
||
(languageModel as any).provider === "gemini-image" ||
|
||
(languageModel as any).provider === "volcano-vision";
|
||
|
||
if (isImageGenerationModel) {
|
||
const quota = await checkUserImageQuota(session.user.id);
|
||
|
||
if (!quota.allowed) {
|
||
console.log(`[Chat API] ⛔ 图片配额已用完: ${quota.used}/${quota.limit}`);
|
||
return NextResponse.json(
|
||
{
|
||
error: "quota_exceeded",
|
||
message: `本月图片生成配额已用完（已使用 ${quota.used}/${quota.limit} 张），配额将于下月1日重置，如需更多配额请联系管理员。`,
|
||
used: quota.used,
|
||
limit: quota.limit,
|
||
},
|
||
{ status: 429 }
|
||
);
|
||
}
|
||
|
||
console.log(`[Chat API] ✅ 图片配额检查通过: ${quota.used}/${quota.limit ?? '无限制'}`);
|
||
}
|
||
|
||
// ========== Gemini 图片生成独立处理 ==========
|
||
if ((languageModel as any).provider === "gemini-image" && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Gemini image generation");
|
||
console.log("[Chat API] Image params:", requestBody.imageParams);
|
||
|
||
// 🆕 获取备用模型配置
|
||
const currentModelConfig = await getAiModelById({ id: finalModelId });
|
||
const fallbackModelIds = [
|
||
currentModelConfig?.fallbackModelId1,
|
||
currentModelConfig?.fallbackModelId2,
|
||
].filter(Boolean) as string[];
|
||
console.log("[Chat API] Fallback model IDs:", fallbackModelIds);
|
||
|
||
const geminiStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
// 🆕 尝试生成图片的函数（支持备用模型切换）
|
||
const tryGenerateImage = async (modelIdToUse: string, isRetry = false): Promise<string> => {
|
||
console.log(`[Chat API] ${isRetry ? "🔄 Retry with fallback" : "🚀 Trying"} model: ${modelIdToUse}`);
|
||
|
||
// 只传递最后一条用户消息，避免上下文历史干扰生图效果
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessages;
|
||
console.log("[Chat API] Gemini: Using only last user message (no history context)");
|
||
|
||
// 转换消息，Gemini 需要 base64 格式的图片
|
||
const modelMessages = await convertMessagesWithImages(messagesForGemini, "gemini");
|
||
|
||
// 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId）
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K" },
|
||
overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID
|
||
});
|
||
|
||
return result.text || "图片生成失败";
|
||
};
|
||
|
||
try {
|
||
let imageMarkdown = ""; // 初始化为空字符串
|
||
let usedModelId = actualModelId;
|
||
|
||
try {
|
||
// 首先尝试主模型
|
||
imageMarkdown = await tryGenerateImage(actualModelId, false);
|
||
} catch (primaryError) {
|
||
console.error("[Chat API] Primary model failed:", primaryError);
|
||
|
||
// 🆕 尝试备用模型（使用同一个 provider，但切换 modelId）
|
||
let fallbackSuccess = false;
|
||
for (const fallbackModelId of fallbackModelIds) {
|
||
try {
|
||
console.log(`[Chat API] 🔄 Trying fallback model: ${fallbackModelId}`);
|
||
imageMarkdown = await tryGenerateImage(fallbackModelId, true);
|
||
usedModelId = fallbackModelId;
|
||
fallbackSuccess = true;
|
||
console.log(`[Chat API] ✅ Fallback model ${fallbackModelId} succeeded!`);
|
||
break;
|
||
} catch (fallbackError) {
|
||
console.error(`[Chat API] ❌ Fallback model ${fallbackModelId} failed:`, fallbackError);
|
||
}
|
||
}
|
||
|
||
// 如果所有备用模型都失败，抛出原始错误触发火山引擎故障转移
|
||
if (!fallbackSuccess) {
|
||
throw primaryError;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Gemini generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Gemini image markdown:", imageMarkdown.substring(0, 200));
|
||
console.log("[Chat API] Used model ID:", usedModelId);
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流 - 使用正确的 parts 格式
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库（现在是 S3 URL，可以完整保存）
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Gemini message saved successfully!");
|
||
|
||
// 提取用户 prompt（最后一条用户消息的文本）
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
actualModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Gemini message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Gemini generation error:", error);
|
||
console.log("[Chat API] Attempting failover to Volcano Vision...");
|
||
|
||
// 故障转移：尝试使用火山引擎模型
|
||
try {
|
||
// 查找火山引擎模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
console.log("[Chat API] Total enabled models:", allModels.length);
|
||
|
||
const volcanoModel = allModels.find(
|
||
(m: any) => m.provider.type === "volcano-vision" && m.model.isEnabled
|
||
);
|
||
|
||
if (volcanoModel) {
|
||
console.log("[Chat API] Found Volcano Vision model for failover:", volcanoModel.model.id);
|
||
|
||
// 获取火山引擎模型实例
|
||
const volcanoLanguageModel = await getLanguageModelById(volcanoModel.model.id);
|
||
|
||
if (volcanoLanguageModel && typeof (volcanoLanguageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Executing Volcano Vision failover...");
|
||
|
||
// 转换消息（火山引擎会自动处理 Base64）
|
||
const volcanoMessages = await convertMessagesWithImages(uiMessages, "volcano");
|
||
|
||
// 调用火山引擎生成（故障转移时只生成 1 张，保持与 Gemini 一致）
|
||
const volcanoResult = await (volcanoLanguageModel as any).doGenerate({
|
||
messages: volcanoMessages,
|
||
maxImages: 1, // 故障转移时只生成 1 张
|
||
});
|
||
|
||
const imageMarkdown = volcanoResult.text || "图片生成失败";
|
||
console.log("[Chat API] Volcano Vision failover success, length:", imageMarkdown.length);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision failover message saved!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
volcanoModel.model.modelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save failover message:", saveError);
|
||
}
|
||
|
||
return; // 成功，直接返回
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] No Volcano Vision model available for failover");
|
||
} catch (failoverError) {
|
||
console.error("[Chat API] Failover to Volcano Vision also failed:", failoverError);
|
||
}
|
||
|
||
// 如果故障转移也失败，返回错误消息
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
}),
|
||
});
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(geminiStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== Gemini 处理结束 ==========
|
||
|
||
// ========== 火山引擎视觉图片生成独立处理 ==========
|
||
if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Volcano Vision image generation");
|
||
|
||
// 🆕 检测是否有标注图片，如果有则在消息中注入系统提示
|
||
const hasAnnotation = hasAnnotatedImage(uiMessages);
|
||
let messagesForVolcano = uiMessages;
|
||
|
||
if (hasAnnotation) {
|
||
console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片，将注入系统提示");
|
||
// 创建消息副本，修改最后一条用户消息的文本
|
||
messagesForVolcano = uiMessages.map((msg, idx) => {
|
||
if (msg.role === "user" && idx === uiMessages.length - 1) {
|
||
// 修改最后一条用户消息
|
||
const modifiedParts = msg.parts.map((part) => {
|
||
if (part.type === "text" && part.text) {
|
||
return {
|
||
...part,
|
||
text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + part.text,
|
||
};
|
||
}
|
||
return part;
|
||
});
|
||
return { ...msg, parts: modifiedParts };
|
||
}
|
||
return msg;
|
||
});
|
||
}
|
||
|
||
const volcanoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
// 转换消息，火山引擎支持 URL 和 base64
|
||
const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano");
|
||
|
||
// 调用 doGenerate 获取图片
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
});
|
||
|
||
const imageMarkdown = result.text || "图片生成失败";
|
||
console.log("[Chat API] Volcano Vision generated, length:", imageMarkdown.length);
|
||
console.log("[Chat API] Volcano Vision image markdown:", imageMarkdown.substring(0, 200));
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流 - 使用正确的 parts 格式
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: imageMarkdown }],
|
||
attachments: [], // 必须提供 attachments 字段（数据库 notNull）
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Volcano Vision message saved successfully!");
|
||
|
||
// 提取用户 prompt
|
||
const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop();
|
||
const userPrompt = lastUserMessage?.parts?.find((p): p is { type: "text"; text: string } => p.type === "text")?.text || "";
|
||
|
||
// 保存图片记录到 GeneratedImage 表
|
||
await saveGeneratedImages(
|
||
imageMarkdown,
|
||
session.user.id,
|
||
id,
|
||
messageId,
|
||
userPrompt,
|
||
actualModelId
|
||
);
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Volcano Vision message to database:", saveError);
|
||
console.error("[Chat API] Message content that failed:", imageMarkdown.substring(0, 300));
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Volcano Vision generation error:", error);
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
}),
|
||
});
|
||
|
||
// 尝试保存错误消息
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "图片生成失败，请稍后重试" }],
|
||
attachments: [], // 必须提供 attachments 字段
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(volcanoStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== 火山引擎视觉处理结束 ==========
|
||
|
||
// ========== 千问视频生成独立处理 ==========
|
||
if ((languageModel as any).provider === "qianwen-video" && typeof (languageModel as any).doGenerate === "function") {
|
||
console.log("[Chat API] Using Qianwen video generation");
|
||
|
||
const { scheduleVideoUrlReplacement } = await import("@/lib/ai/providers/qianwen-video");
|
||
|
||
const qianwenVideoStream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
try {
|
||
// 千问视频直接使用原始 URL，不转换为 base64
|
||
const modelMessages = convertToModelMessages(uiMessages);
|
||
console.log("[Chat API] Qianwen video messages prepared, count:", modelMessages.length);
|
||
|
||
// 调用 doGenerate 获取视频
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: modelMessages,
|
||
});
|
||
|
||
const videoMarkdown = result.text || "视频生成失败";
|
||
console.log("[Chat API] Qianwen video generated, length:", videoMarkdown.length);
|
||
|
||
// 提取临时视频 URL 和提示词（用于后续替换）
|
||
const videoUrlMatch = videoMarkdown.match(/<video\s+src="([^"]+)"/);
|
||
const tempVideoUrl = videoUrlMatch ? videoUrlMatch[1] : "";
|
||
const promptMatch = videoMarkdown.match(/\*\*提示词\*\*:\s*(.+)/);
|
||
const prompt = promptMatch ? promptMatch[1].trim() : "生成视频";
|
||
|
||
// 生成一个统一的消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 写入消息到流
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: videoMarkdown }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
console.log("[Chat API] ✅ Qianwen video message saved successfully!");
|
||
|
||
// 启动异步视频替换任务（5分钟后执行）
|
||
if (tempVideoUrl) {
|
||
console.log("[Chat API] 🕐 Scheduling video URL replacement in 5 minutes...");
|
||
scheduleVideoUrlReplacement(id, messageId, tempVideoUrl, prompt);
|
||
}
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save Qianwen video message:", saveError);
|
||
}
|
||
} catch (error) {
|
||
console.error("[Chat API] Qianwen video generation error:", error);
|
||
const errorMessageId = generateUUID();
|
||
writer.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: "视频生成失败，请稍后重试" }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(qianwenVideoStream.pipeThrough(new JsonToSseTransformStream()));
|
||
}
|
||
// ========== 千问视频处理结束 ==========
|
||
|
||
const stream = createUIMessageStream({
|
||
execute: async ({ writer: dataStream }) => {
|
||
// 构建工具对象，只包含启用的工具
|
||
const tools: any = {};
|
||
const activeTools: string[] = [];
|
||
|
||
if (enabledTools.includes("getWeather")) {
|
||
tools.getWeather = getWeather;
|
||
activeTools.push("getWeather");
|
||
}
|
||
if (enabledTools.includes("createDocument")) {
|
||
tools.createDocument = createDocument({ session, dataStream });
|
||
activeTools.push("createDocument");
|
||
}
|
||
if (enabledTools.includes("updateDocument")) {
|
||
tools.updateDocument = updateDocument({ session, dataStream });
|
||
activeTools.push("updateDocument");
|
||
}
|
||
if (enabledTools.includes("requestSuggestions")) {
|
||
tools.requestSuggestions = requestSuggestions({
|
||
session,
|
||
dataStream,
|
||
});
|
||
activeTools.push("requestSuggestions");
|
||
}
|
||
if (enabledTools.includes("generateVideo")) {
|
||
tools.generateVideo = generateVideo({ dataStream });
|
||
activeTools.push("generateVideo");
|
||
}
|
||
|
||
console.log("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志
|
||
console.log("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志
|
||
|
||
// 转换消息，对于 Gemini 和千问视频将图片 URL 转换为 base64
|
||
const convertedMessages = await convertMessagesWithImages(uiMessages, (languageModel as any).provider || "");
|
||
console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志
|
||
|
||
// 🆕 检测是否是视频模型
|
||
const providerType = (languageModel as any).provider;
|
||
console.log("[Chat API] Provider type for tool detection:", providerType);
|
||
const isVolcanoVideoModel = providerType === "volcano-video" || actualModelId.includes("seedance");
|
||
console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel);
|
||
|
||
// 🎬 视频模型使用专门的生成流程
|
||
if (isVolcanoVideoModel) {
|
||
console.log("[Chat API] Using video generation API");
|
||
|
||
// 导入视频生成函数
|
||
const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video");
|
||
|
||
// 从最后一条用户消息中提取提示词和图片
|
||
const lastUserMessage = uiMessages.filter(m => m.role === "user").pop();
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
|
||
// 🐛 调试：打印原始消息结构
|
||
console.log("[Chat API] 🔍 原始 lastUserMessage:", JSON.stringify(lastUserMessage, null, 2));
|
||
|
||
if (lastUserMessage) {
|
||
// 🆕 处理 parts 数组（AI SDK v4 格式）
|
||
if ((lastUserMessage as any).parts && Array.isArray((lastUserMessage as any).parts)) {
|
||
console.log("[Chat API] 🔍 Found parts array, length:", (lastUserMessage as any).parts.length);
|
||
|
||
for (const part of (lastUserMessage as any).parts) {
|
||
console.log("[Chat API] 🔍 Processing part:", JSON.stringify(part, null, 2));
|
||
|
||
if (part.type === "text" && part.text) {
|
||
// 提取文本
|
||
prompt += part.text;
|
||
console.log("[Chat API] 🔍 ✅ Extracted text:", part.text);
|
||
} else if (part.type === "file" && part.url && part.mediaType?.startsWith("image/")) {
|
||
// 提取图片
|
||
imageUrls.push(part.url);
|
||
console.log("[Chat API] 🔍 ✅ Extracted image URL:", part.url);
|
||
}
|
||
}
|
||
}
|
||
// 备用方案1：content 字符串
|
||
else if (typeof lastUserMessage.content === "string") {
|
||
prompt = lastUserMessage.content;
|
||
}
|
||
// 备用方案2：content 数组
|
||
else if (Array.isArray(lastUserMessage.content)) {
|
||
for (const part of lastUserMessage.content) {
|
||
if (typeof part === "string") {
|
||
prompt += part;
|
||
} else if (part.type === "text") {
|
||
prompt += part.text;
|
||
} else if (part.type === "image" && (part as any).image) {
|
||
imageUrls.push((part as any).image);
|
||
}
|
||
}
|
||
}
|
||
// 备用方案3：experimental_attachments
|
||
else if ((lastUserMessage as any).experimental_attachments) {
|
||
for (const attachment of (lastUserMessage as any).experimental_attachments) {
|
||
if (attachment.url && attachment.contentType?.startsWith("image/")) {
|
||
imageUrls.push(attachment.url);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] 🔍 最终提取结果 - prompt:", prompt, "imageCount:", imageUrls.length);
|
||
console.log("[Chat API] 🔍 imageUrls:", imageUrls);
|
||
|
||
try {
|
||
console.log("[Chat API] Video generation params:", {
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoParams: requestBody.videoParams,
|
||
});
|
||
|
||
// 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID
|
||
let videoModelId = finalModelId; // 默认使用选择的模型 ID
|
||
|
||
// 检查是否是主模型，需要选择子模型
|
||
const selectedModelData = await getAiModelById({ id: finalModelId });
|
||
if (selectedModelData && !selectedModelData.isSubModel) {
|
||
// 是主模型，根据 mode 选择子模型
|
||
const videoSubModels = await getSubModelsByParentModelId(finalModelId);
|
||
console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`);
|
||
|
||
if (videoSubModels.length > 0) {
|
||
const videoMode = requestBody.videoParams?.mode || "text-to-video";
|
||
const modeToModelMap: Record<string, string> = {
|
||
"text-to-video": "lite-t2v",
|
||
"first-last-frame": "pro",
|
||
"first-frame": "pro-fast",
|
||
"reference-images": "lite-i2v",
|
||
};
|
||
|
||
const targetKey = modeToModelMap[videoMode];
|
||
const matchedModel = videoSubModels.find(({ model }) =>
|
||
model.modelId.toLowerCase().includes(targetKey)
|
||
);
|
||
|
||
if (matchedModel) {
|
||
videoModelId = matchedModel.model.id;
|
||
console.log(`[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} for mode ${videoMode}`);
|
||
} else {
|
||
videoModelId = videoSubModels[0].model.id;
|
||
console.log(`[Chat API] Video: Using fallback sub-model ${videoSubModels[0].model.displayName}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 构建视频生成选项
|
||
const videoOptions: any = {
|
||
model: actualModelId,
|
||
prompt: prompt || "生成视频",
|
||
mode: requestBody.videoParams?.mode || "text-to-video",
|
||
resolution: requestBody.videoParams?.resolution || "720p",
|
||
ratio: requestBody.videoParams?.ratio || "16:9",
|
||
duration: requestBody.videoParams?.duration || 5,
|
||
fps: requestBody.videoParams?.fps || 24,
|
||
cameraMove: requestBody.videoParams?.cameraMove || "none",
|
||
cameraFixed: requestBody.videoParams?.cameraFixed || false,
|
||
watermark: requestBody.videoParams?.watermark || false,
|
||
seed: requestBody.videoParams?.seed,
|
||
};
|
||
|
||
console.log("[Chat API] Video options constructed:", videoOptions);
|
||
|
||
// 根据模式添加图片
|
||
if (videoOptions.mode === "first-frame" && imageUrls[0]) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
} else if (videoOptions.mode === "first-last-frame" && imageUrls.length >= 2) {
|
||
videoOptions.firstFrameImage = imageUrls[0];
|
||
videoOptions.lastFrameImage = imageUrls[1];
|
||
} else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) {
|
||
videoOptions.referenceImages = imageUrls;
|
||
}
|
||
|
||
// 获取 provider 配置（使用确定的视频模型 UUID）
|
||
console.log("[Chat API] Getting model config for:", videoModelId);
|
||
const aiModelData = await getAiModelById({ id: videoModelId });
|
||
if (!aiModelData) {
|
||
throw new Error("视频模型配置未找到");
|
||
}
|
||
|
||
// 如果是子模型，获取父模型的 Provider 配置
|
||
let providerModelData = aiModelData;
|
||
if (aiModelData.isSubModel && aiModelData.parentModelId) {
|
||
console.log("[Chat API] This is a sub-model, getting parent model config:", aiModelData.parentModelId);
|
||
const parentModel = await getAiModelById({ id: aiModelData.parentModelId });
|
||
if (parentModel) {
|
||
providerModelData = parentModel;
|
||
}
|
||
}
|
||
|
||
console.log("[Chat API] Getting provider config for:", providerModelData.providerId);
|
||
const providerData = await getAiProviderById({ id: providerModelData.providerId });
|
||
if (!providerData) {
|
||
throw new Error("Provider 配置未找到");
|
||
}
|
||
|
||
const apiKey = providerData.apiKey;
|
||
const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3";
|
||
|
||
console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey });
|
||
|
||
// 调用视频生成流
|
||
console.log("[Chat API] Starting video generation stream...");
|
||
const videoStream = doGenerateStream(apiKey, baseURL, videoOptions);
|
||
|
||
// 收集所有流输出
|
||
console.log("[Chat API] Processing video stream...");
|
||
let fullText = "";
|
||
let chunkCount = 0;
|
||
|
||
for await (const chunk of videoStream) {
|
||
chunkCount++;
|
||
fullText += chunk;
|
||
console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`);
|
||
}
|
||
|
||
console.log(`[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}`);
|
||
|
||
// 生成消息 ID
|
||
const messageId = generateUUID();
|
||
|
||
// 一次性写入完整消息（与千问视频相同）
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: messageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: fullText }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存消息到数据库（提取视频URL用于归档）
|
||
try {
|
||
// 从 fullText 中提取视频URL
|
||
const videoUrlMatch = fullText.match(/<video[^>]+src="([^"]+)"/);
|
||
const temporaryVideoUrl = videoUrlMatch ? videoUrlMatch[1] : null;
|
||
|
||
// 构建视频元数据
|
||
const videoMetadata = temporaryVideoUrl ? {
|
||
provider: "volcano-video",
|
||
mode: requestBody.videoParams?.mode,
|
||
resolution: requestBody.videoParams?.resolution,
|
||
ratio: requestBody.videoParams?.ratio,
|
||
duration: requestBody.videoParams?.duration,
|
||
createdAt: Date.now(),
|
||
} : null;
|
||
|
||
await saveMessages({
|
||
messages: [{
|
||
id: messageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: fullText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: temporaryVideoUrl ? "temporary" : null,
|
||
temporaryVideoUrl: temporaryVideoUrl,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: videoMetadata,
|
||
}],
|
||
});
|
||
console.log("[Chat API] ✅ Video message saved successfully!");
|
||
if (temporaryVideoUrl) {
|
||
console.log("[Chat API] 📹 临时视频URL已保存，将在8小时后自动归档到S3");
|
||
}
|
||
} catch (saveError) {
|
||
console.error("[Chat API] ❌ Failed to save video message:", saveError);
|
||
}
|
||
|
||
console.log("[Chat API] Video generation finished successfully");
|
||
return;
|
||
} catch (error: any) {
|
||
console.error("[Chat API] ❌ Video generation error:", error);
|
||
console.error("[Chat API] Error stack:", error.stack);
|
||
|
||
// 生成错误消息 ID
|
||
const errorMessageId = generateUUID();
|
||
const errorMessage = `❌ 视频生成失败：${error.message}\n\n请检查配置或稍后重试。`;
|
||
|
||
// 使用正确的格式发送错误消息
|
||
dataStream.write({
|
||
type: "data-appendMessage",
|
||
data: JSON.stringify({
|
||
id: errorMessageId,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
}),
|
||
});
|
||
|
||
// 保存错误消息到数据库
|
||
try {
|
||
await saveMessages({
|
||
messages: [{
|
||
id: errorMessageId,
|
||
chatId: id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: errorMessage }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
}],
|
||
});
|
||
} catch (saveError) {
|
||
console.error("[Chat API] Failed to save error message:", saveError);
|
||
}
|
||
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 常规文本模型使用 streamText
|
||
const result = streamText({
|
||
model: languageModel,
|
||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||
messages: convertedMessages,
|
||
stopWhen: stepCountIs(5),
|
||
experimental_activeTools:
|
||
finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools,
|
||
experimental_transform: smoothStream({ chunking: "word" }),
|
||
tools: isVolcanoVideoModel ? {} : tools, // 🆕 视频模型禁用工具
|
||
experimental_telemetry: {
|
||
isEnabled: isProductionEnvironment,
|
||
functionId: "stream-text",
|
||
},
|
||
onFinish: async ({ usage }) => {
|
||
try {
|
||
const providers = await getTokenlensCatalog();
|
||
if (!actualModelId) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
if (!providers) {
|
||
finalMergedUsage = usage;
|
||
dataStream.write({
|
||
type: "data-usage",
|
||
data: finalMergedUsage,
|
||
});
|
||
return;
|
||
}
|
||
|
||
const summary = getUsage({
|
||
modelId: actualModelId,
|
||
usage,
|
||
providers,
|
||
});
|
||
finalMergedUsage = {
|
||
...usage,
|
||
...summary,
|
||
modelId: actualModelId,
|
||
} as AppUsage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
} catch (err) {
|
||
console.warn("TokenLens enrichment failed", err);
|
||
finalMergedUsage = usage;
|
||
dataStream.write({ type: "data-usage", data: finalMergedUsage });
|
||
}
|
||
},
|
||
});
|
||
|
||
result.consumeStream();
|
||
|
||
dataStream.merge(
|
||
result.toUIMessageStream({
|
||
sendReasoning: true,
|
||
})
|
||
);
|
||
},
|
||
generateId: generateUUID,
|
||
onFinish: async ({ messages }) => {
|
||
await saveMessages({
|
||
messages: messages.map((currentMessage) => ({
|
||
id: currentMessage.id,
|
||
role: currentMessage.role,
|
||
parts: currentMessage.parts,
|
||
createdAt: new Date(),
|
||
attachments: [],
|
||
chatId: id,
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
})),
|
||
});
|
||
|
||
// 检测助手消息中是否包含视频，如果有则启动异步替换任务
|
||
for (const msg of messages) {
|
||
if (msg.role === "assistant" && msg.parts) {
|
||
const textContent = msg.parts
|
||
.filter((part: any) => part && part.type === "text" && part.text)
|
||
.map((part: any) => part.text)
|
||
.join("");
|
||
|
||
// 检测是否包含视频标签
|
||
const videoUrlMatch = textContent?.match(/<video[^>]+src="([^"]+)"/);
|
||
if (videoUrlMatch) {
|
||
const tempVideoUrl = videoUrlMatch[1];
|
||
const promptMatch = textContent?.match(/\*\*提示词\*\*:\s*(.+)/);
|
||
const prompt = promptMatch ? promptMatch[1].trim() : "生成视频";
|
||
|
||
console.log("[Chat API] 检测到视频消息，启动URL替换任务");
|
||
console.log("[Chat API] 临时视频URL:", tempVideoUrl);
|
||
|
||
// 动态导入并启动替换任务（5分钟后执行）
|
||
import("@/lib/ai/providers/qianwen-video").then(({ scheduleVideoUrlReplacement }) => {
|
||
scheduleVideoUrlReplacement(id, msg.id, tempVideoUrl, prompt);
|
||
}).catch((err) => {
|
||
console.error("[Chat API] 启动视频替换任务失败:", err);
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
if (finalMergedUsage) {
|
||
try {
|
||
await updateChatLastContextById({
|
||
chatId: id,
|
||
context: finalMergedUsage,
|
||
});
|
||
} catch (err) {
|
||
console.warn("Unable to persist last usage for chat", id, err);
|
||
}
|
||
}
|
||
},
|
||
onError: () => {
|
||
return "Oops, an error occurred!";
|
||
},
|
||
});
|
||
|
||
// const streamContext = getStreamContext();
|
||
|
||
// if (streamContext) {
|
||
// return new Response(
|
||
// await streamContext.resumableStream(streamId, () =>
|
||
// stream.pipeThrough(new JsonToSseTransformStream())
|
||
// )
|
||
// );
|
||
// }
|
||
|
||
return new Response(stream.pipeThrough(new JsonToSseTransformStream()));
|
||
} catch (error) {
|
||
const vercelId = request.headers.get("x-vercel-id");
|
||
|
||
if (error instanceof ChatSDKError) {
|
||
return error.toResponse();
|
||
}
|
||
|
||
// Check for Vercel AI Gateway credit card error
|
||
if (
|
||
error instanceof Error &&
|
||
error.message?.includes(
|
||
"AI Gateway requires a valid credit card on file to service requests"
|
||
)
|
||
) {
|
||
return new ChatSDKError("bad_request:activate_gateway").toResponse();
|
||
}
|
||
|
||
console.error("Unhandled error in chat API:", error, { vercelId });
|
||
return new ChatSDKError("offline:chat").toResponse();
|
||
}
|
||
}
|
||
|
||
export async function DELETE(request: Request) {
|
||
const { searchParams } = new URL(request.url);
|
||
const id = searchParams.get("id");
|
||
|
||
if (!id) {
|
||
return new ChatSDKError("bad_request:api").toResponse();
|
||
}
|
||
|
||
const session = await auth();
|
||
|
||
if (!session?.user) {
|
||
return new ChatSDKError("unauthorized:chat").toResponse();
|
||
}
|
||
|
||
const chat = await getChatById({ id });
|
||
|
||
if (chat?.userId !== session.user.id) {
|
||
return new ChatSDKError("forbidden:chat").toResponse();
|
||
}
|
||
|
||
const deletedChat = await deleteChatById({ id });
|
||
|
||
return Response.json(deletedChat, { status: 200 });
|
||
}
|