diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 49ed6341f..4c5cc7a7b 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -121,7 +121,7 @@ type ToolConfig = { type AiProvider = { id: string; name: string; - type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "volcano-multimodal"; + type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "volcano-multimodal"; baseUrl: string; apiKey: string; isEnabled: boolean; @@ -373,6 +373,7 @@ const PROVIDER_TYPES = [ { value: "gemini", label: "Gemini中转" }, { value: "huanke-gemini", label: "环科Gemini生图" }, // 🆕 环科 Gemini 原生格式 API { value: "qianwen-video", label: "千问视频生成" }, + { value: "doubao-video", label: "豆包AI视频" }, { value: "volcano-vision", label: "火山引擎视觉" }, { value: "volcano-video", label: "火山引擎视频" }, { value: "fal-video", label: "FAL 视频生成" }, @@ -952,7 +953,7 @@ export function AdminSettingsClient() { const [providerForm, setProviderForm] = useState({ id: "", name: "", - type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "volcano-multimodal", + type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "volcano-multimodal", baseUrl: "", apiKey: "", isEnabled: true, @@ -2978,6 +2979,7 @@ export function AdminSettingsClient() { "gemini": "bg-purple-500", "volcano-vision": "bg-orange-500", "volcano-video": "bg-red-500", + "doubao-video": "bg-red-400", "fal-video": "bg-indigo-500", "alibaba": "bg-blue-500", "alibaba-image": "bg-amber-500", @@ -5442,6 +5444,8 @@ export function AdminSettingsClient() { ? "https://fal.run" : !prev.baseUrl && value === "fal-video" ? "https://fal.run" + : !prev.baseUrl && value === "doubao-video" + ? "https://ark.cn-beijing.volces.com/api/v3" : !prev.baseUrl && value === "zhipu" ? "https://open.bigmodel.cn/api/paas/v4" : !prev.baseUrl && value === "volcano-multimodal" @@ -5473,6 +5477,8 @@ export function AdminSettingsClient() { ? "https://fal.run" : providerForm.type === "fal-video" ? "https://fal.run" + : providerForm.type === "doubao-video" + ? "https://ark.cn-beijing.volces.com/api/v3" : providerForm.type === "zhipu" ? "https://open.bigmodel.cn/api/paas/v4" : providerForm.type === "volcano-multimodal" diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 96917c7e6..86a229712 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -41,6 +41,7 @@ import { getMessagesByChatId, saveChat, saveMessages, + updateMessageById, updateChatLastContextById, getAiModelById, getEnabledAiModelsWithProvider, @@ -51,7 +52,7 @@ import { } from "@/lib/db/queries"; import { db } from "@/lib/db"; import type { DBMessage } from "@/lib/db/schema"; -import { generatedImage } from "@/lib/db/schema"; +import { generatedImage, message as messageTable } from "@/lib/db/schema"; import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; import { uploadFile } from "@/lib/storage/service"; @@ -187,6 +188,121 @@ function applyOptimizedPromptToLatestUserMessage( return cloned; } +type StreamWriter = { + write: (chunk: any) => void; +}; + +function safeWriteStreamChunk(writer: StreamWriter, chunk: any, label: string): boolean { + try { + writer.write(chunk); + return true; + } catch (error) { + console.warn(`[Chat API] Stream write skipped (${label})`, error); + return false; + } +} + +function writeAssistantTextChunk( + writer: StreamWriter, + text: string, + label: string, + messageId = generateUUID(), +) { + safeWriteStreamChunk( + writer, + { + type: "data-appendMessage", + data: JSON.stringify({ + id: messageId, + role: "assistant", + parts: [{ type: "text", text }], + }), + }, + label, + ); + return messageId; +} + +function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs = 15000) { + const timer = setInterval(() => { + safeWriteStreamChunk( + writer, + { + type: "data-keepalive", + data: JSON.stringify({ ts: Date.now(), label }), + }, + `${label}:keepalive`, + ); + }, intervalMs); + + return () => clearInterval(timer); +} + +async function replayAssistantMessagesForDuplicateRequest({ + chatId, + userMessageId, + writer, + source, + waitMs = DUPLICATE_REQUEST_WAIT_MS, +}: { + chatId: string; + userMessageId: string; + writer: StreamWriter; + source: string; + waitMs?: number; +}) { + const startedAt = Date.now(); + const seenAssistantMessageIds = new Set(); + + while (Date.now() - startedAt < waitMs) { + try { + const dbMessages = await getMessagesByChatId({ id: chatId }); + const userMessageIndex = dbMessages.findIndex((msg) => msg.id === userMessageId); + + if (userMessageIndex >= 0) { + const assistantMessages = dbMessages + .slice(userMessageIndex + 1) + .filter((msg) => msg.role === "assistant"); + + if (assistantMessages.length > 0) { + for (const assistantMessage of assistantMessages) { + if (seenAssistantMessageIds.has(assistantMessage.id)) continue; + seenAssistantMessageIds.add(assistantMessage.id); + + safeWriteStreamChunk( + writer, + { + type: "data-appendMessage", + data: JSON.stringify({ + id: assistantMessage.id, + role: "assistant", + parts: assistantMessage.parts, + }), + }, + `${source}:replay`, + ); + } + + return; + } + } + } catch (error) { + console.warn(`[Chat API] Failed duplicate replay fetch (${source})`, error); + } + + safeWriteStreamChunk( + writer, + { + type: "data-keepalive", + data: JSON.stringify({ ts: Date.now(), source }), + }, + `${source}:keepalive`, + ); + + await sleep(1000); + } +} + type ChatImageParams = NonNullable; type AgentRecommendedImageParams = { aspectRatio?: string; @@ -528,28 +644,38 @@ function buildAutoModeFallbackMessage(params: { text: string; hasImage: boolean; fromHistory: boolean; + imageSummary?: string; }): string { const demand = compactAutoDemand(params.text); if (params.hasImage) { const sourceLine = params.fromHistory - ? "我会基于你上一轮图片继续处理。" - : "我会基于你当前上传图片处理。"; + ? "基于上一轮图片继续处理。" + : "基于当前上传图片处理。"; return [ - "收到，我先帮你梳理这次修改目标。", - `- 需求理解：${demand}`, - `- 执行基准：${sourceLine}`, - "- 执行方式：优先只改你指定区域，主体、构图和风格尽量保持一致。", - "- 继续迭代：生成后你可直接说“保留A，改B”，我会按最新要求继续优化。", + "### 需求理解", + `- ${demand}`, + "### 图片分析", + params.imageSummary + ? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}` + : `- ${sourceLine}`, + "- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。", + "### 执行计划", + "- 先完成本轮核心改动，再检查边缘与光影一致性。", + "- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。", + "### 输出设置", + "- 默认单图输出，优先稳定命中。", ].join("\n"); } return [ - "收到，我来按自动模式帮你生成。", - `- 需求理解：${demand}`, - "- 执行方式：先锁定主体和场景，再细化风格、材质与光影。", - "- 继续迭代：你可以继续补充偏好（如构图、镜头、色调、质感）。", + "### 需求理解", + `- ${demand}`, + "### 执行计划", + "- 先锁定主体与场景，再细化风格、材质与光影。", + "### 输出设置", + "- 默认单图输出，优先稳定命中。", ].join("\n"); } @@ -1068,6 +1194,42 @@ async function saveGeneratedImages( export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务 let globalStreamContext: ResumableStreamContext | null = null; +const inFlightChatRequests = new Map(); +const IN_FLIGHT_REQUEST_TTL_MS = 5 * 60 * 1000; +const DUPLICATE_REQUEST_WAIT_MS = 2 * 60 * 1000; +const SMART_AUTO_GRAPH_TIMEOUT_MS = 18000; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +class OperationTimeoutError extends Error { + constructor(operation: string, timeoutMs: number) { + super(`${operation} timed out after ${timeoutMs}ms`); + this.name = "OperationTimeoutError"; + } +} + +async function withOperationTimeout( + operation: Promise, + operationName: string, + timeoutMs: number, +): Promise { + let timer: ReturnType | null = null; + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new OperationTimeoutError(operationName, timeoutMs)), + timeoutMs, + ); + }), + ]); + } finally { + if (timer) { + clearTimeout(timer); + } + } +} const getTokenlensCatalog = cache( async (): Promise => { @@ -1107,6 +1269,8 @@ export function getStreamContext() { export async function POST(request: Request) { let requestBody: PostRequestBody; + let requestDedupKey: string | null = null; + let acquiredInFlightLock = false; try { const json = await request.json(); @@ -1134,6 +1298,33 @@ export async function POST(request: Request) { autoContext?: string; } = requestBody; + const now = Date.now(); + for (const [key, startedAt] of inFlightChatRequests.entries()) { + if (now - startedAt > IN_FLIGHT_REQUEST_TTL_MS) { + inFlightChatRequests.delete(key); + } + } + + requestDedupKey = `${id}:${message.id}`; + if (inFlightChatRequests.has(requestDedupKey)) { + console.warn( + `[Chat API] Duplicate in-flight request ignored (chatId=${id}, messageId=${message.id})` + ); + const duplicateStream = createUIMessageStream({ + execute: async ({ writer }) => { + await replayAssistantMessagesForDuplicateRequest({ + chatId: id, + userMessageId: message.id, + writer, + source: "duplicate-in-flight", + }); + }, + }); + return new Response(duplicateStream.pipeThrough(new JsonToSseTransformStream())); + } + inFlightChatRequests.set(requestDedupKey, now); + acquiredInFlightLock = true; + let finalModelId = selectedChatModel; // 🆕 存储智能模式的相关信息 @@ -1187,6 +1378,7 @@ export async function POST(request: Request) { ]); const fixedVideoProviderTypes = new Set([ "volcano-video", + "doubao-video", "qianwen-video", "fal-video", ]); @@ -1366,14 +1558,18 @@ export async function POST(request: Request) { // 2. 使用 LangGraph 驱动的自动模式工作流 const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph"); - const autoGraphResult = await runAutoAgentGraph({ - text: textContent, - hasImage, - imageUrls, - imageFromHistory: smartAutoHistoryImages.length > 0, - historyMessages, - context: autoContextText || undefined, - }); + const autoGraphResult = await withOperationTimeout( + runAutoAgentGraph({ + text: textContent, + hasImage, + imageUrls, + imageFromHistory: smartAutoHistoryImages.length > 0, + historyMessages, + context: autoContextText || undefined, + }), + "runAutoAgentGraph", + SMART_AUTO_GRAPH_TIMEOUT_MS, + ); if (autoGraphResult.analysis) { console.log("[Smart Auto Mode] Analysis result:", { @@ -1490,16 +1686,26 @@ export async function POST(request: Request) { } catch (autoError) { // 智能分析失败时，回退到简单逻辑 console.error("[Smart Auto Mode] ❌ Error, falling back:", autoError); + const isAutoTimeout = autoError instanceof OperationTimeoutError; + if (isAutoTimeout) { + console.warn( + `[Smart Auto Mode] ⏱️ Auto-agent timed out after ${SMART_AUTO_GRAPH_TIMEOUT_MS}ms, using local fallback`, + ); + } try { const { analyzeIntent } = await import("@/lib/ai/smart-analyzer"); - const rescueAnalysis = await analyzeIntent({ - text: autoTextForFallback, - hasImage: autoHasImageForFallback || smartAutoHistoryImages.length > 0, - imageUrls: smartAutoHistoryImages, - imageFromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, - historyMessages: [], - context: undefined, - }); + const rescueAnalysis = await withOperationTimeout( + analyzeIntent({ + text: autoTextForFallback, + hasImage: autoHasImageForFallback || smartAutoHistoryImages.length > 0, + imageUrls: smartAutoHistoryImages, + imageFromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, + historyMessages: [], + context: undefined, + }), + "fallbackAnalyzeIntent", + 8000, + ); if (rescueAnalysis.conversationalReply?.trim()) { smartAutoFriendlyMessage = rescueAnalysis.conversationalReply.trim(); } @@ -1512,17 +1718,69 @@ export async function POST(request: Request) { const hasImage = autoHasImageForFallback || message.parts.some( (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" ); + let fallbackVisualSummary: string | undefined; + + if (isAutoTimeout && hasImage) { + try { + const extractCurrentImageUrls = (parts: any[]) => + parts + .map((part: any) => { + if (!part) return null; + if (part.type === "file") return part.url || part.data || null; + if (part.type === "image") return part.url || part.image || null; + if (part.type === "image_url") return part.image_url?.url || null; + return null; + }) + .filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url)); + + const currentImageUrls = extractCurrentImageUrls(message.parts as any[]); + const summaryImageUrls = + currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages; + + if (summaryImageUrls.length > 0) { + const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer"); + const summary = await withOperationTimeout( + summarizeImageForExecutionReport({ + text: autoTextForFallback, + imageUrls: summaryImageUrls, + context: typeof autoContext === "string" ? autoContext : undefined, + }), + "fallbackVisualSummary", + 8000, + ); + if (summary?.trim()) { + fallbackVisualSummary = summary.trim(); + } + } + } catch (summaryError) { + console.warn("[Smart Auto Mode] Fallback visual summary failed:", summaryError); + } + } // 简单回退：有图选图片模型，无图选默认模型 let bestModel = null; if (hasImage) { bestModel = allModels.find(m => m.model.supportsImageGen && !m.model.isSubModel); - smartAutoFriendlyMessage = - smartAutoFriendlyMessage || - "我正在先做这一轮图片分析，接着按你的要求执行修改。"; - } else { - smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({ - text: autoTextForFallback, + if (isAutoTimeout) { + smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ + text: autoTextForFallback, + hasImage: true, + fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, + imageSummary: fallbackVisualSummary, + }); + } else { + smartAutoFriendlyMessage = + smartAutoFriendlyMessage || + buildAutoModeFallbackMessage({ + text: autoTextForFallback, + hasImage: true, + fromHistory: autoFromHistoryForFallback || smartAutoHistoryImages.length > 0, + imageSummary: fallbackVisualSummary, + }); + } + } else { + smartAutoFriendlyMessage = smartAutoFriendlyMessage || buildAutoModeFallbackMessage({ + text: autoTextForFallback, hasImage: false, fromHistory: false, }); @@ -1715,23 +1973,42 @@ export async function POST(request: Request) { country, }; - await saveMessages({ - messages: [ - { - chatId: id, - id: message.id, - role: "user", - parts: message.parts, - attachments: [], - createdAt: new Date(), - videoStorageStatus: null, - temporaryVideoUrl: null, - permanentVideoUrl: null, - archivedAt: null, - videoMetadata: null, + const insertedUserMessage = await db + .insert(messageTable) + .values({ + chatId: id, + id: message.id, + role: "user", + parts: message.parts as any, + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }) + .onConflictDoNothing({ + target: [messageTable.id], + }) + .returning({ id: messageTable.id }); + + if (insertedUserMessage.length === 0) { + console.warn( + `[Chat API] Duplicate user message ignored (chatId=${id}, messageId=${message.id})` + ); + const duplicateStream = createUIMessageStream({ + execute: async ({ writer }) => { + await replayAssistantMessagesForDuplicateRequest({ + chatId: id, + userMessageId: message.id, + writer, + source: "duplicate-user-message", + }); }, - ], - }); + }); + return new Response(duplicateStream.pipeThrough(new JsonToSseTransformStream())); + } const streamId = generateUUID(); await createStreamId({ streamId, chatId: id }); @@ -1817,6 +2094,7 @@ export async function POST(request: Request) { let languageModel; let actualModelId: string; let resolvedModelSupportsImageGen = false; + let resolvedProviderType: string | undefined; if (isUUID) { // 从数据库获取动态配置的模型 @@ -1824,9 +2102,12 @@ export async function POST(request: Request) { // 🆕 检查是否是视频主模型，如果是，根据 videoParams.mode 选择子模型 const aiModelData = await getAiModelById({ id: finalModelId }); + const aiModelProvider = aiModelData + ? await getAiProviderById({ id: aiModelData.providerId }).catch(() => null) + : null; console.log("[Chat API] Model data:", { id: finalModelId, isSubModel: aiModelData?.isSubModel }); - if (aiModelData && !aiModelData.isSubModel) { + if (aiModelData && !aiModelData.isSubModel && aiModelProvider?.type === "volcano-video") { // 检查是否有子模型 const subModels = await getSubModelsByParentModelId(finalModelId); console.log("[Chat API] Found sub-models:", subModels.length); @@ -1851,13 +2132,22 @@ export async function POST(request: Request) { const targetSubModelKey = modeToModelMap[mode]; if (targetSubModelKey) { // 查找匹配的子模型（通过 modelId 中包含的关键字） - const matchedSubModel = subModels.find(({ model }) => + const compatibleSubModels = subModels.filter(({ model }) => { + const supportedModes = ((model.capabilities as any)?.supportedModes ?? []) as string[]; + return supportedModes.length === 0 || supportedModes.includes(mode); + }); + const matchedSubModel = compatibleSubModels.find(({ model }) => model.modelId.toLowerCase().includes(targetSubModelKey) ); if (matchedSubModel) { console.log(`[Chat API] 视频模式 ${mode} → 选择子模型: ${matchedSubModel.model.displayName}`); targetModelId = matchedSubModel.model.id; + } else if (compatibleSubModels.length > 0) { + console.warn( + `[Chat API] 未找到匹配关键字子模型: ${targetSubModelKey}，使用兼容子模型 ${compatibleSubModels[0].model.displayName}` + ); + targetModelId = compatibleSubModels[0].model.id; } else { console.warn(`[Chat API] 未找到匹配的子模型: ${targetSubModelKey}，将使用第一个可用子模型`); // 如果没找到匹配的，使用第一个子模型作为后备 @@ -1879,10 +2169,16 @@ export async function POST(request: Request) { const finalModelData = await getAiModelById({ id: targetModelId }); actualModelId = finalModelData?.modelId ?? targetModelId; resolvedModelSupportsImageGen = Boolean(finalModelData?.supportsImageGen); + if (finalModelData) { + resolvedProviderType = ( + await getAiProviderById({ id: finalModelData.providerId }).catch(() => null) + )?.type; + } } else { // 使用预定义的模型 languageModel = myProvider.languageModel(finalModelId); actualModelId = languageModel.modelId; + resolvedProviderType = (languageModel as any)?.provider; } // ========== 图片生成配额检查 ========== @@ -1994,14 +2290,12 @@ export async function POST(request: Request) { console.log("[Chat API] 💬 Gemini: Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: chatMessageId, - role: "assistant", - parts: [{ type: "text", text: conversationMessage }], - }), - }); + writeAssistantTextChunk( + writer, + conversationMessage, + "gemini:conversation", + chatMessageId, + ); // 保存对话消息到数据库（不带标记，纯对话） await saveMessages({ @@ -2057,19 +2351,18 @@ export async function POST(request: Request) { return result.text || "图片生成失败"; }; + let stopKeepalive: (() => void) | null = null; try { let imageMarkdown = ""; // 初始化为空字符串 let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "gemini:image-start", + ); + stopKeepalive = startStreamKeepalive(writer, "gemini-image"); try { // 首先尝试主模型 @@ -2106,24 +2399,19 @@ export async function POST(request: Request) { const messageId = generateUUID(); // 写入消息到流 - 使用正确的 parts 格式 - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: messageId, - role: "assistant", - parts: [{ type: "text", text: imageMarkdown }], - }), - }); + writeAssistantTextChunk( + writer, + imageMarkdown, + "gemini:image-result", + messageId, + ); // 🆕 图片生成结束标记（成功） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "gemini:image-end-success", + ); // 保存消息到数据库（现在是 S3 URL，可以完整保存） try { @@ -2205,24 +2493,19 @@ export async function POST(request: Request) { const messageId = generateUUID(); // 写入消息到流 - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: messageId, - role: "assistant", - parts: [{ type: "text", text: imageMarkdown }], - }), - }); + writeAssistantTextChunk( + writer, + imageMarkdown, + "gemini:volcano-failover-result", + messageId, + ); // 🆕 图片生成结束标记（故障转移成功） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "gemini:volcano-failover-end", + ); // 保存消息到数据库 try { @@ -2273,24 +2556,19 @@ export async function POST(request: Request) { // 如果故障转移也失败，返回错误消息 const errorMessageId = generateUUID(); - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: errorMessageId, - role: "assistant", - parts: [{ type: "text", text: "图片生成失败，请稍后重试" }], - }), - }); + writeAssistantTextChunk( + writer, + "图片生成失败，请稍后重试", + "gemini:error", + errorMessageId, + ); // 🆕 图片生成结束标记（失败） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "gemini:image-end-failed", + ); // 尝试保存错误消息 try { @@ -2314,6 +2592,8 @@ export async function POST(request: Request) { } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } + } finally { + stopKeepalive?.(); } }, }); @@ -2352,6 +2632,7 @@ export async function POST(request: Request) { const alibabaStream = createUIMessageStream({ execute: async ({ writer }) => { + let stopKeepalive: (() => void) | null = null; try { // 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到 const conversationMessage = smartAutoFriendlyMessage || null; @@ -2361,14 +2642,12 @@ export async function POST(request: Request) { console.log("[Chat API] 💬 Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: chatMessageId, - role: "assistant", - parts: [{ type: "text", text: conversationMessage }], - }), - }); + writeAssistantTextChunk( + writer, + conversationMessage, + "alibaba:conversation", + chatMessageId, + ); // 保存对话消息到数据库（不带标记，纯对话） await saveMessages({ @@ -2441,14 +2720,12 @@ export async function POST(request: Request) { let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "alibaba:image-start", + ); + stopKeepalive = startStreamKeepalive(writer, "alibaba-image"); try { imageMarkdown = await tryGenerateImage(actualModelId, false); @@ -2562,24 +2839,19 @@ export async function POST(request: Request) { const messageId = generateUUID(); // 写入图片消息到流 - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: messageId, - role: "assistant", - parts: [{ type: "text", text: imageMarkdown }], - }), - }); + writeAssistantTextChunk( + writer, + imageMarkdown, + "alibaba:image-result", + messageId, + ); // 🆕 图片生成结束标记（成功） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "alibaba:image-end-success", + ); // 保存图片消息到数据库 try { @@ -2623,24 +2895,19 @@ export async function POST(request: Request) { // 返回错误消息 const errorMessageId = generateUUID(); const errorText = error instanceof Error ? error.message : "图片生成失败，请稍后重试"; - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: errorMessageId, - role: "assistant", - parts: [{ type: "text", text: errorText }], - }), - }); + writeAssistantTextChunk( + writer, + errorText, + "alibaba:error", + errorMessageId, + ); // 🆕 图片生成结束标记（失败） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "alibaba:image-end-failed", + ); try { await saveMessages({ @@ -2663,6 +2930,8 @@ export async function POST(request: Request) { } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } + } finally { + stopKeepalive?.(); } }, }); @@ -2727,6 +2996,7 @@ export async function POST(request: Request) { const volcanoStream = createUIMessageStream({ execute: async ({ writer }) => { + let stopKeepalive: (() => void) | null = null; try { // 🆕 如果是智能模式且有对话回复，先发送对话消息让用户看到 const conversationMessage = smartAutoFriendlyMessage || null; @@ -2736,14 +3006,12 @@ export async function POST(request: Request) { console.log("[Chat API] 💬 Volcano: Sending conversational reply first:", smartAutoFriendlyMessage); // 发送带标记的对话消息 - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: chatMessageId, - role: "assistant", - parts: [{ type: "text", text: conversationMessage }], - }), - }); + writeAssistantTextChunk( + writer, + conversationMessage, + "volcano:conversation", + chatMessageId, + ); // 保存对话消息到数据库（不带标记，纯对话） await saveMessages({ @@ -2789,14 +3057,12 @@ export async function POST(request: Request) { let usedModelId = actualModelId; // 🆕 图片生成真正开始时发出标记（用于前端显示占位图） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "volcano:image-start", + ); + stopKeepalive = startStreamKeepalive(writer, "volcano-image"); try { imageMarkdown = await tryGenerateImage(actualModelId, false); } catch (primaryError) { @@ -2905,24 +3171,19 @@ export async function POST(request: Request) { const messageId = generateUUID(); // 写入图片消息到流 - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: messageId, - role: "assistant", - parts: [{ type: "text", text: imageMarkdown }], - }), - }); + writeAssistantTextChunk( + writer, + imageMarkdown, + "volcano:image-result", + messageId, + ); // 🆕 图片生成结束标记（成功） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "volcano:image-end-success", + ); // 保存图片消息到数据库 try { @@ -2965,24 +3226,19 @@ export async function POST(request: Request) { } catch (error) { console.error("[Chat API] Volcano Vision generation error:", error); const errorMessageId = generateUUID(); - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: errorMessageId, - role: "assistant", - parts: [{ type: "text", text: "图片生成失败，请稍后重试" }], - }), - }); + writeAssistantTextChunk( + writer, + "图片生成失败，请稍后重试", + "volcano:error", + errorMessageId, + ); // 🆕 图片生成结束标记（失败） - writer.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: generateUUID(), - role: "assistant", - parts: [{ type: "text", text: "" }], - }), - }); + writeAssistantTextChunk( + writer, + "", + "volcano:image-end-failed", + ); // 尝试保存错误消息 try { @@ -3006,6 +3262,8 @@ export async function POST(request: Request) { } catch (saveError) { console.error("[Chat API] Failed to save error message:", saveError); } + } finally { + stopKeepalive?.(); } }, }); @@ -3202,8 +3460,16 @@ export async function POST(request: Request) { // 🆕 检测是否是视频模型 const providerType = (languageModel as any).provider; - console.log("[Chat API] Provider type for tool detection:", providerType); - const isVolcanoVideoModel = providerType === "volcano-video" || actualModelId.includes("seedance"); + const effectiveProviderType = resolvedProviderType || providerType; + console.log("[Chat API] Provider type for tool detection:", { + runtime: providerType, + resolved: resolvedProviderType, + effective: effectiveProviderType, + }); + const isVolcanoVideoModel = + effectiveProviderType === "volcano-video" || + effectiveProviderType === "doubao-video" || + actualModelId.includes("seedance"); const isOpenAICompatImageModel = resolvedModelSupportsImageGen && providerType === "openai.chat"; console.log("[Chat API] Is volcano video model:", isVolcanoVideoModel); @@ -3436,23 +3702,42 @@ export async function POST(request: Request) { console.log("[Chat API] 🔍 最终提取结果 - prompt:", prompt, "imageCount:", imageUrls.length); console.log("[Chat API] 🔍 imageUrls:", imageUrls); + const videoMessageId = generateUUID(); + const videoStartAt = Date.now(); + const baseVideoMetadata: Record = { + provider: effectiveProviderType || "volcano-video", + mode: effectiveVideoParams?.mode || "text-to-video", + resolution: effectiveVideoParams?.resolution || "720p", + ratio: effectiveVideoParams?.ratio || "16:9", + duration: effectiveVideoParams?.duration || 5, + status: "queued", + startedAt: videoStartAt, + updatedAt: videoStartAt, + }; + try { console.log("[Chat API] Video generation params:", { prompt, imageCount: imageUrls.length, videoParams: effectiveVideoParams, }); - + // 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID - let videoModelId = finalModelId; // 默认使用选择的模型 ID - - // 检查是否是主模型，需要选择子模型 + let videoModelId = finalModelId; // 默认使用选择的模型 ID + + // 检查是否是主模型，需要选择子模型（仅 volcano-video 支持子模型） const selectedModelData = await getAiModelById({ id: finalModelId }); - if (selectedModelData && !selectedModelData.isSubModel) { - // 是主模型，根据 mode 选择子模型 + const selectedModelProvider = selectedModelData + ? await getAiProviderById({ id: selectedModelData.providerId }).catch(() => null) + : null; + if ( + selectedModelData && + !selectedModelData.isSubModel && + selectedModelProvider?.type === "volcano-video" + ) { const videoSubModels = await getSubModelsByParentModelId(finalModelId); console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`); - + if (videoSubModels.length > 0) { const videoMode = effectiveVideoParams?.mode || "text-to-video"; const modeToModelMap: Record = { @@ -3461,25 +3746,65 @@ export async function POST(request: Request) { "first-frame": "pro-fast", "reference-images": "lite-i2v", }; - + const targetKey = modeToModelMap[videoMode]; - const matchedModel = videoSubModels.find(({ model }) => + const compatibleModels = videoSubModels.filter(({ model }) => { + const supportedModes = ((model.capabilities as any)?.supportedModes ?? []) as string[]; + return supportedModes.length === 0 || supportedModes.includes(videoMode); + }); + const matchedModel = compatibleModels.find(({ model }) => model.modelId.toLowerCase().includes(targetKey) ); - + if (matchedModel) { videoModelId = matchedModel.model.id; - console.log(`[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} for mode ${videoMode}`); + console.log( + `[Chat API] Video: Selected sub-model ${matchedModel.model.displayName} for mode ${videoMode}` + ); + } else if (compatibleModels.length > 0) { + videoModelId = compatibleModels[0].model.id; + console.log( + `[Chat API] Video: Fallback to compatible sub-model ${compatibleModels[0].model.displayName} for mode ${videoMode}` + ); } else { videoModelId = videoSubModels[0].model.id; - console.log(`[Chat API] Video: Using fallback sub-model ${videoSubModels[0].model.displayName}`); + console.log( + `[Chat API] Video: No compatible sub-model found for mode ${videoMode}, using ${videoSubModels[0].model.displayName}` + ); } } } - + + console.log("[Chat API] Getting model config for:", videoModelId); + const aiModelData = await getAiModelById({ id: videoModelId }); + if (!aiModelData) { + throw new Error("视频模型配置未找到"); + } + + // 如果是子模型，获取父模型的 Provider 配置 + let providerModelData = aiModelData; + if (aiModelData.isSubModel && aiModelData.parentModelId) { + console.log( + "[Chat API] This is a sub-model, getting parent model config:", + aiModelData.parentModelId + ); + const parentModel = await getAiModelById({ id: aiModelData.parentModelId }); + if (parentModel) { + providerModelData = parentModel; + } + } + + console.log("[Chat API] Getting provider config for:", providerModelData.providerId); + const providerData = await getAiProviderById({ id: providerModelData.providerId }); + if (!providerData) { + throw new Error("Provider 配置未找到"); + } + + const actualVideoModelId = aiModelData.modelId; + // 构建视频生成选项 const videoOptions: any = { - model: actualModelId, + model: actualVideoModelId, prompt: prompt || "生成视频", mode: effectiveVideoParams?.mode || "text-to-video", resolution: effectiveVideoParams?.resolution || "720p", @@ -3491,9 +3816,7 @@ export async function POST(request: Request) { watermark: effectiveVideoParams?.watermark || false, seed: effectiveVideoParams?.seed, }; - - console.log("[Chat API] Video options constructed:", videoOptions); - + // 根据模式添加图片 if (videoOptions.mode === "first-frame" && imageUrls[0]) { videoOptions.firstFrameImage = imageUrls[0]; @@ -3503,147 +3826,190 @@ export async function POST(request: Request) { } else if (videoOptions.mode === "reference-images" && imageUrls.length > 0) { videoOptions.referenceImages = imageUrls; } - - // 获取 provider 配置（使用确定的视频模型 UUID） - console.log("[Chat API] Getting model config for:", videoModelId); - const aiModelData = await getAiModelById({ id: videoModelId }); - if (!aiModelData) { - throw new Error("视频模型配置未找到"); - } - - // 如果是子模型，获取父模型的 Provider 配置 - let providerModelData = aiModelData; - if (aiModelData.isSubModel && aiModelData.parentModelId) { - console.log("[Chat API] This is a sub-model, getting parent model config:", aiModelData.parentModelId); - const parentModel = await getAiModelById({ id: aiModelData.parentModelId }); - if (parentModel) { - providerModelData = parentModel; - } - } - - console.log("[Chat API] Getting provider config for:", providerModelData.providerId); - const providerData = await getAiProviderById({ id: providerModelData.providerId }); - if (!providerData) { - throw new Error("Provider 配置未找到"); - } - + + console.log("[Chat API] Video options constructed:", videoOptions); + + // 先写入占位消息，避免刷新后消息丢失 + const pendingText = "🎬 视频任务已创建，正在处理中，请稍候..."; + await saveMessages({ + messages: [ + { + id: videoMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: pendingText }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: "temporary", + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: { + ...baseVideoMetadata, + model: actualVideoModelId, + status: "running", + updatedAt: Date.now(), + } as any, + }, + ], + }); + + dataStream.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: videoMessageId, + role: "assistant", + parts: [{ type: "text", text: pendingText }], + attachments: [], + }), + }); + const apiKey = providerData.apiKey; const baseURL = providerData.baseUrl || "https://ark.cn-beijing.volces.com/api/v3"; - console.log("[Chat API] Provider configured:", { baseURL, hasApiKey: !!apiKey }); - + // 调用视频生成流 console.log("[Chat API] Starting video generation stream..."); const videoStream = doGenerateStream(apiKey, baseURL, videoOptions); - + // 收集所有流输出 console.log("[Chat API] Processing video stream..."); let fullText = ""; let chunkCount = 0; - + for await (const chunk of videoStream) { chunkCount++; fullText += chunk; console.log(`[Chat API] Received chunk ${chunkCount}, length: ${chunk.length}`); } - - console.log(`[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}`); - - // 生成消息 ID - const messageId = generateUUID(); - - // 一次性写入完整消息（与千问视频相同） + + console.log( + `[Chat API] Video stream completed. Total chunks: ${chunkCount}, total text length: ${fullText.length}` + ); + + // 提取流中的元数据并清理输出文案 + const metadataMatch = fullText.match(//s); + let streamMetadata: Record = {}; + if (metadataMatch?.[1]) { + try { + streamMetadata = JSON.parse(metadataMatch[1]); + } catch (parseError) { + console.warn("[Chat API] Failed to parse VIDEO_METADATA from stream", parseError); + } + } + const cleanedText = fullText + .replace(//gs, "") + .trim(); + const videoUrlMatch = cleanedText.match(/]+src="([^"]+)"/); + const temporaryVideoUrl = + (typeof streamMetadata.temporaryVideoUrl === "string" && streamMetadata.temporaryVideoUrl) || + (videoUrlMatch ? videoUrlMatch[1] : null); + + const successMetadata = { + ...baseVideoMetadata, + ...streamMetadata, + model: actualVideoModelId, + taskId: + typeof streamMetadata.taskId === "string" && streamMetadata.taskId + ? streamMetadata.taskId + : undefined, + status: "succeeded", + completedAt: Date.now(), + updatedAt: Date.now(), + }; + + await updateMessageById({ + messageId: videoMessageId, + chatId: id, + parts: [{ type: "text", text: cleanedText }], + videoStorageStatus: temporaryVideoUrl ? "temporary" : null, + temporaryVideoUrl: temporaryVideoUrl, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: successMetadata as any, + }); + dataStream.write({ type: "data-appendMessage", data: JSON.stringify({ - id: messageId, + id: videoMessageId, role: "assistant", - parts: [{ type: "text", text: fullText }], + parts: [{ type: "text", text: cleanedText }], attachments: [], }), }); - - // 保存消息到数据库（提取视频URL用于归档） - try { - // 从 fullText 中提取视频URL - const videoUrlMatch = fullText.match(/]+src="([^"]+)"/); - const temporaryVideoUrl = videoUrlMatch ? videoUrlMatch[1] : null; - - // 构建视频元数据 - const videoMetadata = temporaryVideoUrl ? { - provider: "volcano-video", - mode: effectiveVideoParams?.mode, - resolution: effectiveVideoParams?.resolution, - ratio: effectiveVideoParams?.ratio, - duration: effectiveVideoParams?.duration, - createdAt: Date.now(), - } : null; - await saveMessages({ - messages: [{ - id: messageId, - chatId: id, - role: "assistant", - parts: [{ type: "text", text: fullText }], - attachments: [], - createdAt: new Date(), - videoStorageStatus: temporaryVideoUrl ? "temporary" : null, - temporaryVideoUrl: temporaryVideoUrl, - permanentVideoUrl: null, - archivedAt: null, - videoMetadata: videoMetadata, - }], - }); - console.log("[Chat API] ✅ Video message saved successfully!"); - if (temporaryVideoUrl) { - console.log("[Chat API] 📹 临时视频URL已保存，将在8小时后自动归档到S3"); - } - } catch (saveError) { - console.error("[Chat API] ❌ Failed to save video message:", saveError); + console.log("[Chat API] ✅ Video message updated successfully"); + if (temporaryVideoUrl) { + console.log("[Chat API] 📹 临时视频URL已保存，将在8小时后自动归档到S3"); } - + console.log("[Chat API] Video generation finished successfully"); return; } catch (error: any) { console.error("[Chat API] ❌ Video generation error:", error); console.error("[Chat API] Error stack:", error.stack); - - // 生成错误消息 ID - const errorMessageId = generateUUID(); + const errorMessage = `❌ 视频生成失败：${error.message}\n\n请检查配置或稍后重试。`; - - // 使用正确的格式发送错误消息 - dataStream.write({ - type: "data-appendMessage", - data: JSON.stringify({ - id: errorMessageId, - role: "assistant", - parts: [{ type: "text", text: errorMessage }], - attachments: [], - }), - }); - - // 保存错误消息到数据库 + const failedMetadata = { + ...baseVideoMetadata, + status: "failed", + error: error?.message || "未知错误", + updatedAt: Date.now(), + }; + try { - await saveMessages({ - messages: [{ - id: errorMessageId, - chatId: id, + await updateMessageById({ + messageId: videoMessageId, + chatId: id, + parts: [{ type: "text", text: errorMessage }], + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: failedMetadata as any, + }); + + dataStream.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: videoMessageId, role: "assistant", parts: [{ type: "text", text: errorMessage }], attachments: [], - createdAt: new Date(), - videoStorageStatus: null, - temporaryVideoUrl: null, - permanentVideoUrl: null, - archivedAt: null, - videoMetadata: null, - }], + }), + }); + } catch (updateError) { + console.error("[Chat API] Failed to update placeholder video message:", updateError); + const fallbackMessageId = generateUUID(); + dataStream.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: fallbackMessageId, + role: "assistant", + parts: [{ type: "text", text: errorMessage }], + attachments: [], + }), + }); + await saveMessages({ + messages: [ + { + id: fallbackMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: errorMessage }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: failedMetadata as any, + }, + ], }); - } catch (saveError) { - console.error("[Chat API] Failed to save error message:", saveError); } - + return; } } @@ -3809,6 +4175,10 @@ export async function POST(request: Request) { console.error("Unhandled error in chat API:", error, { vercelId }); return new ChatSDKError("offline:chat").toResponse(); + } finally { + if (requestDedupKey && acquiredInFlightLock) { + inFlightChatRequests.delete(requestDedupKey); + } } } diff --git a/app/(chat)/api/video-tasks/route.ts b/app/(chat)/api/video-tasks/route.ts new file mode 100644 index 000000000..2b77c2b24 --- /dev/null +++ b/app/(chat)/api/video-tasks/route.ts @@ -0,0 +1,72 @@ +import { auth } from "@/app/(auth)/auth"; +import { getChatById, getPendingVideoMessagesByChatId } from "@/lib/db/queries"; + +export const dynamic = "force-dynamic"; +export const revalidate = 0; + +export async function GET(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) { + return new Response(JSON.stringify({ error: "Unauthorized" }), { + status: 401, + headers: { "Content-Type": "application/json" }, + }); + } + + const { searchParams } = new URL(request.url); + const chatId = searchParams.get("chatId"); + if (!chatId) { + return new Response(JSON.stringify({ error: "Missing chatId" }), { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + + const selectedChat = await getChatById({ id: chatId }); + if (!selectedChat || selectedChat.userId !== session.user.id) { + return new Response(JSON.stringify({ error: "Forbidden" }), { + status: 403, + headers: { "Content-Type": "application/json" }, + }); + } + + const pendingMessages = await getPendingVideoMessagesByChatId({ chatId }); + const tasks = pendingMessages.map((msg) => { + const metadata = (msg.videoMetadata as Record) || {}; + return { + messageId: msg.id, + taskId: typeof metadata.taskId === "string" ? metadata.taskId : null, + provider: + typeof metadata.provider === "string" ? metadata.provider : null, + model: typeof metadata.model === "string" ? metadata.model : null, + mode: typeof metadata.mode === "string" ? metadata.mode : null, + status: + typeof metadata.status === "string" ? metadata.status : "queued", + startedAt: metadata.startedAt ?? null, + updatedAt: metadata.updatedAt ?? null, + }; + }); + + return new Response( + JSON.stringify({ + chatId, + count: tasks.length, + tasks, + }), + { + status: 200, + headers: { + "Content-Type": "application/json", + "Cache-Control": "no-store, max-age=0", + }, + } + ); + } catch (error) { + console.error("[Video Tasks API] GET error:", error); + return new Response(JSON.stringify({ error: "Internal server error" }), { + status: 500, + headers: { "Content-Type": "application/json" }, + }); + } +} diff --git a/app/api/admin/ai-providers/route.ts b/app/api/admin/ai-providers/route.ts index 2d0759d4c..6d23ee89c 100644 --- a/app/api/admin/ai-providers/route.ts +++ b/app/api/admin/ai-providers/route.ts @@ -17,6 +17,7 @@ const providerTypes = [ "gemini", "huanke-gemini", "qianwen-video", + "doubao-video", "volcano-vision", "volcano-video", "alibaba-image", @@ -48,6 +49,7 @@ const updateProviderSchema = z.object({ const ZHIPU_BASE_URL = "https://open.bigmodel.cn/api/paas/v4"; const VOLCANO_MULTIMODAL_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3"; +const DOUBAO_VIDEO_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3"; function normalizeProviderTypeForStorage( type?: ProviderType @@ -152,6 +154,8 @@ export async function POST(request: Request) { ? data.baseUrl || ZHIPU_BASE_URL : data.type === "volcano-multimodal" ? data.baseUrl || VOLCANO_MULTIMODAL_BASE_URL + : data.type === "doubao-video" + ? data.baseUrl || DOUBAO_VIDEO_BASE_URL : data.baseUrl, updatedBy: session.user.id, }); diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 87f4bb7cf..027e0ff73 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -1131,7 +1131,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF }>("/api/site-settings", fetcher); const aiDetailGenerationEnabled = siteSettings?.aiDetailGenerationEnabled === true; - const VIDEO_PROVIDER_TYPES = new Set(["volcano-video", "qianwen-video", "fal-video"]); + const VIDEO_PROVIDER_TYPES = new Set(["volcano-video", "doubao-video", "qianwen-video", "fal-video"]); const visualModels = modelsData?.models?.filter( (model) => model.supportsImageGen || VIDEO_PROVIDER_TYPES.has(model.providerType) diff --git a/components/chat.tsx b/components/chat.tsx index 8672c4a33..ad9d09f7c 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -159,6 +159,44 @@ function getMessageTextLength(message?: ChatMessage | null) { .length; } +function hasLikelyGenerationErrorMessage(message?: ChatMessage | null) { + if (!message?.parts || message.parts.length === 0) return false; + const text = message.parts + .filter((part: any) => part?.type === "text" && typeof part.text === "string") + .map((part: any) => part.text) + .join("\n") + .trim(); + if (!text) return false; + return /失败|超时|重试|error|failed/i.test(text); +} + +function getAssistantMessagesAfterTargetUser( + dbMessages: ChatMessage[], + targetUserMessageId?: string | null, +) { + if (!Array.isArray(dbMessages) || dbMessages.length === 0) return [] as ChatMessage[]; + + let targetUserIndex = -1; + + if (targetUserMessageId) { + targetUserIndex = dbMessages.findIndex( + (message) => message.role === "user" && message.id === targetUserMessageId, + ); + } + + if (targetUserIndex < 0) { + targetUserIndex = dbMessages.findLastIndex((message) => message.role === "user"); + } + + if (targetUserIndex < 0) { + return dbMessages.filter((message) => message.role === "assistant"); + } + + return dbMessages + .slice(targetUserIndex + 1) + .filter((message) => message.role === "assistant"); +} + export function Chat({ id, initialMessages, @@ -187,6 +225,7 @@ export function Chat({ // 🆕 持久化生成任务状态（解决用户离开再返回时加载状态丢失） const { pendingGeneration, startGeneration, completeGeneration } = usePendingGeneration(id); + const pendingGenerationRef = useRef(pendingGeneration); // 🆕 标记是否已经尝试刷新过（避免无限刷新） const hasTriedRefreshRef = useRef(false); @@ -198,6 +237,7 @@ export function Chat({ const [imageGenerationStartedAt, setImageGenerationStartedAt] = useState(null); const imageGenerationPhaseRef = useRef("idle"); const imagePlaceholderTimerRef = useRef | null>(null); + const startedGenerationMessageIdRef = useRef(null); // 🆕 存储通过 data-appendMessage 添加的消息，避免与 useChat 内部状态冲突 const [appendedMessages, setAppendedMessages] = useState([]); @@ -226,6 +266,10 @@ export function Chat({ } }, [pendingGeneration, initialMessages, completeGeneration, router]); + useEffect(() => { + pendingGenerationRef.current = pendingGeneration; + }, [pendingGeneration]); + const updateImageGenerationPhase = useCallback((nextPhase: ImageGenerationPhase) => { imageGenerationPhaseRef.current = nextPhase; setImageGenerationPhase(nextPhase); @@ -331,6 +375,7 @@ export function Chat({ const currentModel = modelsData?.models?.find(m => m.id === currentModelId); const isVideoModel = currentModel?.providerType === "volcano-video" || + currentModel?.providerType === "doubao-video" || currentModel?.providerType === "qianwen-video" || currentModel?.providerType === "fal-video"; const isImageModel = currentModel?.supportsImageGen === true; @@ -526,6 +571,7 @@ export function Chat({ imagePlaceholderTimerRef.current = null; } updateImageGenerationPhase("idle"); + completeGeneration(); } // 🆕 将消息添加到 appendedMessages 状态，而不是直接修改 useChat 的 messages @@ -562,10 +608,8 @@ export function Chat({ if (isLikelyGenerationError) { console.log("[Chat] ❌ Generation error detected, reset loading state"); updateImageGenerationPhase("failed"); + completeGeneration(); } - - // 🆕 收到消息，清除持久化状态 - completeGeneration(); } catch (e) { console.error("[Chat] Failed to parse data-appendMessage:", e); } @@ -573,9 +617,15 @@ export function Chat({ }, onFinish: async () => { mutate(unstable_serialize(getChatHistoryPaginationKey)); - // 🆕 生成完成，清除持久化状态 - completeGeneration(); console.log("[Chat] onFinish called"); + const pendingSnapshot = pendingGenerationRef.current; + const keepPendingUntilDbSync = + pendingSnapshot?.modelType === "gemini" || + pendingSnapshot?.modelType === "volcano-vision" || + pendingSnapshot?.modelType === "alibaba-image"; + if (!keepPendingUntilDbSync) { + completeGeneration(); + } // 🆕 从数据库重新加载消息，确保 data-appendMessage 的消息正确显示 // 这是根治方案 B：后端已保存消息到数据库，前端从数据库重新加载 @@ -596,40 +646,65 @@ export function Chat({ const response = await fetch(`/api/messages?chatId=${id}&_t=${Date.now()}`, { cache: "no-store", }); - if (!response.ok) return { hasImage: false }; + if (!response.ok) return { hasImage: false, hasError: false }; const data = await response.json(); if (!data.messages || !Array.isArray(data.messages)) { - return { hasImage: false }; + return { hasImage: false, hasError: false }; } const dbMessages = data.messages as ChatMessage[]; console.log("[Chat] Reloading messages from database:", dbMessages.length); mergeMessagesFromDatabase(dbMessages); - const hasPersistedImage = dbMessages.some((message) => hasImageMarkdown((message as any).parts)); - return { hasImage: hasPersistedImage }; + const targetUserMessageId = + pendingSnapshot?.userMessageId || startedGenerationMessageIdRef.current; + const assistantAfterTargetUser = getAssistantMessagesAfterTargetUser( + dbMessages, + targetUserMessageId, + ); + const hasPersistedImage = assistantAfterTargetUser.some((message) => + hasImageMarkdown((message as any).parts), + ); + const hasPersistedError = assistantAfterTargetUser.some((message) => + hasLikelyGenerationErrorMessage(message), + ); + return { hasImage: hasPersistedImage, hasError: hasPersistedError }; }; // 延迟一小段时间确保数据库写入完成 await new Promise((resolve) => setTimeout(resolve, 500)); - let { hasImage } = await reloadMessagesOnce(); - if (!hasImage && imageGenerationPhaseRef.current === "finalizing") { + let { hasImage, hasError } = await reloadMessagesOnce(); + if ( + !hasImage && + !hasError && + (imageGenerationPhaseRef.current === "queued" || + imageGenerationPhaseRef.current === "processing" || + imageGenerationPhaseRef.current === "finalizing" || + keepPendingUntilDbSync) + ) { for (let attempt = 1; attempt <= 6; attempt++) { await new Promise((resolve) => setTimeout(resolve, 600)); const result = await reloadMessagesOnce(); hasImage = result.hasImage; - if (hasImage) break; + hasError = result.hasError; + if (hasImage || hasError) break; } } - if ( - hasImage || - (imageGenerationPhaseRef.current !== "idle" && - imageGenerationPhaseRef.current !== "failed") - ) { + if (hasImage) { updateImageGenerationPhase("idle"); + completeGeneration(); + } else if (hasError) { + updateImageGenerationPhase("failed"); + completeGeneration(); + } else if ( + imageGenerationPhaseRef.current !== "idle" && + imageGenerationPhaseRef.current !== "failed" + ) { + // 保持 pending，让轮询继续从数据库拉取结果，避免前端“空白直到刷新” + console.log("[Chat] Waiting for persisted result, keep pending state"); } } catch (error) { console.error("[Chat] Failed to reload messages:", error); @@ -640,7 +715,9 @@ export function Chat({ imageGenerationPhaseRef.current === "processing" || imageGenerationPhaseRef.current === "finalizing" ) { - updateImageGenerationPhase("idle"); + if (!keepPendingUntilDbSync) { + updateImageGenerationPhase("idle"); + } } } }, @@ -650,6 +727,7 @@ export function Chat({ imagePlaceholderTimerRef.current = null; } updateImageGenerationPhase("failed"); + completeGeneration(); if (error instanceof ChatSDKError) { // Check if it's a credit card error @@ -664,6 +742,94 @@ export function Chat({ }, }); + useEffect(() => { + const isImagePending = + pendingGeneration?.modelType === "gemini" || + pendingGeneration?.modelType === "volcano-vision" || + pendingGeneration?.modelType === "alibaba-image"; + if (!pendingGeneration || !isImagePending) return; + + let cancelled = false; + const maxDurationMs = 10 * 60 * 1000; + + const mergeMessagesFromDatabase = (dbMessages: ChatMessage[]) => { + setMessages((prev) => { + const mergedMap = new Map(); + prev.forEach((message) => mergedMap.set(message.id, message)); + dbMessages.forEach((message) => mergedMap.set(message.id, message)); + return sortMessagesByCreatedAt(Array.from(mergedMap.values())); + }); + + const persistedIds = new Set(dbMessages.map((message) => message.id)); + setAppendedMessages((prev) => prev.filter((message) => !persistedIds.has(message.id))); + }; + + const pollMessages = async () => { + if (cancelled) return; + + if (Date.now() - pendingGeneration.timestamp > maxDurationMs) { + console.warn("[Chat] Polling timeout for pending generation"); + completeGeneration(); + if ( + imageGenerationPhaseRef.current === "queued" || + imageGenerationPhaseRef.current === "processing" || + imageGenerationPhaseRef.current === "finalizing" + ) { + updateImageGenerationPhase("failed"); + } + return; + } + + try { + const response = await fetch(`/api/messages?chatId=${id}&_t=${Date.now()}`, { + cache: "no-store", + }); + if (!response.ok) return; + + const data = await response.json(); + if (!data.messages || !Array.isArray(data.messages)) return; + + const dbMessages = data.messages as ChatMessage[]; + mergeMessagesFromDatabase(dbMessages); + + const assistantAfterTargetUser = getAssistantMessagesAfterTargetUser( + dbMessages, + pendingGeneration.userMessageId, + ); + if (assistantAfterTargetUser.length === 0) return; + + const hasImage = assistantAfterTargetUser.some((message) => + hasImageMarkdown((message as any).parts), + ); + const hasError = assistantAfterTargetUser.some((message) => + hasLikelyGenerationErrorMessage(message) + ); + if (!hasImage && !hasError) return; + + console.log("[Chat] Polling detected finished assistant response"); + completeGeneration(); + if (hasImage) { + updateImageGenerationPhase("idle"); + } else if ( + imageGenerationPhaseRef.current === "queued" || + imageGenerationPhaseRef.current === "processing" || + imageGenerationPhaseRef.current === "finalizing" + ) { + updateImageGenerationPhase("failed"); + } + } catch (error) { + console.error("[Chat] Polling messages failed:", error); + } + }; + + pollMessages(); + const timer = setInterval(pollMessages, 800); + return () => { + cancelled = true; + clearInterval(timer); + }; + }, [pendingGeneration, id, setMessages, completeGeneration, updateImageGenerationPhase]); + // 🆕 保存 initialMessages 的引用，防止历史消息丢失 // 当 useChat 内部状态重置时，我们仍然有初始消息 const initialMessagesRef = useRef(initialMessages); @@ -683,9 +849,31 @@ export function Chat({ useEffect(() => { console.log(`[Chat] Status changed: ${status}`); + + if (status !== "submitted") { + startedGenerationMessageIdRef.current = null; + } // 🆕 当状态变为 submitted 时，记录生成任务 if (status === "submitted") { + const latestMessage = messages.length > 0 ? messages[messages.length - 1] : null; + const latestUserMessageId = latestMessage?.role === "user" ? latestMessage.id : null; + if (latestUserMessageId && startedGenerationMessageIdRef.current === latestUserMessageId) { + return; + } + + const latestUserHasImageInput = + latestMessage?.role === "user" && + Array.isArray(latestMessage.parts) && + latestMessage.parts.some((part: any) => { + if (!part) return false; + if (part.type === "file" || part.type === "image" || part.type === "image_url") return true; + if (part.type === "text" && typeof part.text === "string") { + return /\!\[thumb\||\!\[Generated Image/i.test(part.text); + } + return false; + }); + // 判断模型类型 - 使用 ref 获取最新的 modelsData const models = modelsDataRef.current?.models; const model = models?.find((m: any) => m.id === currentModelId); @@ -694,7 +882,7 @@ export function Chat({ // 🆕 gemini-image 和 huanke-gemini 都视为 gemini 类型 if (model?.providerType === "gemini" || model?.providerType === "gemini-image" || model?.providerType === "huanke-gemini") modelType = "gemini"; else if (model?.providerType === "volcano-vision") modelType = "volcano-vision"; - else if (model?.providerType === "volcano-video") modelType = "volcano-video"; + else if (model?.providerType === "volcano-video" || model?.providerType === "doubao-video") modelType = "volcano-video"; else if (model?.providerType === "qianwen-video") modelType = "qianwen-video"; else if (model?.providerType === "fal-video") modelType = "fal-video"; else if ( @@ -702,15 +890,23 @@ export function Chat({ model?.providerType === "ruxa-image" || model?.providerType === "fal-image" ) modelType = "alibaba-image"; - + + // 智能自动模式下，如果当前轮输入包含图片，默认按图片生成链路跟踪 + if (currentModelId === "auto" && latestUserHasImageInput) { + modelType = "gemini"; + } + console.log(`[Chat] 🚀 记录生成任务: modelType=${modelType}, modelId=${currentModelId}`); - startGeneration(currentModelId, modelType); + startGeneration(currentModelId, modelType, latestUserMessageId || undefined); + if (latestUserMessageId) { + startedGenerationMessageIdRef.current = latestUserMessageId; + } if (modelType === "gemini" || modelType === "volcano-vision" || modelType === "alibaba-image") { updateImageGenerationPhase("queued"); } } - }, [status, currentModelId, startGeneration, updateImageGenerationPhase]); + }, [status, currentModelId, messages, startGeneration, updateImageGenerationPhase]); const searchParams = useSearchParams(); const query = searchParams.get("query"); @@ -1510,6 +1706,87 @@ export function Chat({ } }; + // 轮询视频任务状态：解决刷新后“处理中”无法自动恢复结果的问题 + useEffect(() => { + if (isReadonly) return; + + let cancelled = false; + let timer: ReturnType | null = null; + let previousPendingCount = -1; + + const mergeMessagesFromDatabase = (dbMessages: ChatMessage[]) => { + setMessages((prev) => { + const mergedMap = new Map(); + prev.forEach((message) => mergedMap.set(message.id, message)); + dbMessages.forEach((message) => mergedMap.set(message.id, message)); + return sortMessagesByCreatedAt(Array.from(mergedMap.values())); + }); + + const persistedIds = new Set(dbMessages.map((message) => message.id)); + setAppendedMessages((prev) => prev.filter((message) => !persistedIds.has(message.id))); + }; + + const resolveModelTypeByProvider = ( + provider?: string + ): PendingGeneration["modelType"] => { + if (provider === "qianwen-video") return "qianwen-video"; + if (provider === "fal-video") return "fal-video"; + if (provider === "volcano-video" || provider === "doubao-video") return "volcano-video"; + return "other"; + }; + + const pollPendingVideoTasks = async () => { + try { + const response = await fetch(`/api/video-tasks?chatId=${id}&_t=${Date.now()}`, { + cache: "no-store", + }); + if (!response.ok) return; + + const data = await response.json(); + const tasks = Array.isArray(data?.tasks) ? data.tasks : []; + const pendingCount = tasks.length; + + if (pendingCount > 0 && !pendingGenerationRef.current) { + const firstTask = tasks[0] as { provider?: string } | undefined; + startGeneration( + currentModelIdRef.current, + resolveModelTypeByProvider(firstTask?.provider), + ); + } + + if (previousPendingCount > 0 && pendingCount === 0) { + const msgResponse = await fetch(`/api/messages?chatId=${id}&_t=${Date.now()}`, { + cache: "no-store", + }); + if (msgResponse.ok) { + const msgData = await msgResponse.json(); + if (Array.isArray(msgData?.messages)) { + mergeMessagesFromDatabase(msgData.messages as ChatMessage[]); + } + } + completeGeneration(); + } + + previousPendingCount = pendingCount; + } catch (error) { + console.warn("[Chat] video task polling failed", error); + } finally { + if (!cancelled) { + timer = setTimeout(pollPendingVideoTasks, 5000); + } + } + }; + + pollPendingVideoTasks(); + + return () => { + cancelled = true; + if (timer) { + clearTimeout(timer); + } + }; + }, [id, isReadonly, setMessages, setAppendedMessages, startGeneration, completeGeneration]); + useAutoResume({ autoResume, initialMessages, diff --git a/components/messages.tsx b/components/messages.tsx index dd0843d45..7398457f0 100644 --- a/components/messages.tsx +++ b/components/messages.tsx @@ -139,6 +139,7 @@ function PureMessages({ (m: DynamicModel) => m.id === selectedModelId && (m.providerType === "volcano-video" || + m.providerType === "doubao-video" || m.modelId?.toLowerCase().includes("video") || m.modelId?.toLowerCase().includes("sora") || m.displayName?.includes("视频")) @@ -327,6 +328,13 @@ function PureMessages({ }) ); + const waitingForFirstAgentReply = !hasAssistantTextReply && !hasImageContent; + const shouldKeepThinking = + status === "streaming" || + status === "submitted" || + (status === "ready" && waitingForFirstAgentReply && !!pendingGeneration) || + (lastMessage?.role === "user" && waitingForFirstAgentReply); + if (hasImageContent) { // 有图片内容，隐藏占位图 console.log("[Messages useEffect] 有图片内容，隐藏占位图"); @@ -339,9 +347,9 @@ function PureMessages({ // 收到了 AI 文字回复但还没开始生图（延迟期间），隐藏占位图让用户看文字 console.log("[Messages useEffect] 已收到AI文字回复，延迟期间隐藏占位图"); setShowThinkingWithDelay(false); - } else if (status === "streaming" || status === "submitted") { - // 正在处理中（分析意图或流式传输），显示普通思考占位图 - console.log("[Messages useEffect] streaming/submitted，显示思考占位图"); + } else if (shouldKeepThinking) { + // 正在处理中（分析意图/流式传输）或 SSE 首包延迟，保持思考占位图 + console.log("[Messages useEffect] keep thinking placeholder"); setShowThinkingWithDelay(true); } else { // ready 或其他状态：不显示占位图 @@ -377,7 +385,7 @@ function PureMessages({ setShowThinkingWithDelay(false); } } - }, [status, messages, isGeminiModel, isQianwenVideoModel, isVolcanoVisionModel, isVideoModel, isAlibabaImageModel, selectedModelId, isGeneratingImageFlag]); + }, [status, messages, isGeminiModel, isQianwenVideoModel, isVolcanoVisionModel, isVideoModel, isAlibabaImageModel, selectedModelId, isGeneratingImageFlag, pendingGeneration]); // 🆕 智能自动模式：确保文字可见后再显示生图占位图 useEffect(() => { diff --git a/components/multimodal-input.tsx b/components/multimodal-input.tsx index b900665d5..355e899e5 100644 --- a/components/multimodal-input.tsx +++ b/components/multimodal-input.tsx @@ -1397,7 +1397,7 @@ function PureModelSelectorCompact({ {(() => { const providerType = model.providerType || ""; - if (providerType === "volcano-video" || providerType === "qianwen-video" || providerType === "fal-video") { + if (providerType === "volcano-video" || providerType === "doubao-video" || providerType === "qianwen-video" || providerType === "fal-video") { return