diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index c56eeb09d..8d18e15c7 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -196,6 +196,32 @@ type AgentRecommendedImageParams = { promptExtend?: boolean; }; +type ChatVideoParams = NonNullable; +type ChatVideoParamsDraft = Partial; +type AgentRecommendedVideoParams = { + mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; + resolution?: "480p" | "720p" | "1080p"; + ratio?: string; + duration?: number; + generateAudio?: boolean; + cameraMove?: string; +}; + +type AgentVideoPlan = { + script: string; + storyboard: Array<{ + shot: number; + title: string; + duration: number; + description: string; + prompt: string; + camera?: string; + }>; + finalPrompt: string; + negativePrompt?: string; + recommendedVideoParams?: AgentRecommendedVideoParams; +}; + function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean { if (!params) return false; const customizedAspectRatio = @@ -250,6 +276,69 @@ function mergeImageParamsWithAgentRecommendation(params: { return Object.keys(merged).length > 0 ? merged : undefined; } +function hasUserCustomizedVideoParams(params?: PostRequestBody["videoParams"]): boolean { + if (!params) return false; + const defaults = { + mode: "text-to-video", + resolution: "720p", + ratio: "16:9", + duration: 5, + generateAudio: true, + cameraMove: "none", + } as const; + + if (params.mode && params.mode !== defaults.mode) return true; + if (params.resolution && params.resolution !== defaults.resolution) return true; + if (params.ratio && params.ratio !== defaults.ratio) return true; + if (typeof params.duration === "number" && params.duration !== defaults.duration) return true; + if (typeof params.generateAudio === "boolean" && params.generateAudio !== defaults.generateAudio) + return true; + if (typeof params.cameraMove === "string" && params.cameraMove !== defaults.cameraMove) + return true; + if (typeof params.seed === "number") return true; + return false; +} + +function mergeVideoParamsWithAgentRecommendation(params: { + incoming?: PostRequestBody["videoParams"]; + recommended?: AgentRecommendedVideoParams; + hasUserCustomSettings: boolean; +}): PostRequestBody["videoParams"] { + const { incoming, recommended, hasUserCustomSettings } = params; + const base: ChatVideoParamsDraft = { ...(incoming || {}) }; + if (!recommended) { + return Object.keys(base).length > 0 ? (base as ChatVideoParams) : undefined; + } + + const merged: ChatVideoParamsDraft = { ...base }; + if (!hasUserCustomSettings) { + if (recommended.mode) merged.mode = recommended.mode; + if (recommended.resolution) merged.resolution = recommended.resolution; + if (recommended.ratio) merged.ratio = recommended.ratio; + if (typeof recommended.duration === "number") merged.duration = recommended.duration; + if (typeof recommended.generateAudio === "boolean") { + merged.generateAudio = recommended.generateAudio; + } + if (recommended.cameraMove) merged.cameraMove = recommended.cameraMove; + } + + if (!merged.mode) { + merged.mode = recommended.mode || (incoming?.mode as ChatVideoParams["mode"]) || "text-to-video"; + } + if (!merged.resolution) { + merged.resolution = + recommended.resolution || (incoming?.resolution as ChatVideoParams["resolution"]) || "720p"; + } + if (!merged.ratio) { + merged.ratio = recommended.ratio || incoming?.ratio || "16:9"; + } + if (typeof merged.duration !== "number") { + merged.duration = recommended.duration || incoming?.duration || 5; + } + + return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined; +} + function sanitizeAgentOutputSettingsSection(params: { message?: string; userImageParams?: PostRequestBody["imageParams"]; @@ -942,6 +1031,8 @@ export async function POST(request: Request) { let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复 let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词(不修改用户消息,只用于API调用) let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined; + let smartAutoVideoPlan: AgentVideoPlan | undefined; + let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined; let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined; let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL(支持多张) let smartAutoNeedsClarification = false; // 是否需要先追问用户 @@ -985,9 +1076,17 @@ export async function POST(request: Request) { "fal-image", "volcano-vision", ]); - const isFixedImageModel = + const fixedVideoProviderTypes = new Set([ + "volcano-video", + "qianwen-video", + "fal-video", + ]); + const isFixedVisualModel = !selectedModelConfig.isSubModel && (selectedModelConfig.supportsImageGen || + (selectedModelProviderType + ? fixedVideoProviderTypes.has(selectedModelProviderType) + : false) || (selectedModelProviderType ? fixedImageProviderTypes.has(selectedModelProviderType) : false)); @@ -998,11 +1097,11 @@ export async function POST(request: Request) { console.log( `[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}` ); - } else if (isFixedImageModel) { + } else if (isFixedVisualModel) { shouldRunAgentConversation = true; shouldKeepSelectedModel = true; console.log( - `[Smart Auto Mode] 🎯 Agent conversation enabled for fixed image model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"})` + `[Smart Auto Mode] 🎯 Agent conversation enabled for fixed visual model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"})` ); } } @@ -1253,6 +1352,19 @@ export async function POST(request: Request) { optimized: autoGraphResult.optimizedPrompt.substring(0, 100), }); } + if (autoGraphResult.videoPlan) { + smartAutoVideoPlan = autoGraphResult.videoPlan; + smartAutoRecommendedVideoParams = autoGraphResult.videoPlan.recommendedVideoParams; + if (!smartAutoOptimizedPrompt && autoGraphResult.videoPlan.finalPrompt?.trim()) { + smartAutoOptimizedPrompt = autoGraphResult.videoPlan.finalPrompt.trim(); + } + console.log("[Smart Auto Mode] 🎬 Agent video plan ready:", { + hasScript: Boolean(autoGraphResult.videoPlan.script), + storyboardCount: autoGraphResult.videoPlan.storyboard?.length || 0, + hasFinalPrompt: Boolean(autoGraphResult.videoPlan.finalPrompt), + recommendedVideoParams: smartAutoRecommendedVideoParams, + }); + } if (autoGraphResult.recommendedImageParams) { smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams; console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams); @@ -1348,10 +1460,15 @@ export async function POST(request: Request) { } const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams); + const hasUserCustomVideoParams = hasUserCustomizedVideoParams(requestBody.videoParams); const shouldApplyAgentImageParams = shouldRunAgentConversation && !smartAutoNeedsClarification && (smartAutoDetectedIntent === "image_gen" || smartAutoDetectedIntent === "image_edit"); + const shouldApplyAgentVideoParams = + shouldRunAgentConversation && + !smartAutoNeedsClarification && + smartAutoDetectedIntent === "video_gen"; const effectiveImageParams = shouldApplyAgentImageParams ? mergeImageParamsWithAgentRecommendation({ incoming: requestBody.imageParams, @@ -1360,6 +1477,13 @@ export async function POST(request: Request) { hasUserCustomSettings: hasUserCustomImageParams, }) : requestBody.imageParams; + const effectiveVideoParams = shouldApplyAgentVideoParams + ? mergeVideoParamsWithAgentRecommendation({ + incoming: requestBody.videoParams, + recommended: smartAutoRecommendedVideoParams, + hasUserCustomSettings: hasUserCustomVideoParams, + }) + : requestBody.videoParams; const defaultImageQualityParams = { aspectRatio: "auto", imageSize: smartAutoDetectedIntent === "image_edit" ? "4K" : "2K", @@ -1374,6 +1498,15 @@ export async function POST(request: Request) { hasUserCustomImageParams, }); } + if (shouldApplyAgentVideoParams) { + console.log("[Smart Auto Mode] 🎯 Effective video params:", { + incoming: requestBody.videoParams, + recommended: smartAutoRecommendedVideoParams, + merged: effectiveVideoParams, + hasUserCustomVideoParams, + hasVideoPlan: Boolean(smartAutoVideoPlan), + }); + } if (shouldRunAgentConversation && smartAutoFriendlyMessage) { smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({ @@ -1591,7 +1724,7 @@ export async function POST(request: Request) { if (subModels.length > 0) { // 这是一个主模型,需要根据 videoParams 选择子模型 - const videoParams = requestBody.videoParams; + const videoParams = effectiveVideoParams; console.log("[Chat API] Video params:", videoParams); // 如果没有 videoParams 或没有 mode,使用默认模式 @@ -2799,14 +2932,54 @@ export async function POST(request: Request) { const qianwenVideoStream = createUIMessageStream({ execute: async ({ writer }) => { try { - // 千问视频直接使用原始 URL,不转换为 base64 - const modelMessages = convertToModelMessages(uiMessagesForModel); + const conversationMessage = smartAutoFriendlyMessage || null; + if (conversationMessage) { + const chatMessageId = generateUUID(); + writer.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: chatMessageId, + role: "assistant", + parts: [{ type: "text", text: conversationMessage }], + }), + }); + + await saveMessages({ + messages: [ + { + id: chatMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: conversationMessage }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + } + + const messagesForVideo = injectHistoryImagesToLastUserMessage( + applyOptimizedPromptToLatestUserMessage( + uiMessagesForModel, + smartAutoOptimizedPrompt, + hiddenPromptPrefix, + ), + smartAutoHistoryImages, + ); + + // 千问/FAL 视频直接使用原始 URL,不转换为 base64 + const modelMessages = convertToModelMessages(messagesForVideo); console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length); // 调用 doGenerate 获取视频 const result = await (languageModel as any).doGenerate({ messages: modelMessages, - videoParams: requestBody.videoParams, + videoParams: effectiveVideoParams, }); const videoMarkdown = result.text || "视频生成失败"; @@ -2847,7 +3020,16 @@ export async function POST(request: Request) { temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, - videoMetadata: null, + videoMetadata: effectiveVideoParams + ? { + provider: currentVideoProvider, + mode: effectiveVideoParams.mode, + resolution: effectiveVideoParams.resolution, + ratio: effectiveVideoParams.ratio, + duration: effectiveVideoParams.duration, + createdAt: Date.now(), + } + : null, }, ], }); @@ -3061,9 +3243,50 @@ export async function POST(request: Request) { // 导入视频生成函数 const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video"); + + const conversationMessage = smartAutoFriendlyMessage || null; + if (conversationMessage) { + const chatMessageId = generateUUID(); + dataStream.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: chatMessageId, + role: "assistant", + parts: [{ type: "text", text: conversationMessage }], + attachments: [], + }), + }); + + await saveMessages({ + messages: [ + { + id: chatMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: conversationMessage }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + } + + const messagesForVideo = injectHistoryImagesToLastUserMessage( + applyOptimizedPromptToLatestUserMessage( + uiMessagesForModel, + smartAutoOptimizedPrompt, + hiddenPromptPrefix, + ), + smartAutoHistoryImages, + ); // 从最后一条用户消息中提取提示词和图片 - const lastUserMessage = uiMessages.filter(m => m.role === "user").pop(); + const lastUserMessage = messagesForVideo.filter(m => m.role === "user").pop(); let prompt = ""; const imageUrls: string[] = []; @@ -3120,7 +3343,7 @@ export async function POST(request: Request) { console.log("[Chat API] Video generation params:", { prompt, imageCount: imageUrls.length, - videoParams: requestBody.videoParams, + videoParams: effectiveVideoParams, }); // 🆕 重新执行子模型选择逻辑,确定要使用的模型 UUID @@ -3134,7 +3357,7 @@ export async function POST(request: Request) { console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`); if (videoSubModels.length > 0) { - const videoMode = requestBody.videoParams?.mode || "text-to-video"; + const videoMode = effectiveVideoParams?.mode || "text-to-video"; const modeToModelMap: Record = { "text-to-video": "lite-t2v", "first-last-frame": "pro", @@ -3161,15 +3384,15 @@ export async function POST(request: Request) { const videoOptions: any = { model: actualModelId, prompt: prompt || "生成视频", - mode: requestBody.videoParams?.mode || "text-to-video", - resolution: requestBody.videoParams?.resolution || "720p", - ratio: requestBody.videoParams?.ratio || "16:9", - duration: requestBody.videoParams?.duration || 5, - fps: requestBody.videoParams?.fps || 24, - cameraMove: requestBody.videoParams?.cameraMove || "none", - cameraFixed: requestBody.videoParams?.cameraFixed || false, - watermark: requestBody.videoParams?.watermark || false, - seed: requestBody.videoParams?.seed, + mode: effectiveVideoParams?.mode || "text-to-video", + resolution: effectiveVideoParams?.resolution || "720p", + ratio: effectiveVideoParams?.ratio || "16:9", + duration: effectiveVideoParams?.duration || 5, + fps: effectiveVideoParams?.fps || 24, + cameraMove: effectiveVideoParams?.cameraMove || "none", + cameraFixed: effectiveVideoParams?.cameraFixed || false, + watermark: effectiveVideoParams?.watermark || false, + seed: effectiveVideoParams?.seed, }; console.log("[Chat API] Video options constructed:", videoOptions); @@ -3252,10 +3475,10 @@ export async function POST(request: Request) { // 构建视频元数据 const videoMetadata = temporaryVideoUrl ? { provider: "volcano-video", - mode: requestBody.videoParams?.mode, - resolution: requestBody.videoParams?.resolution, - ratio: requestBody.videoParams?.ratio, - duration: requestBody.videoParams?.duration, + mode: effectiveVideoParams?.mode, + resolution: effectiveVideoParams?.resolution, + ratio: effectiveVideoParams?.ratio, + duration: effectiveVideoParams?.duration, createdAt: Date.now(), } : null; diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 9627c230f..87fdaf06f 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -24,6 +24,7 @@ import { extractImageUrlsFromText } from "@/lib/image-url"; import type { Attachment, ChatMessage } from "@/lib/types"; import { type CanvasLaunchPayload, getCanvasLaunchStorageKey } from "@/lib/canvas-launch"; import { ImageParamsBar, type ImageGenerationParams } from "@/components/image-params-bar"; +import { VideoParamsBar, type VideoGenerationParams } from "@/components/video-params-bar"; import { ImageGenerationLoading } from "@/components/image-generation-loading"; // 标签悬停预览信息 @@ -105,6 +106,98 @@ function extractImageUrls(text: string): string[] { return extractImageUrlsFromText(text); } +function extractVideoUrls(text: string): string[] { + if (!text) return []; + const urls = new Set(); + + const videoTagPattern = /]+src="([^"]+)"/gi; + let match: RegExpExecArray | null = null; + while ((match = videoTagPattern.exec(text)) !== null) { + if (match[1]) urls.add(match[1]); + } + + const markdownVideoPattern = /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/gi; + while ((match = markdownVideoPattern.exec(text)) !== null) { + if (match[1]) urls.add(match[1]); + } + + const plainVideoPattern = /(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/gi; + while ((match = plainVideoPattern.exec(text)) !== null) { + if (match[1]) urls.add(match[1]); + } + + return Array.from(urls); +} + +function normalizeVideoParamsForProvider( + params: VideoGenerationParams | null | undefined, + providerType?: string +): VideoGenerationParams { + const isFalVideo = providerType === "fal-video"; + const base: VideoGenerationParams = { + mode: isFalVideo ? "first-frame" : "text-to-video", + resolution: "720p", + ratio: "16:9", + duration: 5, + generateAudio: true, + watermark: false, + cameraFixed: false, + cameraMove: "none", + fps: 24, + }; + const merged = { ...base, ...(params || {}) }; + + if (isFalVideo) { + if (merged.mode !== "first-frame" && merged.mode !== "first-last-frame") { + merged.mode = "first-frame"; + } + if (!["16:9", "9:16", "1:1"].includes(merged.ratio)) { + merged.ratio = "16:9"; + } + merged.duration = Math.max(3, Math.min(15, Math.round(Number(merged.duration) || 5))); + merged.generateAudio = merged.generateAudio ?? true; + return merged; + } + + if (!["text-to-video", "first-frame", "first-last-frame", "reference-images"].includes(merged.mode)) { + merged.mode = "text-to-video"; + } + if (!["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"].includes(merged.ratio)) { + merged.ratio = "16:9"; + } + merged.duration = merged.duration >= 8 ? 10 : 5; + return merged; +} + +function getVideoImageLimit( + params: VideoGenerationParams | null | undefined, + providerType?: string +): { min: number; max: number } { + if (!params) { + return { min: 0, max: 0 }; + } + const isFalVideo = providerType === "fal-video"; + if (isFalVideo) { + if (params.mode === "first-last-frame") { + return { min: 2, max: 2 }; + } + return { min: 1, max: 1 }; + } + + switch (params.mode) { + case "text-to-video": + return { min: 0, max: 0 }; + case "first-last-frame": + return { min: 2, max: 2 }; + case "first-frame": + return { min: 1, max: 1 }; + case "reference-images": + return { min: 1, max: 4 }; + default: + return { min: 0, max: 0 }; + } +} + function extractUserTextPrompt(message: ChatMessage): string { const parts = (message as any)?.parts || []; const textFromParts = parts @@ -227,6 +320,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF imageSize: "2K", imageCount: 1, }); + const [videoParams, setVideoParams] = useState(null); // 🆕 保存图片参数到 localStorage const handleImageParamsChange = (params: ImageGenerationParams) => { @@ -251,6 +345,26 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF } }, []); + const handleVideoParamsChange = (params: VideoGenerationParams) => { + setVideoParams(params); + try { + localStorage.setItem("canvas-video-params", JSON.stringify(params)); + } catch (e) { + console.error("Failed to save video params:", e); + } + }; + + useEffect(() => { + try { + const saved = localStorage.getItem("canvas-video-params"); + if (saved) { + setVideoParams(JSON.parse(saved)); + } + } catch (e) { + console.error("Failed to load video params:", e); + } + }, []); + useEffect(() => { if (!fullscreenPreview) return; const handleEsc = (event: KeyboardEvent) => { @@ -711,6 +825,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF providerType: string; supportsVision?: boolean; supportsImageGen?: boolean; + capabilities?: string[] | null; }>; }>("/api/models", fetcher); @@ -723,8 +838,11 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF canvasDefaultModelId?: string | null; }>("/api/site-settings", fetcher); - // 过滤支持图片生成的模型 - const imageModels = modelsData?.models?.filter(m => m.supportsImageGen) || []; + const VIDEO_PROVIDER_TYPES = new Set(["volcano-video", "qianwen-video", "fal-video"]); + const visualModels = + modelsData?.models?.filter( + (model) => model.supportsImageGen || VIDEO_PROVIDER_TYPES.has(model.providerType) + ) || []; // Agent模式(自动路由)模型选项 const SMART_AUTO_MODEL = { @@ -736,8 +854,10 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF supportsImageGen: true, }; - // 所有可选模型(Agent模式 + 生图模型) - const allModels = [SMART_AUTO_MODEL, ...imageModels]; + // 所有可选模型(Agent模式 + 可视化模型) + const allModels = [SMART_AUTO_MODEL, ...visualModels]; + const currentModel = allModels.find((model) => model.id === selectedModelId); + const isVideoModel = Boolean(currentModel && VIDEO_PROVIDER_TYPES.has(currentModel.providerType)); // 默认选择模型(根据系统设置) useEffect(() => { @@ -750,6 +870,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF const currentModelIdRef = useRef(selectedModelId); const imageParamsRef = useRef(imageParams); + const videoParamsRef = useRef(videoParams); + const isVideoModelRef = useRef(isVideoModel); useEffect(() => { currentModelIdRef.current = selectedModelId; @@ -759,6 +881,22 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF imageParamsRef.current = imageParams; }, [imageParams]); + useEffect(() => { + videoParamsRef.current = videoParams; + }, [videoParams]); + + useEffect(() => { + isVideoModelRef.current = isVideoModel; + }, [isVideoModel]); + + useEffect(() => { + if (!isVideoModel) return; + const normalized = normalizeVideoParamsForProvider(videoParams, currentModel?.providerType); + if (!videoParams || JSON.stringify(videoParams) !== JSON.stringify(normalized)) { + setVideoParams(normalized); + } + }, [isVideoModel, currentModel?.providerType, videoParams]); + // 合并 initialMessages 和历史消息 const mergedInitialMessages = historyData?.messages ?? (activeChatId === chatId ? initialMessages : []); @@ -798,8 +936,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF message: request.messages.at(-1), selectedChatModel: currentModelIdRef.current, selectedVisibilityType: "private", - videoParams: undefined, // 必须传递,即使是 undefined - imageParams: imageParamsRef.current || undefined, // 传递图片参数 + videoParams: isVideoModelRef.current ? videoParamsRef.current || undefined : undefined, + imageParams: isVideoModelRef.current ? undefined : imageParamsRef.current || undefined, hiddenPrompt, autoContext, ...request.body, @@ -974,11 +1112,16 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF if (message.role === 'assistant' && !processedMessageIds.current.has(message.id)) { const content = message.parts?.map((p: any) => p.text || '').join('') || ''; const imageUrls = extractImageUrls(content); + const videoUrls = extractVideoUrls(content); const generationPrompt = findNearestUserPrompt(messages, messageIndex); - - if (imageUrls.length > 0) { + + if (imageUrls.length > 0 || videoUrls.length > 0) { processedMessageIds.current.add(message.id); - + + let addedImageCount = 0; + let addedVideoCount = 0; + + if (imageUrls.length > 0) { // 检查画布中是否已存在相同 URL 的图片 const existingUrls = new Set( nodes.filter(n => n.type === 'image' && (n.data as any)?.src) @@ -1017,10 +1160,45 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF }, connections: [], }); + addedImageCount += 1; } - - if (newUrls.length > 0) { - toast.success(`已将 ${newUrls.length} 张图片添加到画布`); + } + + if (videoUrls.length > 0) { + const existingVideoUrls = new Set( + nodes.filter(n => n.type === 'video' && (n.data as any)?.src) + .map(n => (n.data as any).src) + ); + const newVideoUrls = videoUrls.filter(url => !existingVideoUrls.has(url)); + + for (let index = 0; index < newVideoUrls.length; index++) { + const url = newVideoUrls[index]; + const existingNodes = nodes.length + index; + const x = 120 + (existingNodes % 2) * 420; + const y = 120 + Math.floor(existingNodes / 2) * 260; + addNode({ + type: "video", + position: { x, y }, + size: { width: 380, height: 220 }, + data: { + src: url, + status: "completed", + prompt: generationPrompt || "对话生成", + }, + connections: [], + }); + addedVideoCount += 1; + } + } + + if (addedImageCount > 0 || addedVideoCount > 0) { + const notice = [ + addedImageCount > 0 ? `${addedImageCount} 张图片` : "", + addedVideoCount > 0 ? `${addedVideoCount} 条视频` : "", + ] + .filter(Boolean) + .join("、"); + toast.success(`已将 ${notice} 添加到画布`); } } } @@ -1217,7 +1395,15 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF // 🆕 如果没有附件,自动使用最近的图片(支持多张) let effectiveAttachments = [...attachments]; - if (effectiveAttachments.length === 0 && input.trim()) { + const videoImageLimit = isVideoModel + ? getVideoImageLimit(videoParams, currentModel?.providerType) + : null; + + if ( + effectiveAttachments.length === 0 && + input.trim() && + (!videoImageLimit || videoImageLimit.max > 0) + ) { const recentAttachments = getRecentImageAttachments(); if (recentAttachments.length > 0) { // 自动添加所有最近的图片作为附件 @@ -1231,6 +1417,23 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF console.log('[CanvasChat] 自动使用最近图片:', recentAttachments.length, '张'); } } + + if (videoImageLimit) { + if (videoImageLimit.max === 0) { + effectiveAttachments = []; + } else if (effectiveAttachments.length > videoImageLimit.max) { + effectiveAttachments = effectiveAttachments.slice(0, videoImageLimit.max); + } + + if (effectiveAttachments.length < videoImageLimit.min) { + toast.error( + videoImageLimit.min === 1 + ? "当前视频模式需要至少 1 张参考图" + : "当前视频模式需要 2 张首尾帧图片" + ); + return; + } + } // 🆕 检查是否有标注的图片 const annotatedNodes = selectedImageNodes.filter(node => (node.data as any)?.hasAnnotations); @@ -1327,7 +1530,19 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF editableDiv.innerHTML = ''; console.log('[CanvasChat] 已清空输入框和选中状态'); } - }, [input, attachments, selectedModelId, sendMessage, getRecentImageAttachments, clearSelection, buildAutoContext]); + }, [ + input, + attachments, + selectedModelId, + sendMessage, + getRecentImageAttachments, + clearSelection, + buildAutoContext, + isVideoModel, + videoParams, + currentModel?.providerType, + selectedImageNodes, + ]); // 处理文件选择 - 上传后立即添加到画布 const handleFileSelect = async (e: React.ChangeEvent) => { @@ -1552,7 +1767,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF (part: any) => part.type === "text" && part.text?.includes("") ) ); - const currentModel = allModels.find(m => m.id === selectedModelId); const shouldShowImageLoading = hasGenerationStart && !hasGenerationEnd && !!currentModel; const loadingImageCount = imageParams.imageCount || 1; @@ -2118,13 +2332,23 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF {/* 中间:图片参数(可收缩) */}
- m.id === selectedModelId)?.displayName} - initialParams={imageParams} - onParamsChange={handleImageParamsChange} - inline={true} - providerType={allModels.find(m => m.id === selectedModelId)?.providerType} - /> + {isVideoModel ? ( + + ) : ( + + )}
diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index 40b1c0c3f..f5db25a86 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -16,6 +16,32 @@ export interface AgentRecommendedImageParams { promptExtend?: boolean; } +export interface AgentRecommendedVideoParams { + mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; + resolution?: "480p" | "720p" | "1080p"; + ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive"; + duration?: number; + generateAudio?: boolean; + cameraMove?: string; +} + +export interface AgentVideoStoryboardShot { + shot: number; + title: string; + duration: number; + description: string; + prompt: string; + camera?: string; +} + +export interface AgentVideoPlan { + script: string; + storyboard: AgentVideoStoryboardShot[]; + finalPrompt: string; + negativePrompt?: string; + recommendedVideoParams?: AgentRecommendedVideoParams; +} + export interface AutoAgentInput { text: string; hasImage: boolean; @@ -30,6 +56,7 @@ export interface AutoAgentResult { friendlyMessage?: string; optimizedPrompt?: string; recommendedImageParams?: AgentRecommendedImageParams; + videoPlan?: AgentVideoPlan; route?: RouteResult; } @@ -44,6 +71,7 @@ const AutoAgentState = Annotation.Root({ friendlyMessage: Annotation, optimizedPrompt: Annotation, recommendedImageParams: Annotation, + videoPlan: Annotation, route: Annotation, }); @@ -84,6 +112,216 @@ function normalizeImageSize(value?: string): string | undefined { return undefined; } +function normalizeVideoMode(value?: string): AgentRecommendedVideoParams["mode"] | undefined { + if (!value) return undefined; + if (value === "text-to-video") return "text-to-video"; + if (value === "first-frame") return "first-frame"; + if (value === "first-last-frame") return "first-last-frame"; + if (value === "reference-images") return "reference-images"; + return undefined; +} + +function normalizeVideoResolution( + value?: string, +): AgentRecommendedVideoParams["resolution"] | undefined { + if (!value) return undefined; + const normalized = value.toLowerCase(); + if (normalized === "480p") return "480p"; + if (normalized === "720p") return "720p"; + if (normalized === "1080p") return "1080p"; + return undefined; +} + +function normalizeVideoRatio(value?: string): AgentRecommendedVideoParams["ratio"] | undefined { + if (!value) return undefined; + const normalized = value.trim(); + const allowed = new Set(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"]); + return allowed.has(normalized) ? (normalized as AgentRecommendedVideoParams["ratio"]) : undefined; +} + +function normalizeVideoDuration(value?: number): number | undefined { + if (typeof value !== "number" || !Number.isFinite(value)) return undefined; + return Math.min(15, Math.max(3, Math.round(value))); +} + +function sanitizeRecommendedVideoParams( + params: AgentRecommendedVideoParams | undefined, +): AgentRecommendedVideoParams | undefined { + if (!params) return undefined; + const safe: AgentRecommendedVideoParams = {}; + const mode = normalizeVideoMode(params.mode); + const resolution = normalizeVideoResolution(params.resolution); + const ratio = normalizeVideoRatio(params.ratio); + const duration = normalizeVideoDuration(params.duration); + + if (mode) safe.mode = mode; + if (resolution) safe.resolution = resolution; + if (ratio) safe.ratio = ratio; + if (duration) safe.duration = duration; + if (typeof params.generateAudio === "boolean") safe.generateAudio = params.generateAudio; + if (typeof params.cameraMove === "string" && params.cameraMove.trim()) { + safe.cameraMove = params.cameraMove.trim().slice(0, 32); + } + + return Object.keys(safe).length > 0 ? safe : undefined; +} + +function inferVideoParamsByText(text: string, hasImage: boolean): AgentRecommendedVideoParams { + const normalized = text.toLowerCase(); + const durationMatch = normalized.match(/(\d{1,2})\s*秒/); + const duration = durationMatch ? normalizeVideoDuration(Number(durationMatch[1])) : undefined; + + const mode = /首尾帧|首帧.*尾帧|结尾帧|尾帧/.test(normalized) + ? "first-last-frame" + : hasImage + ? "first-frame" + : "text-to-video"; + + const ratio = /竖屏|手机|9:16|竖版/.test(normalized) + ? "9:16" + : /方形|方图|1:1/.test(normalized) + ? "1:1" + : /超宽|21:9|电影感/.test(normalized) + ? "21:9" + : "16:9"; + + const generateAudio = /配音|旁白|音效|有声|带声音/.test(normalized); + return { + mode, + ratio, + duration: duration || 5, + resolution: "720p", + generateAudio, + }; +} + +function buildVideoFinalPrompt(params: { + script: string; + storyboard: AgentVideoStoryboardShot[]; + originalText: string; +}): string { + const script = params.script.trim(); + const storyboard = params.storyboard + .slice(0, 8) + .map( + (shot) => + `镜头${shot.shot}(${shot.duration}s): ${shot.prompt || shot.description || shot.title}`.trim() + ) + .join("\n"); + + return [ + `【用户目标】${params.originalText.trim() || "生成视频"}`, + script ? `【脚本】${script}` : "", + storyboard ? `【分镜】\n${storyboard}` : "", + "【执行要求】镜头衔接自然,人物与主体保持一致,动作清晰,避免突兀跳切。", + ] + .filter(Boolean) + .join("\n\n") + .trim(); +} + +function heuristicVideoPlan(state: AutoAgentStateType): AgentVideoPlan { + const recommendedVideoParams = inferVideoParamsByText(state.text, state.hasImage); + const duration = recommendedVideoParams.duration || 5; + const perShot = Math.max(2, Math.round(duration / 3)); + const storyboard: AgentVideoStoryboardShot[] = [ + { + shot: 1, + title: "开场建立", + duration: perShot, + description: "建立主体与环境关系,明确视觉基调。", + prompt: `开场镜头,展示主体与环境,围绕「${compactText(state.text, 50)}」建立氛围。`, + camera: "稳定推进", + }, + { + shot: 2, + title: "动作推进", + duration: perShot, + description: "主体动作或情绪推进,形成主要叙事段落。", + prompt: `中段镜头,突出主体动作与情绪变化,强化故事推进。`, + camera: "跟拍或环绕", + }, + { + shot: 3, + title: "收束结尾", + duration: Math.max(2, duration - perShot * 2), + description: "情绪收束并输出最终视觉记忆点。", + prompt: "结尾镜头,画面收束,保留主体识别与情绪余韵。", + camera: "缓慢拉远", + }, + ]; + const script = `围绕“${compactText(state.text, 80)}”制作一条节奏清晰、主体连续的短视频,包含开场建立、过程推进与结尾收束三段。`; + return { + script, + storyboard, + finalPrompt: buildVideoFinalPrompt({ + script, + storyboard, + originalText: state.text, + }), + recommendedVideoParams, + }; +} + +function sanitizeVideoPlan(raw: unknown, state: AutoAgentStateType): AgentVideoPlan { + if (!raw || typeof raw !== "object") return heuristicVideoPlan(state); + const payload = raw as Record; + const recommendedVideoParams = sanitizeRecommendedVideoParams(payload.recommendedVideoParams); + const script = + typeof payload.script === "string" && payload.script.trim() + ? payload.script.trim().slice(0, 400) + : `围绕“${compactText(state.text, 80)}”生成视频。`; + + const storyboardInput = Array.isArray(payload.storyboard) ? payload.storyboard : []; + const storyboard: AgentVideoStoryboardShot[] = storyboardInput + .map((item: any, idx) => { + const shot = Number.isFinite(item?.shot) ? Math.max(1, Math.round(item.shot)) : idx + 1; + const duration = normalizeVideoDuration(Number(item?.duration)) || 3; + const title = + typeof item?.title === "string" && item.title.trim() + ? item.title.trim().slice(0, 24) + : `镜头${shot}`; + const description = + typeof item?.description === "string" && item.description.trim() + ? item.description.trim().slice(0, 180) + : title; + const prompt = + typeof item?.prompt === "string" && item.prompt.trim() + ? item.prompt.trim().slice(0, 260) + : description; + const camera = + typeof item?.camera === "string" && item.camera.trim() + ? item.camera.trim().slice(0, 24) + : undefined; + return { shot, title, duration, description, prompt, camera }; + }) + .filter((item) => item.prompt); + + const fallback = heuristicVideoPlan(state); + const safeStoryboard = storyboard.length > 0 ? storyboard.slice(0, 8) : fallback.storyboard; + const finalPrompt = + typeof payload.finalPrompt === "string" && payload.finalPrompt.trim() + ? payload.finalPrompt.trim().slice(0, 1400) + : buildVideoFinalPrompt({ + script, + storyboard: safeStoryboard, + originalText: state.text, + }); + + const negativePrompt = + typeof payload.negativePrompt === "string" && payload.negativePrompt.trim() + ? payload.negativePrompt.trim().slice(0, 360) + : undefined; + + return { + script, + storyboard: safeStoryboard, + finalPrompt, + negativePrompt, + recommendedVideoParams: recommendedVideoParams || fallback.recommendedVideoParams, + }; +} + function sanitizeRecommendedImageParams( params: AgentRecommendedImageParams | undefined, intent?: AnalysisResult["intent"], @@ -422,6 +660,87 @@ async function recommendImageParamsWithAgent( } } +async function recommendVideoPlanWithAgent( + state: AutoAgentStateType, +): Promise { + try { + const { + getEnabledAiModelsWithProvider, + getAgentAiModelWithProvider, + } = await import("@/lib/db/queries"); + const { getLanguageModelById } = await import("./dynamic-provider"); + const allModels = await getEnabledAiModelsWithProvider(); + const agentModel = await getAgentAiModelWithProvider(); + const plannerModelConfig = pickAgentReasoningModel(agentModel, allModels); + if (!plannerModelConfig) { + return heuristicVideoPlan(state); + } + const plannerModel = await getLanguageModelById(plannerModelConfig.model.id); + if (!plannerModel) { + return heuristicVideoPlan(state); + } + + const history = state.historyMessages + .slice(-6) + .map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`) + .join("\n"); + + const prompt = [ + `用户请求:${state.text || "生成视频"}`, + `是否有参考图:${state.hasImage ? "是" : "否"}(${state.imageUrls.length} 张)`, + history ? `最近对话:\n${history}` : "", + "请产出可直接用于视频生成的脚本与分镜。", + ] + .filter(Boolean) + .join("\n\n"); + + const { text } = await generateText({ + model: plannerModel, + system: `你是视频生成 Agent 规划器,目标是最大化一次出片成功率。 +只输出 JSON,不要输出解释。格式: +{ + "script": "120字内剧情脚本", + "recommendedVideoParams": { + "mode": "text-to-video|first-frame|first-last-frame|reference-images", + "resolution": "480p|720p|1080p", + "ratio": "16:9|4:3|1:1|3:4|9:16|21:9|adaptive", + "duration": 5, + "generateAudio": false, + "cameraMove": "none|static|handheld|dynamic-orbit" + }, + "storyboard": [ + { + "shot": 1, + "title": "镜头标题", + "duration": 3, + "description": "画面描述", + "camera": "运镜方式", + "prompt": "该镜头可直接喂给视频模型的提示词" + } + ], + "negativePrompt": "可选", + "finalPrompt": "整合后的最终视频提示词" +} +要求: +- 分镜 2~6 个,时长总计与 duration 接近。 +- 有参考图时优先 first-frame 或 first-last-frame。 +- 避免抽象空话,必须可执行。`, + prompt, + temperature: 0.2, + maxTokens: 720, + } as any); + + const parsed = parseJsonBlock>(text); + const plan = sanitizeVideoPlan(parsed, state); + console.log( + `[Auto Agent] Video planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})` + ); + return plan; + } catch { + return heuristicVideoPlan(state); + } +} + async function refineRouteWithAgent( state: AutoAgentStateType, baseRoute: RouteResult, @@ -525,6 +844,9 @@ const analyzeNode = async (state: AutoAgentStateType) => { const shouldOptimize = (state: AutoAgentStateType) => { if (!state.analysis) return "route"; if (state.analysis.needsClarification) return "route"; + if (state.analysis.intent === "video_gen") { + return "optimize"; + } if (state.analysis.intent === "image_gen") { return "optimize"; } @@ -538,6 +860,34 @@ const optimizeNode = async (state: AutoAgentStateType) => { if (!state.analysis) { return {}; } + if (state.analysis.intent === "video_gen") { + const videoPlan = await recommendVideoPlanWithAgent(state); + const storyboardPreview = videoPlan.storyboard + .slice(0, 4) + .map( + (shot) => + `- 镜头${shot.shot}(${shot.duration}s):${shot.title}|${compactText( + shot.description || shot.prompt, + 64, + )}` + ) + .join("\n"); + const friendlyMessage = [ + "### 需求理解", + `- ${compactText(state.text || "生成视频", 110)}`, + "### 脚本规划", + `- ${videoPlan.script}`, + "### 分镜规划", + storyboardPreview || "- 已生成分镜,执行时自动拼接。", + "### 执行说明", + "- 已基于上述脚本与分镜优化提示词并提交视频模型执行。", + ].join("\n"); + return { + friendlyMessage, + optimizedPrompt: videoPlan.finalPrompt, + videoPlan, + }; + } if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) { const recommendedImageParams = await recommendImageParamsWithAgent(state.analysis, state); return { recommendedImageParams }; @@ -618,6 +968,7 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise