From 2cad27cdcd1db9c38d13c1fb2f9a93bf8fe21170 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 26 Apr 2026 19:55:36 +0800 Subject: [PATCH] feat: add connected image-to-video composer --- .../canvas/LiblibVideoCanvasReplica.tsx | 655 ++++++++++++++++-- 1 file changed, 615 insertions(+), 40 deletions(-) diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 0296f7b5e..9a7e8f8b5 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -60,6 +60,7 @@ import { defaultVideoCanvasFunctionConfig, normalizeVideoCanvasFunctionConfig, } from "@/lib/video-canvas/config"; +import type { VideoGenerationMode, VideoRatio, VideoResolution } from "@/types/ai"; type ActivePanel = | "add" @@ -97,6 +98,7 @@ type ComposerMenu = | "model" | "ratio" | "resolution" + | "duration" | "camera" | "panorama" | "translate" @@ -217,6 +219,22 @@ type ComposerSettings = { resolution: string; }; +type VideoComposerTabKey = + | "text-to-video" + | "reference-images" + | "first-frame" + | "first-last-frame" + | "image-reference"; + +type VideoTargetComposerSubmitPayload = { + mode: VideoGenerationMode; + prompt: string; + settings: ComposerSettings & { + duration: string; + }; + tab: VideoComposerTabKey; +}; + type CanvasModelOption = { id: string; modelId?: string; @@ -1393,6 +1411,87 @@ function getFunctionModelValue( return String(definition.model || fallback || "auto").trim() || "auto"; } +function normalizeVideoGenerationMode(value: unknown): VideoGenerationMode | null { + const mode = String(value || "").trim(); + if ( + mode === "text-to-video" || + mode === "first-frame" || + mode === "first-last-frame" || + mode === "reference-images" + ) { + return mode; + } + return null; +} + +function normalizeVideoRatioValue(value: unknown): VideoRatio { + const ratio = String(value || "").trim(); + if ( + ratio === "16:9" || + ratio === "4:3" || + ratio === "1:1" || + ratio === "3:4" || + ratio === "9:16" || + ratio === "21:9" || + ratio === "adaptive" + ) { + return ratio; + } + return "16:9"; +} + +function normalizeVideoResolutionValue(value: unknown): VideoResolution { + const normalized = String(value || "") + .trim() + .toLowerCase() + .replace(/\s+/g, ""); + if (normalized === "480p") return "480p"; + if (normalized === "1080p" || normalized === "1k") return "1080p"; + return "720p"; +} + +function formatVideoResolutionLabel(value: unknown) { + const resolution = normalizeVideoResolutionValue(value); + return resolution === "480p" ? "480P" : resolution === "1080p" ? "1080P" : "720P"; +} + +function normalizeVideoDurationValue(value: unknown) { + const matched = String(value || "").match(/(\d+)/); + const duration = matched ? Number.parseInt(matched[1], 10) : Number(value); + if (Number.isFinite(duration) && duration > 0) return duration; + return 5; +} + +function formatVideoDurationLabel(value: unknown) { + return `${normalizeVideoDurationValue(value)}s`; +} + +function inferVideoComposerDefaultTab(imageCount: number): VideoComposerTabKey { + if (imageCount >= 2) return "reference-images"; + if (imageCount >= 1) return "reference-images"; + return "text-to-video"; +} + +function resolveVideoComposerMode(tab: VideoComposerTabKey): VideoGenerationMode { + if (tab === "first-last-frame") return "first-last-frame"; + if (tab === "first-frame") return "first-frame"; + if (tab === "reference-images" || tab === "image-reference") return "reference-images"; + return "text-to-video"; +} + +function getVideoFunctionKeyForMode(mode: VideoGenerationMode) { + if (mode === "text-to-video") return "text-to-video" as const; + if (mode === "first-last-frame") return "first-last-frame-video" as const; + return "first-frame-video" as const; +} + +function selectVideoReferenceImages(images: string[], mode: VideoGenerationMode) { + if (mode === "text-to-video") return []; + if (mode === "first-frame") return images.slice(0, 1); + if (mode === "first-last-frame") return images.slice(0, 2); + return images.slice(0, 4); +} + function getCanvasModelRef(model: CanvasModelOption) { return String(model.modelId || model.id || "").trim(); } @@ -2618,6 +2717,10 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib : { id: makeId("edge"), source: node.id, target: anchor.id }; addNodes([node], [edge]); + setSelectedNodeId(node.id); + setCanvasCommand(null); + setNodeContextMenu(null); + setActivePanel(null); setConnectionMenu(null); toast.success(`已添加并连接 ${node.title}`); }, @@ -3997,6 +4100,205 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib [nodes, videoCanvasFunctionConfig] ); + const submitVideoTargetComposer = useCallback( + async ( + nodeId: string, + payload?: VideoTargetComposerSubmitPayload, + ) => { + const node = nodes.find((item) => item.id === nodeId); + if (!node) return; + const incomingNodes = edges + .filter((edge) => edge.target === nodeId) + .map((edge) => nodes.find((item) => item.id === edge.source)) + .filter((item): item is CloneNode => Boolean(item)); + const imageReferenceLists = await Promise.all( + incomingNodes + .filter((item) => item.inputType === "image") + .map((item) => resolveNodeReferenceImages(item)) + ); + const imageUrls = Array.from( + new Set( + imageReferenceLists + .flat() + .map((item) => String(item || "").trim()) + .filter(Boolean) + ) + ).slice(0, 12); + const params = (node.draftValues?.params || {}) as Record; + const paramsSettings = + params.settings && typeof params.settings === "object" + ? (params.settings as Record) + : {}; + const requestedMode = + payload?.mode || + normalizeVideoGenerationMode(params.mode || paramsSettings.mode) || + (imageUrls.length >= 2 ? "first-last-frame" : imageUrls.length >= 1 ? "first-frame" : "text-to-video"); + const definition = getConfiguredFunction( + videoCanvasFunctionConfig, + getVideoFunctionKeyForMode(requestedMode), + ); + const requestedSettings = payload?.settings; + const preferredModelOverride = requestedSettings?.model || String(params.model || paramsSettings.model || ""); + const preferredModelRef = getFunctionModel(videoCanvasFunctionConfig, definition, preferredModelOverride); + const ratio = normalizeVideoRatioValue(requestedSettings?.ratio || params.ratio || paramsSettings.ratio); + const resolution = normalizeVideoResolutionValue( + requestedSettings?.resolution || params.resolution || paramsSettings.resolution, + ); + const duration = normalizeVideoDurationValue( + requestedSettings?.duration || params.duration || paramsSettings.duration, + ); + const referenceImages = selectVideoReferenceImages(imageUrls, requestedMode); + if (requestedMode === "first-frame" && referenceImages.length < 1) { + toast.error("图生视频至少需要 1 张参考图片"); + return; + } + if (requestedMode === "first-last-frame" && referenceImages.length < 2) { + toast.error("首尾帧模式至少需要 2 张参考图片"); + return; + } + if (requestedMode === "reference-images" && referenceImages.length < 1) { + toast.error("全能参考至少需要 1 张参考图片"); + return; + } + + const rawPrompt = String( + payload?.prompt ?? params.prompt ?? node.draftValues?.prompt ?? "", + ).trim(); + const finalPrompt = composeFunctionPrompt(definition, rawPrompt || node.title); + const previewUrl = + referenceImages[0] || + getNodeReferenceMediaUrl(incomingNodes.find((item) => item.inputType === "image")) || + getNodeReferenceMediaUrl(node) || + ""; + const savedSettings: ComposerSettings & { duration: string } = { + camera: requestedSettings?.camera || String(paramsSettings.camera || "摄像机"), + count: requestedSettings?.count || String(paramsSettings.count || "1个"), + model: normalizeModelRef(preferredModelOverride) || "auto", + panorama: requestedSettings?.panorama || String(paramsSettings.panorama || "全能参考"), + ratio, + resolution: formatVideoResolutionLabel(resolution), + duration: formatVideoDurationLabel(duration), + }; + + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + kind: "media", + subtitle: "生成中", + draftValues: { + ...clearGridSplitDraftValues(item.draftValues), + action: "VIDEO_GENERATE", + generated: false, + generationError: undefined, + generationProgress: 2, + generationStatus: "pending", + mediaType: "video", + mediaUrl: "", + nodeType: "video", + poster: previewUrl, + params: { + ...getNodeParams(item), + duration, + mode: requestedMode, + model: normalizeModelRef(preferredModelOverride) || undefined, + prompt: rawPrompt || finalPrompt, + ratio, + referenceImages, + resolution, + settings: savedSettings, + }, + prompt: rawPrompt || finalPrompt, + sourceUrl: previewUrl, + url: [], + }, + } + : item + ) + ); + + try { + const response = await videoGenerationService.generateVideo({ + duration, + images: referenceImages, + mode: requestedMode, + preferredModelRef, + prompt: finalPrompt, + ratio, + resolution, + }); + if (!response.success || !response.data?.url) { + throw new Error(response.error || "视频生成失败"); + } + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + kind: "media", + subtitle: "生成结果", + draftValues: { + ...clearGridSplitDraftValues(item.draftValues), + action: "VIDEO_GENERATE", + generated: true, + generationProgress: 100, + generationStatus: "complete", + mediaType: "video", + mediaUrl: response.data?.url, + nodeType: "video", + outputSizeLabel: `${formatVideoResolutionLabel(resolution)} · ${duration}s`, + params: { + ...getNodeParams(item), + duration, + mode: requestedMode, + model: normalizeModelRef(preferredModelOverride) || undefined, + prompt: rawPrompt || finalPrompt, + ratio, + referenceImages, + resolution, + settings: savedSettings, + }, + poster: previewUrl, + prompt: rawPrompt || finalPrompt, + sourceUrl: previewUrl, + url: [response.data?.url], + videoMeta: response.data?.metadata || null, + }, + } + : item + ) + ); + setSelectedNodeId(nodeId); + setCanvasCommand(null); + setNodeContextMenu(null); + setConnectionMenu(null); + setActivePanel(null); + toast.success("视频已开始生成并返回结果"); + } catch (error) { + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + kind: "media", + subtitle: "生成失败", + draftValues: { + ...clearGridSplitDraftValues(item.draftValues), + generationStatus: "failed", + generationError: error instanceof Error ? error.message : "视频生成失败", + poster: previewUrl, + }, + } + : item + ) + ); + toast.error(error instanceof Error ? error.message : "视频生成失败"); + } + }, + [edges, nodes, videoCanvasFunctionConfig] + ); + const runPresetComposerSubmit = useCallback( async ( nodeId: string, @@ -4292,43 +4594,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib } if (node.inputType === "video" || /VIDEO_GENERATE/.test(action)) { - const definition = getConfiguredFunction( - videoCanvasFunctionConfig, - imageUrls.length >= 2 ? "first-last-frame-video" : imageUrls.length === 1 ? "first-frame-video" : "text-to-video", - ); - const prompt = composeFunctionPrompt(definition, rawPrompt || node.title); - const response = await videoGenerationService.generateVideo({ - images: imageUrls.slice(0, 4), - preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition, nodeModelRef), - prompt, - ratio: "16:9", - resolution: "720p", - }); - if (!response.success || !response.data?.url) { - throw new Error(response.error || "视频生成失败"); - } - setNodes((current) => - current.map((item) => - item.id === nodeId - ? { - ...item, - kind: "media", - subtitle: "生成结果", - draftValues: { - ...(item.draftValues || {}), - generated: true, - mediaType: "video", - mediaUrl: response.data?.url, - nodeType: "video", - prompt, - url: [response.data?.url], - videoMeta: response.data?.metadata || null, - }, - } - : item - ) - ); - toast.success(`${node.title} 已生成视频结果`); + await submitVideoTargetComposer(nodeId); return; } @@ -4377,7 +4643,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib } catch (error) { toast.error(error instanceof Error ? error.message : `${node.title} 生成失败`); } - }, [edges, nodes, videoCanvasFunctionConfig]); + }, [edges, nodes, submitVideoTargetComposer, videoCanvasFunctionConfig]); const deleteNode = useCallback((nodeId: string) => { setNodes((current) => current.filter((node) => node.id !== nodeId)); @@ -4622,6 +4888,16 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib const selectedTargetModuleKind = selectedNode ? getTargetModuleKind(selectedNode) : null; const selectedTextResourceNode = selectedNode && getDraftString(selectedNode, "action") === "TEXT_RESOURCE" ? selectedNode : null; + const selectedTargetSourceNodes = useMemo( + () => + selectedNode && selectedTargetModuleKind + ? edges + .filter((edge) => edge.target === selectedNode.id) + .map((edge) => nodes.find((item) => item.id === edge.source)) + .filter((item): item is CloneNode => Boolean(item)) + : [], + [edges, nodes, selectedNode, selectedTargetModuleKind] + ); const selectedPresetAnchor = useMemo(() => { if (!selectedPresetControlNode) return null; return { @@ -4814,6 +5090,19 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib /> ) : null} + {selectedNode && selectedTargetModuleKind === "video" && selectedTargetAnchor ? ( + { + void submitVideoTargetComposer(selectedNode.id, payload); + }} + /> + ) : null} + void; + sourceNodes: CloneNode[]; +}) { + const params = (node.draftValues?.params || {}) as Record; + const paramsSettings = + params.settings && typeof params.settings === "object" + ? (params.settings as Record) + : {}; + const imageSources = sourceNodes.filter((item) => item.inputType === "image"); + const referenceCount = imageSources.length; + const savedMode = normalizeVideoGenerationMode(params.mode || paramsSettings.mode); + const initialTab = (() => { + if (savedMode === "first-last-frame") return "first-last-frame" as const; + if (savedMode === "first-frame") return "first-frame" as const; + if (savedMode === "reference-images") return "reference-images" as const; + if (savedMode === "text-to-video") return "text-to-video" as const; + return inferVideoComposerDefaultTab(referenceCount); + })(); + const initialPrompt = String(params.prompt || node.draftValues?.prompt || "").trim(); + const initialSettings: ComposerSettings & { duration: string } = { + camera: String(paramsSettings.camera || "摄像机"), + count: String(paramsSettings.count || "1个"), + model: + normalizeModelRef(String(params.model || paramsSettings.model || "")) || + getFunctionModelValue( + functionConfig, + getConfiguredFunction(functionConfig, getVideoFunctionKeyForMode(resolveVideoComposerMode(initialTab))), + ), + panorama: String(paramsSettings.panorama || "全能参考"), + ratio: normalizeVideoRatioValue(params.ratio || paramsSettings.ratio), + resolution: formatVideoResolutionLabel(params.resolution || paramsSettings.resolution), + duration: formatVideoDurationLabel(params.duration || paramsSettings.duration), + }; + const [activeTab, setActiveTab] = useState(initialTab); + const [menu, setMenu] = useState(null); + const [prompt, setPrompt] = useState(initialPrompt); + const [settings, setSettings] = useState(initialSettings); + const activeMode = resolveVideoComposerMode(activeTab); + const activeDefinition = getConfiguredFunction(functionConfig, getVideoFunctionKeyForMode(activeMode)); + const activeModelFallback = normalizeModelRef(getFunctionModelValue(functionConfig, activeDefinition)) || "auto"; + const activeMenu = + menu === "model" || menu === "ratio" || menu === "resolution" || menu === "duration" ? menu : null; + const activePromptPlaceholder = + activeMode === "text-to-video" + ? "描述视频画面、动作、情绪与镜头语言。" + : activeMode === "first-last-frame" + ? "描述首帧到尾帧之间的动作变化、转场与镜头节奏。" + : "补充参考图中的动作、镜头、节奏与氛围，让视频更准确。"; + const tabItems: Array<{ key: VideoComposerTabKey; label: string; enabled: boolean }> = [ + { key: "text-to-video", label: "文生视频", enabled: true }, + { key: "reference-images", label: "全能参考", enabled: referenceCount >= 1 }, + { key: "first-frame", label: "图生视频", enabled: referenceCount >= 1 }, + { key: "first-last-frame", label: "首尾帧", enabled: referenceCount >= 2 }, + { key: "image-reference", label: "图片参考", enabled: referenceCount >= 1 }, + ]; + const referenceThumbnails = imageSources + .map((item) => ({ id: item.id, title: item.title, url: getNodeReferenceMediaUrl(item) })) + .filter((item) => item.url) + .slice(0, 4); + + useEffect(() => { + setActiveTab(initialTab); + setPrompt(initialPrompt); + setSettings(initialSettings); + }, [initialPrompt, initialTab, node.id]); + + useEffect(() => { + if (!tabItems.find((item) => item.key === activeTab)?.enabled) { + setActiveTab(inferVideoComposerDefaultTab(referenceCount)); + } + }, [activeTab, referenceCount, tabItems]); + + useEffect(() => { + setSettings((current) => { + const normalized = normalizeModelRef(current.model) || "auto"; + if (normalized !== "auto") return current; + if (normalized === activeModelFallback) return current; + return { ...current, model: activeModelFallback }; + }); + }, [activeModelFallback]); + + return ( +
+
+ {activeMenu ? ( + { + if (activeMenu === "model") { + setSettings((current) => ({ ...current, model: option })); + } else if (activeMenu === "ratio") { + setSettings((current) => ({ ...current, ratio: option })); + } else if (activeMenu === "resolution") { + setSettings((current) => ({ ...current, resolution: option })); + } else if (activeMenu === "duration") { + setSettings((current) => ({ ...current, duration: option })); + } + setMenu(null); + }} + /> + ) : null} + +
+ {tabItems.map((item) => ( + + ))} +
+
+ {[ + ["标记", "◎"], + ["运镜", "▣"], + ["角色库", "♙"], + ].map(([label, icon]) => ( + + ))} + {referenceThumbnails.length > 0 ? ( + referenceThumbnails.map((thumbnail, index) => ( +
+ {thumbnail.title} + + {index + 1} + +
+ )) + ) : ( +
+ 接入图片参考后可直接图生视频 +
+ )} +
+