diff --git a/app/(admin)/settings/video-canvas/page.tsx b/app/(admin)/settings/video-canvas/page.tsx new file mode 100644 index 000000000..d65dc7b5b --- /dev/null +++ b/app/(admin)/settings/video-canvas/page.tsx @@ -0,0 +1,15 @@ +import { redirect } from "next/navigation"; +import { requireAdmin } from "@/lib/middleware/admin"; +import { VideoCanvasSettingsClient } from "./video-canvas-settings-client"; + +export const dynamic = "force-dynamic"; + +export default async function VideoCanvasSettingsPage() { + try { + await requireAdmin(); + } catch { + redirect("/"); + } + + return ; +} diff --git a/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx b/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx new file mode 100644 index 000000000..f4282fe52 --- /dev/null +++ b/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx @@ -0,0 +1,242 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import useSWR from "swr"; +import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import { fetcher } from "@/lib/utils"; +import { + type VideoCanvasFunctionConfig, + type VideoCanvasFunctionDefinition, + VIDEO_CANVAS_FUNCTION_CONFIG_KEY, + defaultVideoCanvasFunctionConfig, + normalizeVideoCanvasFunctionConfig, +} from "@/lib/video-canvas/config"; + +type SettingsResponse = { + setting?: { + value?: unknown; + }; +}; + +const functionKindLabels: Record = { + audio: "音频", + image: "图片生成", + "image-edit": "图片编辑", + multimodal: "多模态", + utility: "工具", + video: "视频生成", + workflow: "工作流", +}; + +export function VideoCanvasSettingsClient() { + const { data, isLoading, mutate } = useSWR( + `/api/admin/settings?key=${VIDEO_CANVAS_FUNCTION_CONFIG_KEY}`, + fetcher, + ); + const [config, setConfig] = useState(defaultVideoCanvasFunctionConfig); + const [saving, setSaving] = useState(false); + + useEffect(() => { + if (!data) return; + setConfig(normalizeVideoCanvasFunctionConfig(data.setting?.value)); + }, [data]); + + const functions = useMemo( + () => + Object.values(config.functions).sort((a, b) => { + const kindCompare = a.kind.localeCompare(b.kind); + return kindCompare || a.label.localeCompare(b.label); + }), + [config.functions], + ); + + const updateRoot = (updates: Partial) => { + setConfig((current) => ({ ...current, ...updates })); + }; + + const updateFunction = ( + key: string, + updates: Partial, + ) => { + setConfig((current) => ({ + ...current, + functions: { + ...current.functions, + [key]: { + ...current.functions[key], + ...updates, + }, + }, + })); + }; + + const save = async () => { + setSaving(true); + try { + const response = await fetch("/api/admin/settings", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY, + value: config, + }), + }); + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(payload?.message || payload?.error || "保存失败"); + } + toast.success("视频画布功能配置已保存"); + await mutate(); + } catch (error) { + toast.error(error instanceof Error ? error.message : "保存失败"); + } finally { + setSaving(false); + } + }; + + const resetToDefault = () => { + setConfig(defaultVideoCanvasFunctionConfig); + toast.info("已恢复默认配置,保存后生效"); + }; + + return ( + + + + 视频画布功能设置 + + 这里集中配置复刻画布里的工具能力、预设提示词、默认模型和扣点。前端优先按复刻目标触发,后端缺口可以先留空或后续替换。 + + + + + 恢复默认 + + + {saving ? "保存中..." : "保存配置"} + + + + + + + + 全局模型与开关 + + 默认模型可填写现有后端的模型标识;保持 auto 时由各接口按当前可用模型选择。 + + + + 启用视频画布功能串联 + updateRoot({ enabled: checked })} + /> + + + + + 默认图片/编辑模型 + updateRoot({ defaultImageModel: event.target.value })} + value={config.defaultImageModel} + /> + + + 默认视频模型 + updateRoot({ defaultVideoModel: event.target.value })} + value={config.defaultVideoModel} + /> + + + 默认多模态模型 + updateRoot({ defaultMultimodalModel: event.target.value })} + value={config.defaultMultimodalModel} + /> + + + + + + {functions.map((item) => ( + + + + + + {item.label} + + {functionKindLabels[item.kind]} + + + {item.route} + + + {item.description} + + + 启用 + updateFunction(item.key, { enabled: checked })} + /> + + + + + 模型覆盖 + updateFunction(item.key, { model: event.target.value })} + placeholder="留空使用全局默认模型" + value={item.model || ""} + /> + + + 扣点 + + updateFunction(item.key, { creditCost: Number(event.target.value || 0) }) + } + type="number" + value={item.creditCost ?? 0} + /> + + + + + 预设提示词 / 规则 + updateFunction(item.key, { prompt: event.target.value })} + value={item.prompt} + /> + + + ))} + + + ); +} diff --git a/app/api/admin/settings/route.ts b/app/api/admin/settings/route.ts index 9dabc5a44..aee21f78d 100644 --- a/app/api/admin/settings/route.ts +++ b/app/api/admin/settings/route.ts @@ -38,6 +38,10 @@ import { normalizeThreeDModelAliasConfig, THREED_MODEL_ALIAS_CONFIG_SETTING_KEY, } from "@/lib/model3d/model-alias-config"; +import { + normalizeVideoCanvasFunctionConfig, + VIDEO_CANVAS_FUNCTION_CONFIG_KEY, +} from "@/lib/video-canvas/config"; const settingSchema = z.object({ key: z.string(), @@ -196,6 +200,10 @@ export async function POST(request: Request) { nextValue = normalizedValue; } + if (key === VIDEO_CANVAS_FUNCTION_CONFIG_KEY) { + nextValue = normalizeVideoCanvasFunctionConfig(value); + } + if (key === THIRD_PARTY_SQLSERVER_CONFIG_KEY) { const existingSetting = await getSystemSetting({ key: THIRD_PARTY_SQLSERVER_CONFIG_KEY, diff --git a/app/api/video-canvas/config/route.ts b/app/api/video-canvas/config/route.ts new file mode 100644 index 000000000..a86c10d1b --- /dev/null +++ b/app/api/video-canvas/config/route.ts @@ -0,0 +1,17 @@ +import { getSystemSetting } from "@/lib/db/queries"; +import { + VIDEO_CANVAS_FUNCTION_CONFIG_KEY, + defaultVideoCanvasFunctionConfig, + normalizeVideoCanvasFunctionConfig, +} from "@/lib/video-canvas/config"; + +export const dynamic = "force-dynamic"; + +export async function GET() { + const setting = await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(() => null); + const config = setting?.value + ? normalizeVideoCanvasFunctionConfig(setting.value) + : defaultVideoCanvasFunctionConfig; + + return Response.json({ config }); +} diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index cce12b360..cbc26ef6f 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -44,7 +44,18 @@ import { toast } from "sonner"; import useSWR from "swr"; import capturedCanvasModules from "@/artifacts/liblib-frontend-capture-ed9a2c12570a49b9a1186017b2c38b52/canvas-modules.json"; import { cn, fetcher } from "@/lib/utils"; -import { canvasService } from "@/services/canvas"; +import { + canvasService, + imageGenerationService, + videoGenerationService, + workflowAnalysisService, +} from "@/services/canvas"; +import { + type VideoCanvasFunctionConfig, + type VideoCanvasFunctionDefinition, + defaultVideoCanvasFunctionConfig, + normalizeVideoCanvasFunctionConfig, +} from "@/lib/video-canvas/config"; type ActivePanel = | "add" @@ -371,7 +382,8 @@ const OFFICIAL_NODE_GAP = 100; const OFFICIAL_TEXT_RESOURCE_HEIGHT = 232; const OFFICIAL_TEXT_RESOURCE_WIDTH = 350; const OFFICIAL_IMAGE_TOOLBAR_SCALE = 0.7; -const OFFICIAL_COMPOSER_SCALE = 1; +const OFFICIAL_COMPOSER_SCALE = 0.42; +const OFFICIAL_ZOOM_DOCK_SCALE = 0.5; const PROMPT_COMPOSER_BASE_WIDTH = 895; const PROMPT_COMPOSER_BASE_HEIGHT = 286; const PRESET_COMPOSER_BASE_HEIGHT = 238; @@ -1138,6 +1150,45 @@ function getNodeSourceMediaUrl(node: CloneNode | null | undefined) { return getDraftString(node, "sourceUrl") || getNodeMediaUrl(node); } +function getConfiguredFunction( + config: VideoCanvasFunctionConfig, + key: string, +): VideoCanvasFunctionDefinition { + return config.functions[key] || defaultVideoCanvasFunctionConfig.functions[key]; +} + +function getFunctionModel( + config: VideoCanvasFunctionConfig, + definition: VideoCanvasFunctionDefinition, +) { + const fallback = + definition.kind === "video" + ? config.defaultVideoModel + : definition.kind === "multimodal" + ? config.defaultMultimodalModel + : config.defaultImageModel; + const model = (definition.model || fallback || "").trim(); + return model && model !== "auto" ? model : undefined; +} + +function composeFunctionPrompt( + definition: VideoCanvasFunctionDefinition, + userPrompt?: string, +) { + const base = definition.prompt.trim(); + const extra = String(userPrompt || "").trim(); + if (!base) return extra; + if (!extra || extra === base) return base; + return `${base}\n\n用户补充:${extra}`; +} + +function getFirstGeneratedImageUrl( + response: Awaited>, +) { + const first = response.data?.images?.[0]; + return first?.originalUrl || first?.thumbnailUrl || ""; +} + function createEqualGridRects(rows: number, columns: number): GridCropRect[] { return Array.from({ length: rows * columns }).map((_, index) => { const column = index % columns; @@ -1669,6 +1720,14 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib const canvasRef = useRef(null); const dragRef = useRef(null); const { data: siteBrandSettings } = useSWR("/api/site-settings", fetcher); + const { data: videoCanvasFunctionSettings } = useSWR<{ config?: VideoCanvasFunctionConfig }>( + "/api/video-canvas/config", + fetcher + ); + const videoCanvasFunctionConfig = useMemo( + () => normalizeVideoCanvasFunctionConfig(videoCanvasFunctionSettings?.config), + [videoCanvasFunctionSettings?.config] + ); const [nodes, setNodes] = useState([]); const [edges, setEdges] = useState([]); const [selectedNodeId, setSelectedNodeId] = useState(null); @@ -2435,7 +2494,8 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib const title = labels[action]; if (action === "grid") { const { columns, rows } = normalizeGridOption(option || "9宫格"); - const gridNode = createNineGridCompositeNode({ + const gridDefinition = getConfiguredFunction(videoCanvasFunctionConfig, "multi-view-nine-grid"); + let gridNode = createNineGridCompositeNode({ anchor, columns, option, @@ -2444,6 +2504,33 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib x: rightX, y: centerY(347), }); + if (videoCanvasFunctionConfig.enabled && gridDefinition?.enabled) { + const prompt = composeFunctionPrompt(gridDefinition, option); + const response = await imageGenerationService.generateImage({ + aspectRatio: "16:9", + imageCount: 1, + model: getFunctionModel(videoCanvasFunctionConfig, gridDefinition), + prompt, + referenceImage: sourceUrl, + }); + const generatedUrl = response.success ? getFirstGeneratedImageUrl(response) : ""; + if (generatedUrl) { + gridNode = { + ...gridNode, + draftValues: { + ...(gridNode.draftValues || {}), + generated: true, + mediaUrl: generatedUrl, + prompt, + sourceNodeId: anchor.id, + sourceUrl, + url: [generatedUrl], + }, + }; + } else if (!response.success) { + toast.error(response.error || "九宫格生成失败,已保留可切分占位节点"); + } + } setNodes((current) => [...current, gridNode]); setEdges((current) => [...current, { id: makeId("edge"), source: anchor.id, target: gridNode.id }]); setSelectedNodeId(gridNode.id); @@ -2455,8 +2542,92 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib return; } + const functionKey = + action === "lighting" + ? "lighting" + : action === "panorama" + ? "panorama-720" + : /扩/.test(option || "") + ? "image-outpaint" + : /重绘/.test(option || "") + ? "image-repaint" + : /擦/.test(option || "") + ? "image-erase" + : /抠/.test(option || "") + ? "remove-bg" + : "hd-upscale"; + const definition = getConfiguredFunction(videoCanvasFunctionConfig, functionKey); + const prompt = composeFunctionPrompt(definition, option?.trim() || title); + + if (videoCanvasFunctionConfig.enabled && definition?.enabled) { + let generatedUrl = ""; + if (definition.route === "enhance") { + const response = await imageGenerationService.enhanceImage(sourceUrl); + generatedUrl = response.success ? response.data?.url || "" : ""; + if (!generatedUrl && !response.success) toast.error(response.error || "高清处理失败"); + } else if (definition.route === "extend") { + const response = await imageGenerationService.extendImage(sourceUrl, { prompt }); + generatedUrl = response.success ? response.data?.url || "" : ""; + if (!generatedUrl && !response.success) toast.error(response.error || "扩图处理失败"); + } else if (definition.route === "remove-bg") { + const response = await imageGenerationService.removeBackground(sourceUrl); + generatedUrl = response.success ? response.data?.url || "" : ""; + if (!generatedUrl && !response.success) toast.error(response.error || "抠图处理失败"); + } else if (definition.route === "canvas-generate-image") { + const response = await imageGenerationService.generateImage({ + aspectRatio: "16:9", + imageCount: 1, + model: getFunctionModel(videoCanvasFunctionConfig, definition), + prompt, + referenceImage: sourceUrl, + }); + generatedUrl = response.success ? getFirstGeneratedImageUrl(response) : ""; + if (!generatedUrl && !response.success) toast.error(response.error || `${definition.label}处理失败`); + } + + if (generatedUrl) { + const resultNode = createMediaResourceNode({ + action: "IMAGE_RESOURCE", + height: 280, + mediaType: "image", + mediaUrl: generatedUrl, + prompt, + title: definition.label || title, + width: 420, + x: rightX, + y: centerY(280), + }); + setNodes((current) => [ + ...current, + { + ...resultNode, + draftValues: { + ...(resultNode.draftValues || {}), + generated: true, + params: { + mode: action, + prompt, + sourceNodeId: anchor.id, + sourceUrl, + }, + sourceNodeId: anchor.id, + sourceUrl, + toolbarOption: option || "", + }, + }, + ]); + setEdges((current) => [...current, { id: makeId("edge"), source: anchor.id, target: resultNode.id }]); + setSelectedNodeId(resultNode.id); + setCanvasCommand(null); + setNodeContextMenu(null); + setConnectionMenu(null); + setActivePanel(null); + toast.success(`已生成「${definition.label || title}」结果`); + return; + } + } + const created = createTargetModuleNode("image", rightX, 0, nodes.length + 1); - const prompt = option?.trim() || title; const resultNode: CloneNode = { ...created, title, @@ -2485,7 +2656,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib setActivePanel(null); toast.success(`已创建「${title}」工作流节点`); }, - [nodes] + [nodes, videoCanvasFunctionConfig] ); const updateTextResourceContent = useCallback((nodeId: string, content: string) => { @@ -2505,19 +2676,63 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib }, []); const submitSelectedImagePrompt = useCallback( - (nodeId: string, payload: ComposerSubmitPayload) => { + async (nodeId: string, payload: ComposerSubmitPayload) => { const anchor = nodes.find((node) => node.id === nodeId); if (!anchor) return; const sourceUrl = getNodeMediaUrl(anchor) || CHARACTER_IMAGE_URL; const rightX = Math.round(anchor.x + anchor.width + OFFICIAL_NODE_GAP); - const created = createTargetModuleNode("image", rightX, 0, nodes.length + 1); - const prompt = payload.prompt.trim() || payload.slashScene || "生成角色三视图"; + const starterMode = getPresetMode(anchor); + const functionKey = + payload.slashScene && /九宫格|宫格|多机位/.test(payload.slashScene) + ? "multi-view-nine-grid" + : starterMode === "character-three-view" + ? "character-three-view" + : "image-repaint"; + const definition = getConfiguredFunction(videoCanvasFunctionConfig, functionKey); + const prompt = composeFunctionPrompt( + definition, + payload.prompt.trim() || payload.slashScene || "生成角色三视图", + ); + const count = Math.max(1, Math.min(4, Number.parseInt(payload.settings.count, 10) || 1)); + let generatedUrl = ""; + + if (videoCanvasFunctionConfig.enabled && definition?.enabled) { + const response = await imageGenerationService.generateImage({ + aspectRatio: payload.settings.ratio, + imageCount: count, + model: getFunctionModel(videoCanvasFunctionConfig, definition), + prompt, + referenceImage: sourceUrl, + }); + generatedUrl = response.success ? getFirstGeneratedImageUrl(response) : ""; + if (!generatedUrl && !response.success) { + toast.error(response.error || `${definition.label}生成失败,已保留生成节点`); + } + } + + const generatedNode = generatedUrl + ? createMediaResourceNode({ + action: "IMAGE_RESOURCE", + height: functionKey === "multi-view-nine-grid" ? 347 : 280, + mediaType: "image", + mediaUrl: generatedUrl, + prompt, + title: payload.slashScene || definition.label || "图片生成", + width: functionKey === "multi-view-nine-grid" ? 560 : 420, + x: rightX, + y: Math.round(anchor.y + anchor.height / 2 - (functionKey === "multi-view-nine-grid" ? 347 : 280) / 2), + }) + : null; + const created = generatedNode || createTargetModuleNode("image", rightX, 0, nodes.length + 1); const resultNode: CloneNode = { ...created, - title: payload.slashScene || "图片生成", - y: Math.round(anchor.y + anchor.height / 2 - created.height / 2), + title: payload.slashScene || definition.label || "图片生成", + y: + generatedNode?.y ?? + Math.round(anchor.y + anchor.height / 2 - created.height / 2), draftValues: { ...(created.draftValues || {}), + generated: Boolean(generatedUrl) || created.draftValues?.generated, params: { prompt, references: payload.references, @@ -2528,6 +2743,8 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib }, sourceNodeId: anchor.id, sourceUrl, + targetPreset: starterMode || undefined, + toolbarOption: payload.slashScene || "", }, }; const referenceEdges = payload.references @@ -2545,9 +2762,113 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib setNodeContextMenu(null); setConnectionMenu(null); setActivePanel(null); - toast.success("已按当前提示词创建图片生成节点"); + toast.success(generatedUrl ? "已按当前提示词生成图片结果" : "已按当前提示词创建图片生成节点"); }, - [nodes] + [nodes, videoCanvasFunctionConfig] + ); + + const runPresetComposerSubmit = useCallback( + async (nodeId: string, mode: Exclude, draftPrompt: string) => { + const node = nodes.find((item) => item.id === nodeId); + if (!node) return; + const incomingNodes = edges + .filter((edge) => edge.target === nodeId) + .map((edge) => nodes.find((item) => item.id === edge.source)) + .filter((item): item is CloneNode => Boolean(item)); + const imageUrls = incomingNodes + .filter((item) => item.inputType === "image") + .map((item) => getNodeSourceMediaUrl(item)) + .filter(Boolean); + const videoUrls = incomingNodes + .filter((item) => item.inputType === "video") + .map((item) => getNodeSourceMediaUrl(item)) + .filter(Boolean); + + try { + if (mode === "story-script") { + const definition = getConfiguredFunction(videoCanvasFunctionConfig, "story-script"); + const textInputs = incomingNodes + .flatMap((item) => { + const content = item.draftValues?.content; + return Array.isArray(content) ? content.filter((value): value is string => typeof value === "string") : []; + }) + .join("\n\n"); + const prompt = composeFunctionPrompt(definition, `${draftPrompt}\n\n${textInputs}`.trim()); + const response = await workflowAnalysisService.analyze({ + imageUrls, + mode: "agent-plan", + nodeTitle: node.title, + prompt, + videoUrls, + }); + if (!response.success) throw new Error(response.error || "故事脚本生成失败"); + const content = + response.data?.raw || + response.data?.agentPlan?.join("\n") || + response.data?.summary || + ""; + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + draftValues: { + ...(item.draftValues || {}), + content: content ? [content] : item.draftValues?.content, + generated: true, + prompt, + rows: response.data?.shotIdeas || item.draftValues?.rows, + }, + } + : item + ) + ); + toast.success("故事脚本已生成"); + return; + } + + const definition = getConfiguredFunction( + videoCanvasFunctionConfig, + mode === "audio-to-video" ? "audio-to-video" : "first-frame-video", + ); + const prompt = composeFunctionPrompt(definition, draftPrompt); + const response = await videoGenerationService.generateVideo({ + images: imageUrls.slice(0, 4), + preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition), + prompt, + ratio: "16:9", + resolution: "720p", + }); + if (!response.success || !response.data?.url) { + throw new Error(response.error || `${definition.label}生成失败`); + } + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + kind: "media", + subtitle: "生成结果", + draftValues: { + ...(item.draftValues || {}), + generated: true, + mediaType: "video", + mediaUrl: response.data?.url, + nodeType: "video", + prompt, + url: [response.data?.url], + videoMeta: response.data?.metadata || null, + }, + } + : item + ) + ); + toast.success(`${definition.label}已生成`); + } catch (error) { + toast.error(error instanceof Error ? error.message : "生成失败"); + } + }, + [edges, nodes, videoCanvasFunctionConfig] ); const createFreeNode = useCallback( @@ -2598,14 +2919,170 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib toast.success("已复制节点"); }, [nodes]); - const runNode = useCallback((nodeId: string) => { + const runNode = useCallback(async (nodeId: string) => { const node = nodes.find((item) => item.id === nodeId); if (!node) return; setSelectedNodeId(nodeId); setNodeContextMenu(null); setConnectionMenu(null); - toast.message(`${node.title} 生成接口待接入。`); - }, [nodes]); + + const incomingNodes = edges + .filter((edge) => edge.target === nodeId) + .map((edge) => nodes.find((item) => item.id === edge.source)) + .filter((item): item is CloneNode => Boolean(item)); + const imageUrls = incomingNodes + .filter((item) => item.inputType === "image") + .map((item) => getNodeSourceMediaUrl(item)) + .filter(Boolean); + const videoUrls = incomingNodes + .filter((item) => item.inputType === "video") + .map((item) => getNodeSourceMediaUrl(item)) + .filter(Boolean); + const action = String(node.draftValues?.action || ""); + const params = (node.draftValues?.params || {}) as Record; + const rawPrompt = String(params.prompt || node.draftValues?.prompt || ""); + + try { + if (node.inputType === "image" || /IMAGE_GENERATE/.test(action)) { + const functionKey = + action === "IMAGE_GENERATE_NINE_GRID" + ? "multi-view-nine-grid" + : node.draftValues?.generatorType === "ENHANCE" + ? "hd-upscale" + : "image-repaint"; + const definition = getConfiguredFunction(videoCanvasFunctionConfig, functionKey); + const sourceUrl = imageUrls[0] || getNodeSourceMediaUrl(node); + const prompt = composeFunctionPrompt(definition, rawPrompt || String(params.option || node.title)); + let generatedUrl = ""; + + if (definition.route === "enhance" && sourceUrl) { + const response = await imageGenerationService.enhanceImage(sourceUrl); + generatedUrl = response.success ? response.data?.url || "" : ""; + if (!generatedUrl && !response.success) throw new Error(response.error || "图片增强失败"); + } else { + const response = await imageGenerationService.generateImage({ + aspectRatio: action === "IMAGE_GENERATE_NINE_GRID" ? "16:9" : undefined, + imageCount: 1, + model: getFunctionModel(videoCanvasFunctionConfig, definition), + prompt, + referenceImage: sourceUrl || undefined, + }); + generatedUrl = response.success ? getFirstGeneratedImageUrl(response) : ""; + if (!generatedUrl && !response.success) throw new Error(response.error || "图片生成失败"); + } + + if (!generatedUrl) throw new Error("图片生成未返回结果"); + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + kind: "media", + subtitle: "生成结果", + draftValues: { + ...(item.draftValues || {}), + generated: true, + mediaType: "image", + mediaUrl: generatedUrl, + nodeType: "image", + prompt, + sourceUrl, + url: [generatedUrl], + }, + } + : item + ) + ); + toast.success(`${node.title} 已生成图片结果`); + return; + } + + if (node.inputType === "video" || /VIDEO_GENERATE/.test(action)) { + const definition = getConfiguredFunction( + videoCanvasFunctionConfig, + imageUrls.length >= 2 ? "first-last-frame-video" : imageUrls.length === 1 ? "first-frame-video" : "text-to-video", + ); + const prompt = composeFunctionPrompt(definition, rawPrompt || node.title); + const response = await videoGenerationService.generateVideo({ + images: imageUrls.slice(0, 4), + preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition), + prompt, + ratio: "16:9", + resolution: "720p", + }); + if (!response.success || !response.data?.url) { + throw new Error(response.error || "视频生成失败"); + } + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + kind: "media", + subtitle: "生成结果", + draftValues: { + ...(item.draftValues || {}), + generated: true, + mediaType: "video", + mediaUrl: response.data?.url, + nodeType: "video", + prompt, + url: [response.data?.url], + videoMeta: response.data?.metadata || null, + }, + } + : item + ) + ); + toast.success(`${node.title} 已生成视频结果`); + return; + } + + if (node.inputType === "text" || /SCRIPT_GENERATE|TEXT_GENERATE/.test(action)) { + const definition = getConfiguredFunction( + videoCanvasFunctionConfig, + imageUrls.length > 0 || videoUrls.length > 0 ? "image-to-prompt" : "story-script", + ); + const prompt = composeFunctionPrompt(definition, rawPrompt || STORY_SCRIPT_PROMPT); + const response = await workflowAnalysisService.analyze({ + imageUrls, + mode: "agent-plan", + nodeTitle: node.title, + prompt, + videoUrls, + }); + if (!response.success) throw new Error(response.error || "多模态分析失败"); + const content = + response.data?.raw || + response.data?.agentPlan?.join("\n") || + response.data?.summary || + response.data?.recommendedNextSteps?.join("\n") || + ""; + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + draftValues: { + ...(item.draftValues || {}), + content: content ? [content] : item.draftValues?.content, + generated: true, + prompt, + rows: response.data?.shotIdeas || item.draftValues?.rows, + }, + } + : item + ) + ); + toast.success(`${node.title} 已生成文本结果`); + return; + } + + toast.info(`${node.title} 暂无可执行的后端动作`); + } catch (error) { + toast.error(error instanceof Error ? error.message : `${node.title} 生成失败`); + } + }, [edges, nodes, videoCanvasFunctionConfig]); const deleteNode = useCallback((nodeId: string) => { setNodes((current) => current.filter((node) => node.id !== nodeId)); @@ -2971,6 +3448,13 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib nodes={nodes} onComposerSubmit={(payload) => submitSelectedImagePrompt(selectedPresetControlNode.id, payload)} onImageToolAction={(action, option) => runSelectedImageToolAction(selectedPresetControlNode.id, action, option)} + onPresetSubmit={(prompt) => + runPresetComposerSubmit( + selectedPresetControlNode.id, + selectedPresetMode ?? "character-three-view", + prompt + ) + } selectedNode={selectedPresetControlNode} /> ) : null} @@ -3139,7 +3623,22 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib {angleEditorNode ? ( { + onApply={async (settings) => { + const angleDefinition = getConfiguredFunction(videoCanvasFunctionConfig, "angle-edit"); + const sourceUrl = getNodeMediaUrl(angleEditorNode) || getNodeSourceMediaUrl(angleEditorNode); + let generatedUrl = ""; + if (sourceUrl && videoCanvasFunctionConfig.enabled && angleDefinition?.enabled) { + const response = await imageGenerationService.editAngle({ + horizontalAngle: settings.horizontalAngle, + imageUrl: sourceUrl, + verticalAngle: settings.verticalAngle, + zoom: settings.zoom, + }); + generatedUrl = response.success ? response.data?.url || response.data?.thumbnailUrl || "" : ""; + if (!generatedUrl && !response.success) { + toast.error(response.error || "多角度生成失败,已保留参数"); + } + } setNodes((current) => current.map((node) => node.id === angleEditorNode.id @@ -3160,8 +3659,39 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib : node ) ); + if (generatedUrl) { + const resultNode = createMediaResourceNode({ + action: "IMAGE_RESOURCE", + height: 280, + mediaType: "image", + mediaUrl: generatedUrl, + prompt: angleDefinition.prompt, + title: "多角度结果", + width: 420, + x: Math.round(angleEditorNode.x + angleEditorNode.width + OFFICIAL_NODE_GAP), + y: Math.round(angleEditorNode.y + angleEditorNode.height / 2 - 140), + }); + setNodes((current) => [ + ...current, + { + ...resultNode, + draftValues: { + ...(resultNode.draftValues || {}), + angleSettings: settings, + generated: true, + sourceNodeId: angleEditorNode.id, + sourceUrl, + }, + }, + ]); + setEdges((current) => [ + ...current, + { id: makeId("edge"), source: angleEditorNode.id, target: resultNode.id }, + ]); + setSelectedNodeId(resultNode.id); + } setAngleEditorNodeId(null); - toast.success("已应用多角度参数,生成接口待接入。"); + toast.success(generatedUrl ? "已按多角度参数生成结果" : "已应用多角度参数"); }} onClose={() => setAngleEditorNodeId(null)} /> @@ -3356,41 +3886,43 @@ function ZoomDock({ return ( - {overviewOpen ? ( - - ) : null} - - - - - - - - - - - - - - - - - {Math.round(zoom * 100)}% - - - + - + + {overviewOpen ? ( + + ) : null} + + + + + + + + + + + + + - + + + {Math.round(zoom * 100)}% + + + + + + ); @@ -3412,7 +3944,7 @@ function ZoomDockButton({ return ( void; onImageToolAction: (action: ImageToolbarAction, option?: string) => void; + onPresetSubmit: (prompt: string) => void; selectedNode: CloneNode; }) { if (mode === "character-three-view") { @@ -3667,6 +4201,7 @@ function PresetWorkflowControls({ - + ); } @@ -3779,7 +4315,15 @@ function CharacterPresetToolbar({ ); } -function AudioToVideoComposerPanel({ anchor }: { anchor?: PresetControlAnchor }) { +function AudioToVideoComposerPanel({ + anchor, + onSubmit, +}: { + anchor?: PresetControlAnchor; + onSubmit?: (prompt: string) => void; +}) { + const [prompt, setPrompt] = useState(AUDIO_TO_VIDEO_PROMPT); + return ( setPrompt(event.target.value)} placeholder="根据上传的音频生成对应场景画面,镜头语言、节奏、音乐匹配情绪变化,电影级质感。" + value={prompt} /> toast.info("视频模型选择后端待接入。")} /> @@ -3862,7 +4407,13 @@ function AudioToVideoComposerPanel({ anchor }: { anchor?: PresetControlAnchor }) toast.info("音频生视频生成接口待接入。")} + onClick={() => { + if (onSubmit) { + onSubmit(prompt); + return; + } + toast.info("音频生视频生成接口待接入。"); + }} type="button" > ↑ @@ -4341,6 +4892,7 @@ function AudioResourcePanel() { function PresetComposerPanel({ anchor, credit, + onSubmit, prompt, submitMessage, thumbnailUrl, @@ -4348,6 +4900,7 @@ function PresetComposerPanel({ }: { anchor?: PresetControlAnchor; credit: string; + onSubmit?: (prompt: string) => void; prompt: string; submitMessage: string; thumbnailUrl?: string; @@ -4409,7 +4962,13 @@ function PresetComposerPanel({ toast.info(submitMessage)} + onClick={() => { + if (onSubmit) { + onSubmit(draftPrompt); + return; + } + toast.info(submitMessage); + }} type="button" > ↑ diff --git a/docs/video-canvas-official-function-map.md b/docs/video-canvas-official-function-map.md new file mode 100644 index 000000000..ccc7176af --- /dev/null +++ b/docs/video-canvas-official-function-map.md @@ -0,0 +1,68 @@ +# 视频画布复刻功能映射 + +本文档记录 LibTV 目标画布的功能分类、官方接口线索,以及当前项目的前后端串联策略。目标优先级是“复刻前端和工作流逻辑优先,后端缺口留空可配置”。 + +## 官方接口线索 + +- 工具定义:`GET /api/tool_spec/list` +- 工作流预设:`GET /api/canvas/workflow/list` +- 创建生成任务:`POST /api/task/generation/create` +- 生成进度:`POST /api/task/generation/progress/batch` +- 打光专项:`POST /api/task/generation/lighting` +- 视频优化:`POST /api/task/generation/video/opt` +- 扣点估算:`POST /api/task/generation/power/calculator` 和 `/batch` +- 画布保存:`POST /api/canvas/project/draft/update` + +抓取到的官方工具规格里,工具大类约为:图片 22、视频 38、音频 3、文本 3、合成 1。模式大类包括 image2image、image2video、singleImage2video、frames2video、audio2video、image2text、video2text、videoEdit2video、video2video。 + +## 功能分类 + +| 功能 | 官方逻辑判断 | 当前串联 | +| --- | --- | --- | +| 角色三视图 | 图片编辑/图生图模型,参考图保持角色一致性 | `/api/canvas/generate/image`,提示词在系统设置配置 | +| 多机位九宫格 | 图片编辑/图生图模型生成带黑色分割线的 3x3 分镜图 | `/api/canvas/generate/image`,生成结果再支持宫格切分 | +| 宫格切分 | 图像识别/本地分割线检测,按黑线切 4/9/16/25 张 | 当前前端本地检测黑线并生成切片节点 | +| 多角度编辑器 | 图片编辑模型,参数含水平环绕、垂直俯仰、景别缩放 | `/api/angle-edit` | +| 高清 | 图像增强/超分接口 | `/api/enhance` | +| 扩图/720 全景 | 图片拓展或图片编辑模型 | `/api/extend` | +| 重绘/擦除 | 图片编辑生图模型,擦除后续可替换蒙版接口 | `/api/canvas/generate/image` | +| 抠图 | 图像分割/抠图接口 | `/api/rmbg` | +| 图片反推提示词 | 多模态模型 image2text | `/api/canvas/workflow/analyze` | +| 故事/分镜脚本 | 文本模型或多模态模型,输出表格化镜号 | `/api/canvas/workflow/analyze` | +| 文生视频 | 视频生成模型 text2video | `/api/canvas/generate/video` | +| 首帧图生视频 | 视频生成模型 image2video | `/api/canvas/generate/video` | +| 首尾帧生视频 | 视频生成模型 frames2video | `/api/canvas/generate/video`,最多传 4 张图 | +| 音频生视频 | audio2video,官方会结合音频节拍/情绪和参考图 | 当前先用提示词+参考图走 `/api/canvas/generate/video`,音频节拍解析后端后续补 | +| 工具箱工作流模板 | 预设画布工作流,点击后在当前画布展开节点 | 前端已按官方工作流思路展开,后续模板内容可配置化扩充 | + +## 配置入口 + +新增系统设置键:`video_canvas_function_config` + +新增页面:`/settings/video-canvas` + +可配置内容: + +- 总开关 +- 默认图片/视频/多模态模型 +- 每个功能的启用状态 +- 每个功能的模型覆盖 +- 每个功能的扣点 +- 每个功能的预设提示词/规则 + +## 当前已接入的前端触发点 + +- 选中图片后的九宫格、高清、扩图、打光、全景、重绘、擦除、抠图 +- 九宫格切分,识别黑色分割线拆为单独镜头图 +- 多角度编辑器提交 +- 角色三视图提示词框提交 +- 故事脚本、首帧视频、音频生视频预设 Composer 提交 +- 右键“立即生成”按节点类型执行图片/视频/多模态分析 + +## 后续后端缺口 + +- 真正的 audio2video 音频节拍/情绪解析输入 +- 视频编辑到视频、视频参考生成分镜脚本的专用接口 +- 擦除/局部重绘的蒙版编辑接口 +- 官方扣点估算等价接口 +- 官方 67 个工具规格的完整后台模型路由表 diff --git a/lib/video-canvas/config.ts b/lib/video-canvas/config.ts new file mode 100644 index 000000000..cb9821a3c --- /dev/null +++ b/lib/video-canvas/config.ts @@ -0,0 +1,313 @@ +export const VIDEO_CANVAS_FUNCTION_CONFIG_KEY = "video_canvas_function_config"; + +export type VideoCanvasFunctionKind = + | "multimodal" + | "image" + | "image-edit" + | "video" + | "audio" + | "utility" + | "workflow"; + +export type VideoCanvasBackendRoute = + | "canvas-generate-image" + | "canvas-generate-video" + | "canvas-workflow-analyze" + | "angle-edit" + | "enhance" + | "enhance-pro" + | "extend" + | "remove-bg" + | "layer-split" + | "local"; + +export type VideoCanvasFunctionDefinition = { + key: string; + label: string; + kind: VideoCanvasFunctionKind; + route: VideoCanvasBackendRoute; + enabled: boolean; + model?: string; + creditCost?: number; + prompt: string; + description: string; + requiresBackend?: boolean; +}; + +export type VideoCanvasFunctionConfig = { + enabled: boolean; + defaultImageModel: string; + defaultVideoModel: string; + defaultMultimodalModel: string; + functions: Record; +}; + +const defaultFunctions: VideoCanvasFunctionDefinition[] = [ + { + key: "character-three-view", + label: "角色三视图", + kind: "image-edit", + route: "canvas-generate-image", + enabled: true, + creditCost: 14, + prompt: + "以输入角色图为唯一角色参考,生成正面、侧面、背面三视图角色设定图。保持五官、发型、服装、颜色和体型一致,干净白底,三视图水平排布,适合后续视频角色一致性使用。", + description: "官方角色三视图预设,使用参考图生图/编辑模型。", + }, + { + key: "multi-view-nine-grid", + label: "多机位九宫格", + kind: "image-edit", + route: "canvas-generate-image", + enabled: true, + creditCost: 14, + prompt: + "以选中图片为角色/主体参考,生成多视角 3x3 九宫格分镜图。保持主体一致,九个镜头包含远景、全身、侧身、背面、半身、近景、局部细节等视角,使用清晰黑色分割线分隔每一格,白色或干净背景。", + description: "官方九宫格工具:先生成带黑色分割线的 3x3 分镜图,再由宫格切分拆成单张镜头。", + }, + { + key: "grid-split", + label: "宫格切分", + kind: "utility", + route: "local", + enabled: true, + creditCost: 0, + prompt: "识别黑色宫格分割线,将合成图切分为可单独调整的镜头图片。", + description: "浏览器端识别黑色分割线并生成切片节点;后端可后续替换为更稳的图像分割服务。", + }, + { + key: "lighting", + label: "打光", + kind: "image-edit", + route: "canvas-generate-image", + enabled: true, + creditCost: 14, + prompt: + "在保持主体、构图和身份一致的前提下,对图片进行电影级光影重塑,增强轮廓光、层次、对比和空间感,保留自然材质细节。", + description: "图片编辑生图模型,复刻官方打光工具。", + }, + { + key: "hd-upscale", + label: "高清", + kind: "utility", + route: "enhance", + enabled: true, + creditCost: 8, + prompt: "提升图片清晰度、细节和分辨率,保持原图内容不变。", + description: "接入现有图片增强接口;高清重绘可在模型字段指定增强 Pro。", + }, + { + key: "panorama-720", + label: "720 全景", + kind: "image-edit", + route: "extend", + enabled: true, + creditCost: 14, + prompt: + "基于当前图片向四周扩展为 720 全景视野,保持主体与场景一致,补全自然环境和空间透视,避免变形和重复纹理。", + description: "优先使用拓图接口,必要时可切到图片编辑生图模型。", + }, + { + key: "image-outpaint", + label: "扩图", + kind: "utility", + route: "extend", + enabled: true, + creditCost: 10, + prompt: "在保持主体不变的前提下扩展画面边界,补全自然背景和构图。", + description: "对应官方高清菜单里的扩图。", + }, + { + key: "image-repaint", + label: "重绘", + kind: "image-edit", + route: "canvas-generate-image", + enabled: true, + creditCost: 14, + prompt: "保持主体结构和身份一致,按照用户补充提示重绘画面局部或整体风格。", + description: "对应官方高清菜单里的重绘。", + }, + { + key: "image-erase", + label: "擦除", + kind: "image-edit", + route: "canvas-generate-image", + enabled: true, + creditCost: 10, + prompt: "根据用户标记或文字说明移除指定元素,并自然补全背景纹理。", + description: "当前以前端提示词占位接图片编辑模型,后续可替换为蒙版编辑接口。", + }, + { + key: "remove-bg", + label: "抠图", + kind: "utility", + route: "remove-bg", + enabled: true, + creditCost: 4, + prompt: "移除图片背景,保留主体透明或干净背景结果。", + description: "接入现有抠图接口。", + }, + { + key: "image-crop", + label: "裁剪", + kind: "utility", + route: "local", + enabled: true, + creditCost: 0, + prompt: "按用户选择的裁剪比例或宫格切片范围处理图片。", + description: "本地画布裁剪/宫格切分能力。", + }, + { + key: "angle-edit", + label: "多角度编辑器", + kind: "image-edit", + route: "angle-edit", + enabled: true, + creditCost: 1, + prompt: "根据水平环绕、垂直俯仰和景别缩放参数生成新的角色/主体视角。", + description: "接现有 /api/angle-edit,多角度编辑器提交时触发。", + }, + { + key: "story-script", + label: "故事脚本生成", + kind: "multimodal", + route: "canvas-workflow-analyze", + enabled: true, + creditCost: 6, + prompt: "根据输入文本、图片或视频参考,生成结构化分镜脚本,包含镜号、场景、画面、声音和镜头节奏。", + description: "多模态大模型/文本模型生成脚本表格。", + }, + { + key: "image-to-prompt", + label: "图片反推提示词", + kind: "multimodal", + route: "canvas-workflow-analyze", + enabled: true, + creditCost: 3, + prompt: "分析参考图片,输出结构化中文提示词,包括主体、环境、构图、光影、镜头语言和风格关键词。", + description: "多模态识图生成提示词。", + }, + { + key: "text-to-video", + label: "文生视频", + kind: "video", + route: "canvas-generate-video", + enabled: true, + creditCost: 14, + prompt: "根据文本描述生成短视频,保持镜头运动、节奏和电影级质感。", + description: "视频生成模型,走现有视频生成接口。", + }, + { + key: "first-frame-video", + label: "首帧图生视频", + kind: "video", + route: "canvas-generate-video", + enabled: true, + creditCost: 14, + prompt: "以参考图片作为视频首帧,生成自然流畅的短视频。", + description: "图片到视频/首帧生视频。", + }, + { + key: "first-last-frame-video", + label: "首尾帧生成视频", + kind: "video", + route: "canvas-generate-video", + enabled: true, + creditCost: 14, + prompt: "以两张参考图作为首帧和尾帧,生成首尾连贯的短视频。", + description: "首尾帧视频生成。", + }, + { + key: "audio-to-video", + label: "音频生视频", + kind: "video", + route: "canvas-generate-video", + enabled: true, + creditCost: 14, + prompt: "根据上传音频生成对应场景画面,镜头语言、节奏、音乐匹配情绪变化,电影级质感。", + description: "官方为音频+参考图到视频;当前先接视频生成提示词和参考图,音频节拍解析接口后续补充。", + requiresBackend: true, + }, +]; + +export const defaultVideoCanvasFunctionConfig: VideoCanvasFunctionConfig = { + enabled: true, + defaultImageModel: "auto", + defaultVideoModel: "auto", + defaultMultimodalModel: "auto", + functions: Object.fromEntries(defaultFunctions.map((item) => [item.key, item])), +}; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function normalizeFunctionDefinition( + key: string, + value: unknown, + fallback: VideoCanvasFunctionDefinition, +): VideoCanvasFunctionDefinition { + const incoming = isRecord(value) ? value : {}; + const normalized: VideoCanvasFunctionDefinition = { + ...fallback, + key, + label: String(incoming.label || fallback.label), + kind: (incoming.kind as VideoCanvasFunctionKind) || fallback.kind, + route: (incoming.route as VideoCanvasBackendRoute) || fallback.route, + enabled: typeof incoming.enabled === "boolean" ? incoming.enabled : fallback.enabled, + model: typeof incoming.model === "string" ? incoming.model : fallback.model, + creditCost: + typeof incoming.creditCost === "number" && Number.isFinite(incoming.creditCost) + ? Math.max(0, incoming.creditCost) + : fallback.creditCost, + prompt: String(incoming.prompt || fallback.prompt), + description: String(incoming.description || fallback.description), + requiresBackend: + typeof incoming.requiresBackend === "boolean" + ? incoming.requiresBackend + : fallback.requiresBackend, + }; + return normalized; +} + +export function normalizeVideoCanvasFunctionConfig(value: unknown): VideoCanvasFunctionConfig { + if (!isRecord(value)) return defaultVideoCanvasFunctionConfig; + const incomingFunctions = isRecord(value.functions) ? value.functions : {}; + const functions: Record = {}; + + for (const [key, fallback] of Object.entries(defaultVideoCanvasFunctionConfig.functions)) { + functions[key] = normalizeFunctionDefinition(key, incomingFunctions[key], fallback); + } + + for (const [key, customValue] of Object.entries(incomingFunctions)) { + if (functions[key] || !isRecord(customValue)) continue; + const fallback: VideoCanvasFunctionDefinition = { + key, + label: String(customValue.label || key), + kind: (customValue.kind as VideoCanvasFunctionKind) || "workflow", + route: (customValue.route as VideoCanvasBackendRoute) || "local", + enabled: customValue.enabled !== false, + creditCost: typeof customValue.creditCost === "number" ? Math.max(0, customValue.creditCost) : 0, + prompt: String(customValue.prompt || ""), + description: String(customValue.description || "自定义视频画布功能"), + }; + functions[key] = normalizeFunctionDefinition(key, customValue, fallback); + } + + return { + enabled: value.enabled !== false, + defaultImageModel: + typeof value.defaultImageModel === "string" && value.defaultImageModel.trim() + ? value.defaultImageModel.trim() + : defaultVideoCanvasFunctionConfig.defaultImageModel, + defaultVideoModel: + typeof value.defaultVideoModel === "string" && value.defaultVideoModel.trim() + ? value.defaultVideoModel.trim() + : defaultVideoCanvasFunctionConfig.defaultVideoModel, + defaultMultimodalModel: + typeof value.defaultMultimodalModel === "string" && value.defaultMultimodalModel.trim() + ? value.defaultMultimodalModel.trim() + : defaultVideoCanvasFunctionConfig.defaultMultimodalModel, + functions, + }; +}
+ 这里集中配置复刻画布里的工具能力、预设提示词、默认模型和扣点。前端优先按复刻目标触发,后端缺口可以先留空或后续替换。 +