diff --git a/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx b/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx index f9743b16d..56bc3e8f7 100644 --- a/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx +++ b/app/(admin)/settings/video-canvas/video-canvas-settings-client.tsx @@ -13,8 +13,19 @@ import { } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; +import { + isImageGenerationModelDescriptor, + isVideoModelDescriptor, +} from "@/lib/ai/model-classification"; import { fetcher } from "@/lib/utils"; import { type VideoCanvasFunctionConfig, @@ -30,6 +41,26 @@ type SettingsResponse = { }; }; +type AiModelOption = { + id: string; + modelId: string; + displayName: string; + providerName?: string; + providerType?: string; + supportsVision?: boolean; + supportsImageGen?: boolean; + isEnabled?: boolean; + isHidden?: boolean; + isMultimodal?: boolean; + capabilities?: unknown; +}; + +type AiModelsResponse = { + models?: AiModelOption[]; +}; + +const INHERIT_MODEL_VALUE = "__inherit__"; + const functionKindLabels: Record = { audio: "音频", image: "图片生成", @@ -40,13 +71,125 @@ const functionKindLabels: Record workflow: "工作流", }; +function getModelOptionValue(model: AiModelOption) { + return String(model.modelId || model.id || "").trim(); +} + +function getModelOptionLabel(model: AiModelOption) { + const name = model.displayName || model.modelId || model.id; + const id = model.modelId && model.modelId !== name ? ` · ${model.modelId}` : ""; + return `${name}${id}`; +} + +function isImageModelOption(model: AiModelOption) { + return isImageGenerationModelDescriptor({ + providerType: model.providerType, + modelId: model.modelId, + capabilities: model.capabilities, + supportsImageGen: model.supportsImageGen, + }); +} + +function isVideoModelOption(model: AiModelOption) { + return isVideoModelDescriptor({ + providerType: model.providerType, + modelId: model.modelId, + capabilities: model.capabilities, + supportsImageGen: model.supportsImageGen, + }); +} + +function filterModelsForFunctionKind( + models: AiModelOption[], + kind: VideoCanvasFunctionDefinition["kind"], +) { + const enabled = models.filter((model) => model.isEnabled !== false && getModelOptionValue(model)); + let filtered = enabled; + if (kind === "image" || kind === "image-edit") { + filtered = enabled.filter(isImageModelOption); + } else if (kind === "video") { + filtered = enabled.filter(isVideoModelOption); + } else if (kind === "multimodal") { + filtered = enabled.filter( + (model) => model.isMultimodal || model.supportsVision || (!isImageModelOption(model) && !isVideoModelOption(model)), + ); + } + return filtered.length ? filtered : enabled; +} + +function ModelSelect({ + allowInherit = false, + kind, + models, + onChange, + value, +}: { + allowInherit?: boolean; + kind: VideoCanvasFunctionDefinition["kind"]; + models: AiModelOption[]; + onChange: (value: string) => void; + value: string; +}) { + const filteredModels = filterModelsForFunctionKind(models, kind); + const normalizedValue = value?.trim() || (allowInherit ? INHERIT_MODEL_VALUE : "auto"); + const hasCurrent = + normalizedValue === "auto" || + normalizedValue === INHERIT_MODEL_VALUE || + filteredModels.some((model) => getModelOptionValue(model) === normalizedValue); + + return ( + + ); +} + export function VideoCanvasSettingsClient() { const { data, isLoading, mutate } = useSWR( `/api/admin/settings?key=${VIDEO_CANVAS_FUNCTION_CONFIG_KEY}`, fetcher, ); + const { data: aiModelsData } = useSWR("/api/admin/ai-models", fetcher); const [config, setConfig] = useState(defaultVideoCanvasFunctionConfig); const [saving, setSaving] = useState(false); + const modelOptions = useMemo( + () => (aiModelsData?.models || []).filter((model) => model.isEnabled !== false), + [aiModelsData?.models], + ); useEffect(() => { if (!data) return; @@ -135,7 +278,7 @@ export function VideoCanvasSettingsClient() {
全局模型与开关 - 默认模型可填写现有后端的模型标识；保持 auto 时由各接口按当前可用模型选择。 + 默认模型从后台已启用模型中选择；保持自动时由各接口按当前可用模型选择。
@@ -150,25 +293,28 @@ export function VideoCanvasSettingsClient() {
- updateRoot({ defaultImageModel: event.target.value })} + updateRoot({ defaultImageModel: value || "auto" })} value={config.defaultImageModel} />
- updateRoot({ defaultVideoModel: event.target.value })} + updateRoot({ defaultVideoModel: value || "auto" })} value={config.defaultVideoModel} />
- updateRoot({ defaultMultimodalModel: event.target.value })} + updateRoot({ defaultMultimodalModel: value || "auto" })} value={config.defaultMultimodalModel} />
@@ -212,10 +358,11 @@ export function VideoCanvasSettingsClient() {
- updateFunction(item.key, { model: event.target.value })} - placeholder="留空使用全局默认模型" + updateFunction(item.key, { model: value })} value={item.model || ""} />
diff --git a/app/api/canvas/workflow/analyze/route.ts b/app/api/canvas/workflow/analyze/route.ts index e4795b079..4730a0306 100644 --- a/app/api/canvas/workflow/analyze/route.ts +++ b/app/api/canvas/workflow/analyze/route.ts @@ -19,6 +19,8 @@ const requestSchema = z.object({ imageUrls: z.array(z.string().trim()).optional(), videoUrls: z.array(z.string().trim()).optional(), nodeTitle: z.string().trim().optional(), + preferredModelRef: z.string().trim().optional(), + model: z.string().trim().optional(), }); function sanitizeUrls(value: unknown, limit = 6) { @@ -263,6 +265,7 @@ export async function POST(request: Request) { const prompt = normalizeText(parsed.data.prompt); const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4); const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4); + const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model); if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) { return NextResponse.json( { success: false, error: "请至少提供脚本、图片或视频上下文" }, @@ -278,14 +281,25 @@ export async function POST(request: Request) { (item) => !item.model.isSubModel && item.model.isEnabled ); + const preferredModel = + preferredModelRef + ? compatibleModels.find( + (item) => + item.model.id === preferredModelRef || + item.model.modelId === preferredModelRef, + ) + : null; + let selectedModel = - imageUrls.length > 0 && - agentModel && - supportsVision(agentModel.model) - ? agentModel - : imageUrls.length > 0 - ? compatibleModels.find((item) => supportsVision(item.model)) - : agentModel || compatibleModels[0]; + preferredModel && (imageUrls.length === 0 || supportsVision(preferredModel.model)) + ? preferredModel + : imageUrls.length > 0 && + agentModel && + supportsVision(agentModel.model) + ? agentModel + : imageUrls.length > 0 + ? compatibleModels.find((item) => supportsVision(item.model)) + : agentModel || compatibleModels[0]; if (!selectedModel) { selectedModel = compatibleModels[0]; diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 2b2f28929..a6d7e883e 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -43,6 +43,10 @@ import { import { toast } from "sonner"; import useSWR from "swr"; import capturedCanvasModules from "@/artifacts/liblib-frontend-capture-ed9a2c12570a49b9a1186017b2c38b52/canvas-modules.json"; +import { + isImageGenerationModelDescriptor, + isVideoModelDescriptor, +} from "@/lib/ai/model-classification"; import { cn, fetcher } from "@/lib/utils"; import { canvasService, @@ -201,6 +205,22 @@ type ComposerSettings = { resolution: string; }; +type CanvasModelOption = { + id: string; + modelId?: string; + displayName: string; + providerName?: string; + providerType?: string; + supportsVision?: boolean; + supportsImageGen?: boolean; + isMultimodal?: boolean; + isDefault?: boolean; + isSubModel?: boolean; + capabilities?: unknown; +}; + +type CanvasModelKind = "image" | "video" | "multimodal" | "any"; + type AddModuleKind = "text" | "image" | "video" | "video-clip" | "audio" | "script"; type CanvasCreateKind = AddModuleKind; type TargetModuleActionLabel = @@ -1178,9 +1198,30 @@ function getDefaultFunctionPrompt(key: string) { return defaultVideoCanvasFunctionConfig.functions[key]?.prompt || ""; } +function normalizeModelRef(value: unknown) { + const model = String(value || "").trim(); + return model && model !== "auto" ? model : ""; +} + function getFunctionModel( config: VideoCanvasFunctionConfig, definition: VideoCanvasFunctionDefinition, + overrideModel?: string, +) { + const override = normalizeModelRef(overrideModel); + if (override) return override; + const fallback = + definition.kind === "video" + ? config.defaultVideoModel + : definition.kind === "multimodal" + ? config.defaultMultimodalModel + : config.defaultImageModel; + return normalizeModelRef(definition.model || fallback) || undefined; +} + +function getFunctionModelValue( + config: VideoCanvasFunctionConfig, + definition: VideoCanvasFunctionDefinition, ) { const fallback = definition.kind === "video" @@ -1188,8 +1229,57 @@ function getFunctionModel( : definition.kind === "multimodal" ? config.defaultMultimodalModel : config.defaultImageModel; - const model = (definition.model || fallback || "").trim(); - return model && model !== "auto" ? model : undefined; + return String(definition.model || fallback || "auto").trim() || "auto"; +} + +function getCanvasModelRef(model: CanvasModelOption) { + return String(model.modelId || model.id || "").trim(); +} + +function isCanvasImageModel(model: CanvasModelOption) { + return isImageGenerationModelDescriptor({ + providerType: model.providerType, + modelId: model.modelId, + capabilities: model.capabilities, + supportsImageGen: model.supportsImageGen, + }); +} + +function isCanvasVideoModel(model: CanvasModelOption) { + return isVideoModelDescriptor({ + providerType: model.providerType, + modelId: model.modelId, + capabilities: model.capabilities, + supportsImageGen: model.supportsImageGen, + }); +} + +function filterCanvasModelsByKind(models: CanvasModelOption[], kind: CanvasModelKind) { + const usable = models.filter((model) => getCanvasModelRef(model)); + let filtered = usable; + if (kind === "image") { + filtered = usable.filter(isCanvasImageModel); + } else if (kind === "video") { + filtered = usable.filter(isCanvasVideoModel); + } else if (kind === "multimodal") { + filtered = usable.filter( + (model) => model.isMultimodal || model.supportsVision || (!isCanvasImageModel(model) && !isCanvasVideoModel(model)), + ); + } + return filtered.length ? filtered : usable; +} + +function getCanvasModelDisplayName( + modelRef: string | undefined, + models: CanvasModelOption[], + fallback = "自动模型", +) { + const normalized = normalizeModelRef(modelRef); + if (!normalized) return fallback; + const match = models.find( + (model) => model.id === normalized || model.modelId === normalized || getCanvasModelRef(model) === normalized, + ); + return match?.displayName || normalized; } function composeFunctionPrompt( @@ -1782,10 +1872,18 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib "/api/video-canvas/config", fetcher ); + const { data: canvasModelsData } = useSWR<{ models?: CanvasModelOption[] }>( + "/api/models", + fetcher + ); const videoCanvasFunctionConfig = useMemo( () => normalizeVideoCanvasFunctionConfig(videoCanvasFunctionSettings?.config), [videoCanvasFunctionSettings?.config] ); + const canvasModelOptions = useMemo( + () => (canvasModelsData?.models || []).filter((model) => getCanvasModelRef(model)), + [canvasModelsData?.models] + ); const [nodes, setNodes] = useState([]); const [edges, setEdges] = useState([]); const [selectedNodeId, setSelectedNodeId] = useState(null); @@ -2879,7 +2977,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib const response = await imageGenerationService.generateImage({ aspectRatio: payload.settings.ratio, imageCount: count, - model: getFunctionModel(videoCanvasFunctionConfig, definition), + model: getFunctionModel(videoCanvasFunctionConfig, definition, payload.settings.model), prompt: finalPrompt, referenceImage: sourceUrl, }); @@ -2947,7 +3045,12 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib ); const runPresetComposerSubmit = useCallback( - async (nodeId: string, mode: Exclude, draftPrompt: string) => { + async ( + nodeId: string, + mode: Exclude, + draftPrompt: string, + settings?: ComposerSettings, + ) => { const node = nodes.find((item) => item.id === nodeId); if (!node) return; const incomingNodes = edges @@ -2977,7 +3080,8 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib imageUrls, mode: "agent-plan", nodeTitle: node.title, - prompt: finalPrompt, + prompt, + preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition, settings?.model), videoUrls, }); if (!response.success) throw new Error(response.error || "故事脚本生成失败"); @@ -3013,7 +3117,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib const prompt = composeFunctionPrompt(definition, draftPrompt); const response = await videoGenerationService.generateVideo({ images: imageUrls.slice(0, 4), - preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition), + preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition, settings?.model), prompt, ratio: "16:9", resolution: "720p", @@ -3120,6 +3224,11 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib .filter(Boolean); const action = String(node.draftValues?.action || ""); const params = (node.draftValues?.params || {}) as Record; + const paramsSettings = + params.settings && typeof params.settings === "object" + ? (params.settings as Record) + : {}; + const nodeModelRef = normalizeModelRef(params.model || paramsSettings.model); const rawPrompt = String(params.prompt || node.draftValues?.prompt || ""); try { @@ -3146,7 +3255,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib const response = await imageGenerationService.generateImage({ aspectRatio: action === "IMAGE_GENERATE_NINE_GRID" ? "16:9" : undefined, imageCount: 1, - model: getFunctionModel(videoCanvasFunctionConfig, definition), + model: getFunctionModel(videoCanvasFunctionConfig, definition, nodeModelRef), prompt, referenceImage: sourceUrl || undefined, }); @@ -3188,7 +3297,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib const prompt = composeFunctionPrompt(definition, rawPrompt || node.title); const response = await videoGenerationService.generateVideo({ images: imageUrls.slice(0, 4), - preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition), + preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition, nodeModelRef), prompt, ratio: "16:9", resolution: "720p", @@ -3232,6 +3341,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib mode: "agent-plan", nodeTitle: node.title, prompt, + preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition, nodeModelRef), videoUrls, }); if (!response.success) throw new Error(response.error || "多模态分析失败"); @@ -3644,14 +3754,17 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib submitSelectedImagePrompt(selectedPresetControlNode.id, payload)} onImageToolAction={(action, option) => runSelectedImageToolAction(selectedPresetControlNode.id, action, option)} - onPresetSubmit={(prompt) => + onPresetSubmit={(prompt, settings) => runPresetComposerSubmit( selectedPresetControlNode.id, selectedPresetMode ?? "character-three-view", - prompt + prompt, + settings ) } selectedNode={selectedPresetControlNode} @@ -3661,8 +3774,13 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib {selectedNode && selectedTargetModuleKind === "text" && selectedTargetAnchor ? ( { + onSubmit={(prompt, model) => { setNodes((current) => current.map((node) => node.id === selectedNode.id @@ -3675,6 +3793,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib manualTextComposer: true, params: { ...((node.draftValues?.params as Record | undefined) || {}), + model, prompt, }, }, @@ -4385,7 +4504,9 @@ function MagnetGlyph() { function PresetWorkflowControls({ anchor, + functionConfig, mode, + modelOptions, nodes, onComposerSubmit, onImageToolAction, @@ -4393,19 +4514,25 @@ function PresetWorkflowControls({ selectedNode, }: { anchor: PresetControlAnchor; + functionConfig: VideoCanvasFunctionConfig; mode: Exclude; + modelOptions: CanvasModelOption[]; nodes: CloneNode[]; onComposerSubmit: (payload: ComposerSubmitPayload) => void; onImageToolAction: (action: ImageToolbarAction, option?: string) => void; - onPresetSubmit: (prompt: string) => void; + onPresetSubmit: (prompt: string, settings?: ComposerSettings) => void; selectedNode: CloneNode; }) { if (mode === "character-three-view") { + const definition = getConfiguredFunction(functionConfig, "character-three-view"); return (
- +
); } function TextTargetModuleComposer({ anchor, + defaultModel = "auto", + modelOptions = [], node, onSubmit, }: { anchor: PresetControlAnchor; + defaultModel?: string; + modelOptions?: CanvasModelOption[]; node: CloneNode; - onSubmit: (prompt: string) => void; + onSubmit: (prompt: string, model: string) => void; }) { const contentValues = node.draftValues?.content; const initialPrompt = Array.isArray(contentValues) && typeof contentValues[0] === "string" ? contentValues[0] : ""; + const initialModel = normalizeModelRef( + ((node.draftValues?.params as Record | undefined)?.model as string | undefined) || + defaultModel, + ) || "auto"; + const [menu, setMenu] = useState(null); + const [selectedModel, setSelectedModel] = useState(initialModel); const [prompt, setPrompt] = useState(initialPrompt); useEffect(() => { setPrompt(initialPrompt); }, [initialPrompt, node.id]); + useEffect(() => { + setSelectedModel(initialModel); + }, [initialModel, node.id]); + return (
+ {menu === "model" ? ( + { + setSelectedModel(option); + setMenu(null); + }} + /> + ) : null}