From 82c9173467de9e840fe92530f8e9819d7b25c0e2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 5 Mar 2026 11:42:54 +0800 Subject: [PATCH] feat(video): add capability-driven video tools contract --- app/(chat)/api/models/route.ts | 6 + .../api/video-model-capabilities/route.ts | 47 ++ components/canvas/CanvasChatPanel.tsx | 91 +++- components/chat.tsx | 77 +++- components/multimodal-input.tsx | 30 +- components/video-params-bar.tsx | 423 +++++++++++------- lib/ai/video-tool-capability.ts | 322 +++++++++++++ types/video-tool-capability.ts | 45 ++ 8 files changed, 880 insertions(+), 161 deletions(-) create mode 100644 app/(chat)/api/video-model-capabilities/route.ts create mode 100644 lib/ai/video-tool-capability.ts create mode 100644 types/video-tool-capability.ts diff --git a/app/(chat)/api/models/route.ts b/app/(chat)/api/models/route.ts index ce9a0babe..34ea4776c 100644 --- a/app/(chat)/api/models/route.ts +++ b/app/(chat)/api/models/route.ts @@ -5,6 +5,7 @@ import { getEnabledAiModelsWithProvider, getUserById, } from "@/lib/db/queries"; +import { buildVideoToolCapability } from "@/lib/ai/video-tool-capability"; import { ChatSDKError } from "@/lib/errors"; import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags"; @@ -91,6 +92,11 @@ export async function GET() { isSubModel: model.isSubModel, parentModelId: model.parentModelId, capabilities: model.capabilities, // 🆕 返回子模型的能力配置 + videoToolCapability: buildVideoToolCapability({ + providerType: provider.type, + modelId: model.modelId, + rawCapabilities: model.capabilities, + }), imageGenConfig: model.imageGenConfig, sortOrder: model.sortOrder ?? 0, })); diff --git a/app/(chat)/api/video-model-capabilities/route.ts b/app/(chat)/api/video-model-capabilities/route.ts new file mode 100644 index 000000000..2c4400b7d --- /dev/null +++ b/app/(chat)/api/video-model-capabilities/route.ts @@ -0,0 +1,47 @@ +import { auth } from "@/app/(auth)/auth"; +import { getAiModelById, getAiProviderById } from "@/lib/db/queries"; +import { buildVideoToolCapability } from "@/lib/ai/video-tool-capability"; +import { ChatSDKError } from "@/lib/errors"; + +export async function GET(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) { + return new ChatSDKError("unauthorized:chat").toResponse(); + } + + const { searchParams } = new URL(request.url); + const modelDbId = searchParams.get("modelId"); + if (!modelDbId) { + return new ChatSDKError("bad_request:api", "modelId is required").toResponse(); + } + + const model = await getAiModelById({ id: modelDbId }); + if (!model || !model.isEnabled) { + return new ChatSDKError("bad_request:api", "Model not found or disabled").toResponse(); + } + + const provider = await getAiProviderById({ id: model.providerId }); + if (!provider || !provider.isEnabled) { + return new ChatSDKError("bad_request:api", "Provider not found or disabled").toResponse(); + } + + const capability = buildVideoToolCapability({ + providerType: provider.type, + modelId: model.modelId, + rawCapabilities: model.capabilities, + }); + + return Response.json({ + modelId: model.id, + modelName: model.displayName, + providerType: provider.type, + capability, + }); + } catch (error) { + if (error instanceof ChatSDKError) { + return error.toResponse(); + } + return new ChatSDKError("offline:chat").toResponse(); + } +} diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 3cab66513..21e621a07 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -57,6 +57,7 @@ import { getCanvasSkillIconComponent } from "@/components/canvas/canvas-skill-ic import { ImageParamsBar, type ImageGenerationParams } from "@/components/image-params-bar"; import { VideoParamsBar, type VideoGenerationParams } from "@/components/video-params-bar"; import { ImageGenerationLoading } from "@/components/image-generation-loading"; +import type { VideoToolCapabilityContract } from "@/types/video-tool-capability"; // 标签悬停预览信息 interface TagPreviewInfo { @@ -760,8 +761,70 @@ function extractCanvasImageAnalysisSection(content: string): CanvasAnalysisSecti function normalizeVideoParamsForProvider( params: VideoGenerationParams | null | undefined, - providerType?: string + providerType?: string, + capability?: VideoToolCapabilityContract | null ): VideoGenerationParams { + if (capability) { + const modeOptions = capability.modeOptions || []; + const ratioOptions = capability.ratioOptions || []; + const durationOptions = capability.durationOptions || []; + const resolutionOptions = capability.resolutionOptions || []; + const modeSet = new Set(modeOptions.map((item) => item.mode)); + const ratioSet = new Set(ratioOptions); + const durationSet = new Set(durationOptions); + const resolutionSet = new Set(resolutionOptions); + + const fallbackMode = capability.defaults?.mode || modeOptions[0]?.mode || "text-to-video"; + const fallbackRatio = capability.defaults?.ratio || ratioOptions[0] || "16:9"; + const fallbackDuration = capability.defaults?.duration || durationOptions[0] || 5; + const fallbackResolution = capability.defaults?.resolution || resolutionOptions[0] || "720p"; + + const merged = { + mode: fallbackMode, + resolution: fallbackResolution as VideoGenerationParams["resolution"], + ratio: fallbackRatio as VideoGenerationParams["ratio"], + duration: Number(fallbackDuration) || 5, + generateAudio: capability.defaults?.generateAudio ?? false, + watermark: false, + cameraFixed: false, + cameraMove: "none" as VideoGenerationParams["cameraMove"], + fps: capability.defaults?.fps || 24, + ...(params || {}), + } as VideoGenerationParams; + + if (!modeSet.has(merged.mode)) { + merged.mode = fallbackMode as VideoGenerationParams["mode"]; + } + if (!ratioSet.has(merged.ratio)) { + merged.ratio = fallbackRatio as VideoGenerationParams["ratio"]; + } + if (!durationSet.has(Number(merged.duration))) { + merged.duration = Number(fallbackDuration) || 5; + } else { + merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5; + } + if (resolutionOptions.length > 0 && !resolutionSet.has(merged.resolution)) { + merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; + } + if (!capability.ui.showGenerateAudio) { + merged.generateAudio = false; + } + if (!capability.ui.showFps) { + merged.fps = capability.defaults?.fps || 24; + } + if (!capability.ui.showCameraMove) { + merged.cameraMove = "none"; + merged.cameraFixed = false; + } + if (!capability.ui.showReturnLastFrame) { + merged.returnLastFrame = false; + } + if (!capability.ui.showSeed) { + merged.seed = undefined; + } + return merged; + } + const isFalVideo = providerType === "fal-video"; const isRuxaVideo = providerType === "ruxa-video"; const base: VideoGenerationParams = { @@ -814,11 +877,18 @@ function normalizeVideoParamsForProvider( function getVideoImageLimit( params: VideoGenerationParams | null | undefined, - providerType?: string + providerType?: string, + capability?: VideoToolCapabilityContract | null ): { min: number; max: number } { if (!params) { return { min: 0, max: 0 }; } + if (capability?.modeOptions?.length) { + const modeOption = capability.modeOptions.find((item) => item.mode === params.mode); + if (modeOption) { + return { min: modeOption.imageMin, max: modeOption.imageMax }; + } + } const isFalVideo = providerType === "fal-video"; const isRuxaVideo = providerType === "ruxa-video"; if (isRuxaVideo) { @@ -2253,7 +2323,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF providerType: string; supportsVision?: boolean; supportsImageGen?: boolean; - capabilities?: string[] | null; + capabilities?: any; + videoToolCapability?: VideoToolCapabilityContract | null; imageGenConfig?: { maxImages?: number; supportsSize?: boolean; @@ -2501,6 +2572,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF sortOrder: -999, providerType: "auto", supportsImageGen: true, + videoToolCapability: null, imageGenConfig: modelsData?.agentModel?.imageGenConfig || undefined, }; @@ -2768,11 +2840,15 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF useEffect(() => { if (!isVideoModel) return; - const normalized = normalizeVideoParamsForProvider(videoParams, currentModel?.providerType); + const normalized = normalizeVideoParamsForProvider( + videoParams, + currentModel?.providerType, + currentModel?.videoToolCapability + ); if (!videoParams || JSON.stringify(videoParams) !== JSON.stringify(normalized)) { setVideoParams(normalized); } - }, [isVideoModel, currentModel?.providerType, videoParams]); + }, [isVideoModel, currentModel?.providerType, currentModel?.videoToolCapability, videoParams]); useEffect(() => { if (!isVideoModel || videoFrameSlots.length === 0) return; @@ -5581,7 +5657,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF let effectiveAttachments = [...attachments]; const videoImageLimit = isVideoModel - ? getVideoImageLimit(videoParams, currentModel?.providerType) + ? getVideoImageLimit(videoParams, currentModel?.providerType, currentModel?.videoToolCapability) : null; if (videoImageLimit) { @@ -6157,7 +6233,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ? detailValidationIssues.length === 0 && !isDetailGenerating : hasInputOrAttachments; const currentVideoImageLimit = isVideoModel - ? getVideoImageLimit(videoParams, currentModel?.providerType) + ? getVideoImageLimit(videoParams, currentModel?.providerType, currentModel?.videoToolCapability) : null; const showVideoAttachmentButton = Boolean( isVideoModel && @@ -8134,6 +8210,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF onParamsChange={handleVideoParamsChange} inline={true} providerType={currentModel?.providerType} + capabilities={currentModel?.videoToolCapability} /> diff --git a/components/chat.tsx b/components/chat.tsx index 9ee7e77e3..f8049f21e 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -37,6 +37,7 @@ import { toast } from "sonner"; import type { VisibilityType } from "@/components/visibility-selector"; import type { VideoGenerationParams } from "@/components/video-params-bar"; import type { ImageGenerationParams } from "@/components/image-params-bar"; +import type { VideoToolCapabilityContract } from "@/types/video-tool-capability"; type ImageGenerationPhase = "idle" | "queued" | "processing" | "finalizing" | "failed"; @@ -68,8 +69,72 @@ function normalizeImageParamsForProvider( function normalizeVideoParamsForProvider( params: VideoGenerationParams | null | undefined, - providerType?: string + providerType?: string, + capability?: VideoToolCapabilityContract | null ): VideoGenerationParams { + if (capability) { + const modeOptions = capability.modeOptions || []; + const ratioOptions = capability.ratioOptions || []; + const durationOptions = capability.durationOptions || []; + const resolutionOptions = capability.resolutionOptions || []; + const modeSet = new Set(modeOptions.map((item) => item.mode)); + const ratioSet = new Set(ratioOptions); + const durationSet = new Set(durationOptions); + const resolutionSet = new Set(resolutionOptions); + + const fallbackMode = capability.defaults?.mode || modeOptions[0]?.mode || "text-to-video"; + const fallbackRatio = capability.defaults?.ratio || ratioOptions[0] || "16:9"; + const fallbackDuration = capability.defaults?.duration || durationOptions[0] || 5; + const fallbackResolution = capability.defaults?.resolution || resolutionOptions[0] || "720p"; + + const merged = { + mode: fallbackMode, + resolution: fallbackResolution as VideoGenerationParams["resolution"], + ratio: fallbackRatio as VideoGenerationParams["ratio"], + duration: Number(fallbackDuration) || 5, + generateAudio: capability.defaults?.generateAudio ?? false, + watermark: false, + cameraFixed: false, + cameraMove: "none" as VideoGenerationParams["cameraMove"], + fps: capability.defaults?.fps || 24, + ...(params || {}), + } as VideoGenerationParams; + + if (!modeSet.has(merged.mode)) { + merged.mode = fallbackMode as VideoGenerationParams["mode"]; + } + if (!ratioSet.has(merged.ratio)) { + merged.ratio = fallbackRatio as VideoGenerationParams["ratio"]; + } + if (!durationSet.has(Number(merged.duration))) { + merged.duration = Number(fallbackDuration) || 5; + } else { + merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5; + } + if (!resolutionSet.has(merged.resolution)) { + merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; + } + + if (!capability.ui.showGenerateAudio) { + merged.generateAudio = false; + } + if (!capability.ui.showFps) { + merged.fps = capability.defaults?.fps || 24; + } + if (!capability.ui.showCameraMove) { + merged.cameraMove = "none"; + merged.cameraFixed = false; + } + if (!capability.ui.showReturnLastFrame) { + merged.returnLastFrame = false; + } + if (!capability.ui.showSeed) { + merged.seed = undefined; + } + + return merged; + } + const isFalVideo = providerType === "fal-video"; const isRuxaVideo = providerType === "ruxa-video"; const base: VideoGenerationParams = { @@ -411,6 +476,7 @@ export function Chat({ providerType: string; supportsVision?: boolean; supportsImageGen?: boolean; + videoToolCapability?: VideoToolCapabilityContract | null; imageGenConfig?: { maxImages?: number; supportsSize?: boolean; @@ -473,12 +539,16 @@ export function Chat({ return; } - const normalized = normalizeVideoParamsForProvider(videoParams, currentModel?.providerType); + const normalized = normalizeVideoParamsForProvider( + videoParams, + currentModel?.providerType, + currentModel?.videoToolCapability + ); if (!videoParams || JSON.stringify(videoParams) !== JSON.stringify(normalized)) { setVideoParams(normalized); console.log("[Chat] 视频参数已按当前模型规范化:", normalized); } - }, [isVideoModel, videoParams, currentModel?.providerType]); + }, [isVideoModel, videoParams, currentModel?.providerType, currentModel?.videoToolCapability]); // 🆕 当切换到图片模型时，强制更新 imageParams（确保组件重新渲染） useEffect(() => { @@ -1984,6 +2054,7 @@ export function Chat({ videoModelName={currentModel?.displayName} videoParams={videoParams} videoProviderType={currentModel?.providerType} + videoToolCapability={currentModel?.videoToolCapability} onVideoParamsChange={(params) => { setVideoParams(params); console.log("[Chat] Video params updated:", params); diff --git a/components/multimodal-input.tsx b/components/multimodal-input.tsx index e4ee40233..602256395 100644 --- a/components/multimodal-input.tsx +++ b/components/multimodal-input.tsx @@ -19,6 +19,7 @@ type DynamicModel = { isSubModel?: boolean; // 🆕 是否为子模型 parentModelId?: string | null; // 🆕 父模型ID capabilities?: any; // 🆕 模型能力配置 + videoToolCapability?: VideoToolCapabilityContract | null; imageGenConfig?: { maxImages?: number; supportsSize?: boolean; @@ -128,6 +129,7 @@ import { Button } from "./ui/button"; import type { VisibilityType } from "./visibility-selector"; import { VideoParamsBar, type VideoGenerationParams } from "./video-params-bar"; import { ImageParamsBar, type ImageGenerationParams } from "./image-params-bar"; +import type { VideoToolCapabilityContract } from "@/types/video-tool-capability"; function PureMultimodalInput({ chatId, @@ -149,6 +151,7 @@ function PureMultimodalInput({ videoModelName, videoParams, videoProviderType, + videoToolCapability, onVideoParamsChange, isImageModel, imageModelName, @@ -176,6 +179,7 @@ function PureMultimodalInput({ videoModelName?: string; videoParams?: VideoGenerationParams | null; videoProviderType?: string; + videoToolCapability?: VideoToolCapabilityContract | null; onVideoParamsChange?: (params: VideoGenerationParams) => void; isImageModel?: boolean; imageModelName?: string; @@ -691,6 +695,26 @@ function PureMultimodalInput({ if (!isVideoModel || !videoParams) { return { max: Number.POSITIVE_INFINITY, label: "" as const }; } + if (videoToolCapability?.modeOptions?.length) { + const modeOption = videoToolCapability.modeOptions.find( + (item) => item.mode === videoParams.mode + ); + if (modeOption) { + if (modeOption.imageMax === 0) { + return { max: 0, label: `${modeOption.label}模式不支持图片` as const }; + } + if (modeOption.imageMin === modeOption.imageMax) { + return { + max: modeOption.imageMax, + label: `${modeOption.label}模式仅支持 ${modeOption.imageMax} 张图` as const, + }; + } + return { + max: modeOption.imageMax, + label: `${modeOption.label}模式最多 ${modeOption.imageMax} 张图` as const, + }; + } + } if (videoProviderType === "fal-video" || videoProviderType === "ruxa-video") { const providerLabel = videoProviderType === "ruxa-video" ? "RUXA" : "FAL"; switch (videoParams.mode) { @@ -713,7 +737,7 @@ function PureMultimodalInput({ default: return { max: Number.POSITIVE_INFINITY, label: "" as const }; } - }, [isVideoModel, videoParams, videoProviderType]); + }, [isVideoModel, videoParams, videoProviderType, videoToolCapability]); const enforceVideoImageLimit = useCallback( (incomingCount: number) => { @@ -1257,6 +1281,7 @@ function PureMultimodalInput({ onParamsChange={onVideoParamsChange} inline={true} providerType={videoProviderType} + capabilities={videoToolCapability} /> )} void; inline?: boolean; // 内联模式：只显示参数按钮，不显示外层容器 providerType?: string; + capabilities?: VideoToolCapabilityContract | null; } const MODE_LABELS: Record = { @@ -165,8 +167,69 @@ const RUXA_RATIO_SET = new Set(["16:9", "9:16", "1:1"]); function normalizeVideoParamsByProvider( source: Partial | undefined, - providerType?: string + providerType?: string, + capability?: VideoToolCapabilityContract | null ): VideoGenerationParams { + if (capability) { + const modes = capability.modeOptions.map((item) => item.mode); + const ratios = capability.ratioOptions; + const durations = capability.durationOptions; + const resolutions = capability.resolutionOptions; + + const fallbackMode = capability.defaults.mode || modes[0] || "text-to-video"; + const fallbackRatio = capability.defaults.ratio || ratios[0] || "16:9"; + const fallbackDuration = capability.defaults.duration || durations[0] || 5; + const fallbackResolution = + capability.defaults.resolution || resolutions[0] || "720p"; + + const merged = { + mode: fallbackMode, + resolution: fallbackResolution as VideoGenerationParams["resolution"], + ratio: fallbackRatio as VideoGenerationParams["ratio"], + duration: Number(fallbackDuration) || 5, + generateAudio: capability.defaults.generateAudio ?? false, + watermark: false, + cameraFixed: false, + cameraMove: "none" as const, + fps: capability.defaults.fps || 24, + ...source, + } as VideoGenerationParams; + + if (!modes.includes(merged.mode)) { + merged.mode = fallbackMode as VideoGenerationParams["mode"]; + } + if (!ratios.includes(merged.ratio)) { + merged.ratio = fallbackRatio as VideoGenerationParams["ratio"]; + } + if (!durations.includes(Number(merged.duration))) { + merged.duration = Number(fallbackDuration) || 5; + } else { + merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5; + } + if (resolutions.length > 0 && !resolutions.includes(merged.resolution)) { + merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; + } + + if (!capability.ui.showGenerateAudio) { + merged.generateAudio = false; + } + if (!capability.ui.showFps) { + merged.fps = capability.defaults.fps || 24; + } + if (!capability.ui.showCameraMove) { + merged.cameraMove = "none"; + merged.cameraFixed = false; + } + if (!capability.ui.showReturnLastFrame) { + merged.returnLastFrame = false; + } + if (!capability.ui.showSeed) { + merged.seed = undefined; + } + + return merged; + } + const isFalVideo = providerType === "fal-video"; const isRuxaVideo = providerType === "ruxa-video"; const base: VideoGenerationParams = { @@ -223,40 +286,70 @@ export function VideoParamsBar({ onParamsChange, inline = false, providerType = "volcano-video", + capabilities, }: VideoParamsBarProps) { const isFalVideo = providerType === "fal-video"; const isRuxaVideo = providerType === "ruxa-video"; - const modeOptions = isFalVideo + const capabilityModeOptions = capabilities?.modeOptions?.length + ? capabilities.modeOptions.map((item) => ({ + value: item.mode, + label: item.label || MODE_LABELS[item.mode], + })) + : null; + const modeOptions = capabilityModeOptions || (isFalVideo ? FAL_MODE_OPTIONS : isRuxaVideo ? RUXA_MODE_OPTIONS - : STANDARD_MODE_OPTIONS; - const ratioOptions = isFalVideo + : STANDARD_MODE_OPTIONS); + const capabilityRatioOptions = capabilities?.ratioOptions?.length + ? capabilities.ratioOptions.map((value) => ({ value, label: value })) + : null; + const ratioOptions = capabilityRatioOptions || (isFalVideo ? FAL_RATIO_OPTIONS : isRuxaVideo ? RUXA_RATIO_OPTIONS - : RATIO_OPTIONS; - const durationOptions = isFalVideo + : RATIO_OPTIONS); + const durationOptions = capabilities?.durationOptions?.length + ? capabilities.durationOptions + : (isFalVideo ? FAL_DURATION_OPTIONS : isRuxaVideo ? RUXA_DURATION_OPTIONS - : STANDARD_DURATION_OPTIONS; + : STANDARD_DURATION_OPTIONS); + const resolutionOptions = capabilities?.resolutionOptions?.length + ? capabilities.resolutionOptions.map((value) => ({ + value, + label: value.toUpperCase(), + })) + : RESOLUTION_OPTIONS; + const showResolutionControl = capabilities + ? capabilities.ui.showResolution && resolutionOptions.length > 0 + : !isFalVideo && !isRuxaVideo; + const showGenerateAudioControl = capabilities + ? capabilities.ui.showGenerateAudio + : isFalVideo; + const showCameraMoveControl = capabilities + ? capabilities.ui.showCameraMove + : !isFalVideo && !isRuxaVideo; + const showFpsControl = capabilities ? capabilities.ui.showFps : !isFalVideo && !isRuxaVideo; + const showSeedControl = capabilities ? capabilities.ui.showSeed : true; + const showReturnLastFrameControl = capabilities ? capabilities.ui.showReturnLastFrame : true; const [params, setParams] = useState(() => - normalizeVideoParamsByProvider(initialParams, providerType) + normalizeVideoParamsByProvider(initialParams, providerType, capabilities) ); const [collapsed, setCollapsed] = useState(false); useEffect(() => { - const normalized = normalizeVideoParamsByProvider(initialParams, providerType); + const normalized = normalizeVideoParamsByProvider(initialParams, providerType, capabilities); setParams((prev) => { if (JSON.stringify(prev) === JSON.stringify(normalized)) { return prev; } return normalized; }); - }, [initialParams, providerType]); + }, [initialParams, providerType, capabilities]); // 获取当前运镜模式的标签 const getCurrentCameraLabel = () => { @@ -269,6 +362,23 @@ export function VideoParamsBar({ // 🆕 根据模式获取图片上传配置（根据火山引擎官方文档） const getImageUploadConfig = () => { + if (capabilities?.modeOptions?.length) { + const modeOption = capabilities.modeOptions.find((item) => item.mode === params.mode); + if (!modeOption) { + return { min: 0, max: 0, label: "" }; + } + const label = + modeOption.imageMax === 0 + ? `${modeOption.label}（无需图片）` + : modeOption.imageMin === modeOption.imageMax + ? `${modeOption.label}（${modeOption.imageMax} 张）` + : `${modeOption.label}（${modeOption.imageMin}-${modeOption.imageMax} 张）`; + return { + min: modeOption.imageMin, + max: modeOption.imageMax, + label, + }; + } if (isRuxaVideo) { return { min: 1, max: 1, label: "图生视频（1 张起始图）" }; } @@ -373,7 +483,7 @@ export function VideoParamsBar({ ? "border-white bg-white text-[#1f2937] shadow-[0_1px_5px_rgba(15,23,42,0.08)]" : "border-transparent text-[#5f6b7a] hover:bg-white/70" )} - onClick={() => updateParam("ratio", option.value)} + onClick={() => updateParam("ratio", option.value as VideoGenerationParams["ratio"])} > {option.label} @@ -382,11 +492,11 @@ export function VideoParamsBar({ {/* 分辨率（非 fal） */} - {!isFalVideo && !isRuxaVideo && ( + {showResolutionControl && (

选择分辨率

- {RESOLUTION_OPTIONS.map((option) => ( + {resolutionOptions.map((option) => (
- ) : isRuxaVideo ? ( + ) : !showFpsControl && !showCameraMoveControl ? (
当前模型支持图生视频（1 张起始图），已自动按官方参数约束处理。
) : ( <> - {/* 帧率 */} -
-

帧率

-
- {FPS_OPTIONS.map((option) => ( - - ))} + {showFpsControl && ( +
+

帧率

+
+ {FPS_OPTIONS.map((option) => ( + + ))} +
-
+ )} - {/* 运镜 */} -
-

运镜模式

-
- {CAMERA_MOVE_OPTIONS.map((option) => ( - - ))} + {showCameraMoveControl && ( +
+

运镜模式

+
+ {CAMERA_MOVE_OPTIONS.map((option) => ( + + ))} +
-
+ )} )} {/* 高级选项 */}
-
- 返回尾帧 - updateParam("returnLastFrame", checked)} - /> -
- {isFalVideo && ( + {showReturnLastFrameControl && ( +
+ 返回尾帧 + updateParam("returnLastFrame", checked)} + /> +
+ )} + {showGenerateAudioControl && (
生成音频
)} -
- 随机种子 - updateParam("seed", Number(e.target.value) || undefined)} - className="h-7 rounded-[8px] border-[#dce1e8] bg-[#eef1f4]/90 text-[12px] text-[#2f3b48] placeholder:text-[#b0b8c4]" - /> -
+ {showSeedControl && ( +
+ 随机种子 + updateParam("seed", Number(e.target.value) || undefined)} + className="h-7 rounded-[8px] border-[#dce1e8] bg-[#eef1f4]/90 text-[12px] text-[#2f3b48] placeholder:text-[#b0b8c4]" + /> +
+ )}
@@ -689,7 +804,7 @@ export function VideoParamsBar({ variant={params.ratio === option.value ? "secondary" : "ghost"} size="sm" className="h-7 text-xs" - onClick={() => updateParam("ratio", option.value)} + onClick={() => updateParam("ratio", option.value as VideoGenerationParams["ratio"])} > {option.label} @@ -698,7 +813,7 @@ export function VideoParamsBar({ - {!isFalVideo && !isRuxaVideo && ( + {showResolutionControl && ( - {RESOLUTION_OPTIONS.map((option) => ( + {resolutionOptions.map((option) => ( updateParam("resolution", option.value)}> {option.label} @@ -734,7 +849,7 @@ export function VideoParamsBar({ - {isFalVideo ? ( + {showGenerateAudioControl ? ( - ) : isRuxaVideo ? ( + ) : !showFpsControl && !showCameraMoveControl ? ( - - - {FPS_OPTIONS.map((option) => ( - updateParam("fps", option.value as VideoGenerationParams["fps"])} - > - - {option.label} - - ))} - - + {showFpsControl && ( + + + + + + {FPS_OPTIONS.map((option) => ( + updateParam("fps", option.value as VideoGenerationParams["fps"])} + > + + {option.label} + + ))} + + + )} - - - - - - {CAMERA_MOVE_OPTIONS.map((option) => ( - { - updateParam("cameraMove", option.value); - updateParam("cameraFixed", option.value === "static"); - }} - > - - {option.label} - - ))} - - + {showCameraMoveControl && ( + + + + + + {CAMERA_MOVE_OPTIONS.map((option) => ( + { + updateParam("cameraMove", option.value); + updateParam("cameraFixed", option.value === "static"); + }} + > + + {option.label} + + ))} + + + )} )}
@@ -820,15 +939,17 @@ export function VideoParamsBar({ 高级设置
-
- - updateParam("returnLastFrame", checked)} - className="scale-75" - /> -
- {isFalVideo && ( + {showReturnLastFrameControl && ( +
+ + updateParam("returnLastFrame", checked)} + className="scale-75" + /> +
+ )} + {showGenerateAudioControl && (
)} -
- - - updateParam("seed", Number(e.target.value) || undefined) - } - className="h-8 text-xs" - /> -
+ {showSeedControl && ( +
+ + + updateParam("seed", Number(e.target.value) || undefined) + } + className="h-8 text-xs" + /> +
+ )}
diff --git a/lib/ai/video-tool-capability.ts b/lib/ai/video-tool-capability.ts new file mode 100644 index 000000000..a6686893a --- /dev/null +++ b/lib/ai/video-tool-capability.ts @@ -0,0 +1,322 @@ +import type { VideoModelCapabilities } from "@/types/ai"; +import type { + VideoToolCapabilityContract, + VideoToolMode, + VideoToolModeOption, + VideoToolResolution, +} from "@/types/video-tool-capability"; + +const MODE_LABELS: Record = { + "text-to-video": "文生视频", + "first-frame": "主体参考", + "first-last-frame": "首尾帧", + "reference-images": "智能多帧", +}; + +const MODE_ORDER: VideoToolMode[] = [ + "text-to-video", + "first-last-frame", + "reference-images", + "first-frame", +]; + +const STANDARD_RATIOS = ["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"]; +const PORTRAIT_RATIOS = ["16:9", "9:16", "1:1"]; +const STANDARD_RESOLUTIONS: VideoToolResolution[] = ["480p", "720p", "1080p"]; +const STANDARD_DURATIONS = [5, 10]; + +function asObject(value: unknown) { + return value && typeof value === "object" ? (value as Record) : null; +} + +function normalizeDurations(min?: number, max?: number) { + const safeMin = Number.isFinite(min) ? Math.max(1, Math.round(Number(min))) : undefined; + const safeMax = Number.isFinite(max) ? Math.max(1, Math.round(Number(max))) : undefined; + if (!safeMin || !safeMax || safeMax < safeMin) { + return [...STANDARD_DURATIONS]; + } + + if (safeMax - safeMin <= 12) { + return Array.from({ length: safeMax - safeMin + 1 }, (_, index) => safeMin + index); + } + return [safeMin, safeMax]; +} + +function normalizeModeOptions( + modes: VideoToolMode[], + maxReferenceImages = 4 +): VideoToolModeOption[] { + return MODE_ORDER.filter((mode) => modes.includes(mode)).map((mode) => { + if (mode === "text-to-video") { + return { mode, label: MODE_LABELS[mode], imageMin: 0, imageMax: 0 }; + } + if (mode === "first-frame") { + return { mode, label: MODE_LABELS[mode], imageMin: 1, imageMax: 1 }; + } + if (mode === "first-last-frame") { + return { mode, label: MODE_LABELS[mode], imageMin: 2, imageMax: 2 }; + } + return { + mode, + label: MODE_LABELS[mode], + imageMin: 1, + imageMax: Math.max(1, Math.min(8, maxReferenceImages)), + }; + }); +} + +function normalizeVideoCapabilities(rawCapabilities: unknown): Partial { + const raw = asObject(rawCapabilities); + if (!raw) return {}; + + const supportedModes = Array.isArray(raw.supportedModes) + ? raw.supportedModes.filter((item): item is VideoToolMode => + item === "text-to-video" || + item === "first-frame" || + item === "first-last-frame" || + item === "reference-images" + ) + : undefined; + + const supportedRatios = Array.isArray(raw.supportedRatios) + ? raw.supportedRatios + .map((item) => String(item || "").trim()) + .filter(Boolean) + : undefined; + + const supportedResolutions = Array.isArray(raw.supportedResolutions) + ? raw.supportedResolutions.filter( + (item): item is VideoToolResolution => item === "480p" || item === "720p" || item === "1080p" + ) + : undefined; + + const durationRange = asObject(raw.durationRange); + const min = Number(durationRange?.min); + const max = Number(durationRange?.max); + + return { + supportedModes, + supportedRatios: supportedRatios as any, + supportedResolutions: supportedResolutions as any, + durationRange: + Number.isFinite(min) && Number.isFinite(max) + ? { min: Math.round(min), max: Math.round(max) } + : undefined, + maxReferenceImages: Number.isFinite(Number(raw.maxReferenceImages)) + ? Math.round(Number(raw.maxReferenceImages)) + : undefined, + supportReturnLastFrame: + typeof raw.supportReturnLastFrame === "boolean" ? raw.supportReturnLastFrame : undefined, + supportSeed: typeof raw.supportSeed === "boolean" ? raw.supportSeed : undefined, + }; +} + +function createProviderDefaultCapability(providerType: string): VideoToolCapabilityContract { + if (providerType === "fal-video") { + return { + schemaVersion: "v1", + source: "provider-default", + providerType, + modeOptions: normalizeModeOptions(["first-frame", "first-last-frame"]), + ratioOptions: [...PORTRAIT_RATIOS], + durationOptions: Array.from({ length: 13 }, (_, index) => index + 3), + resolutionOptions: [], + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: true, + showReturnLastFrame: true, + showSeed: true, + }, + defaults: { + mode: "first-frame", + ratio: "16:9", + duration: 5, + generateAudio: true, + fps: 24, + }, + }; + } + + if (providerType === "ruxa-video") { + return { + schemaVersion: "v1", + source: "provider-default", + providerType, + modeOptions: normalizeModeOptions(["first-frame"]), + ratioOptions: [...PORTRAIT_RATIOS], + durationOptions: [5, 10], + resolutionOptions: [], + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: false, + showReturnLastFrame: false, + showSeed: false, + }, + defaults: { + mode: "first-frame", + ratio: "16:9", + duration: 10, + generateAudio: false, + fps: 24, + }, + }; + } + + if (providerType === "qianwen-video") { + return { + schemaVersion: "v1", + source: "provider-default", + providerType, + modeOptions: normalizeModeOptions(["first-frame"]), + ratioOptions: ["16:9"], + durationOptions: [5], + resolutionOptions: [], + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: false, + showReturnLastFrame: false, + showSeed: false, + }, + defaults: { + mode: "first-frame", + ratio: "16:9", + duration: 5, + generateAudio: false, + fps: 24, + }, + }; + } + + return { + schemaVersion: "v1", + source: "provider-default", + providerType, + modeOptions: normalizeModeOptions([ + "text-to-video", + "first-last-frame", + "reference-images", + "first-frame", + ]), + ratioOptions: [...STANDARD_RATIOS], + durationOptions: [...STANDARD_DURATIONS], + resolutionOptions: [...STANDARD_RESOLUTIONS], + ui: { + showResolution: true, + showFps: true, + showCameraMove: true, + showGenerateAudio: false, + showReturnLastFrame: true, + showSeed: true, + }, + defaults: { + mode: "text-to-video", + ratio: "16:9", + duration: 5, + resolution: "720p", + generateAudio: false, + fps: 24, + }, + }; +} + +function pickPreferredRatio(ratioOptions: string[]) { + if (ratioOptions.includes("16:9")) return "16:9"; + return ratioOptions[0] || "16:9"; +} + +function pickPreferredDuration(durationOptions: number[]) { + if (durationOptions.includes(5)) return 5; + if (durationOptions.includes(10)) return 10; + return durationOptions[0] || 5; +} + +function pickPreferredMode(modeOptions: VideoToolModeOption[], fallback: VideoToolMode) { + const modeSet = new Set(modeOptions.map((item) => item.mode)); + if (modeSet.has(fallback)) return fallback; + if (modeSet.has("first-frame")) return "first-frame"; + if (modeSet.has("text-to-video")) return "text-to-video"; + return modeOptions[0]?.mode || fallback; +} + +export function buildVideoToolCapability(params: { + providerType?: string | null; + modelId?: string | null; + rawCapabilities?: unknown; +}): VideoToolCapabilityContract | null { + const providerType = String(params.providerType || "").trim(); + if (!providerType) return null; + const base = createProviderDefaultCapability(providerType); + + if (providerType === "fal-video" || providerType === "qianwen-video" || providerType === "ruxa-video") { + return { + ...base, + modelId: params.modelId || undefined, + }; + } + + const parsed = normalizeVideoCapabilities(params.rawCapabilities); + const hasModelCapability = + Boolean(parsed.supportedModes?.length) || + Boolean(parsed.supportedRatios?.length) || + Boolean(parsed.supportedResolutions?.length) || + Boolean(parsed.durationRange); + + if (!hasModelCapability) { + return { + ...base, + modelId: params.modelId || undefined, + }; + } + + const supportedModes = + parsed.supportedModes && parsed.supportedModes.length > 0 + ? parsed.supportedModes + : base.modeOptions.map((item) => item.mode); + + const modeOptions = normalizeModeOptions(supportedModes, parsed.maxReferenceImages || 4); + const ratioOptions = + parsed.supportedRatios && parsed.supportedRatios.length > 0 + ? Array.from(new Set(parsed.supportedRatios)) + : base.ratioOptions; + const resolutionOptions = + parsed.supportedResolutions && parsed.supportedResolutions.length > 0 + ? parsed.supportedResolutions + : base.resolutionOptions; + const durationOptions = parsed.durationRange + ? normalizeDurations(parsed.durationRange.min, parsed.durationRange.max) + : base.durationOptions; + + return { + ...base, + source: "model-capabilities", + modelId: params.modelId || undefined, + modeOptions, + ratioOptions, + resolutionOptions, + durationOptions, + ui: { + ...base.ui, + showReturnLastFrame: + typeof parsed.supportReturnLastFrame === "boolean" + ? parsed.supportReturnLastFrame + : base.ui.showReturnLastFrame, + showSeed: + typeof parsed.supportSeed === "boolean" ? parsed.supportSeed : base.ui.showSeed, + }, + defaults: { + ...base.defaults, + mode: pickPreferredMode(modeOptions, base.defaults.mode), + ratio: pickPreferredRatio(ratioOptions), + duration: pickPreferredDuration(durationOptions), + resolution: resolutionOptions.includes("720p") + ? "720p" + : resolutionOptions[0] || undefined, + }, + }; +} diff --git a/types/video-tool-capability.ts b/types/video-tool-capability.ts new file mode 100644 index 000000000..61e33bc5b --- /dev/null +++ b/types/video-tool-capability.ts @@ -0,0 +1,45 @@ +export type VideoToolMode = + | "text-to-video" + | "first-frame" + | "first-last-frame" + | "reference-images"; + +export type VideoToolResolution = "480p" | "720p" | "1080p"; + +export interface VideoToolModeOption { + mode: VideoToolMode; + label: string; + imageMin: number; + imageMax: number; +} + +export interface VideoToolUiToggles { + showResolution: boolean; + showFps: boolean; + showCameraMove: boolean; + showGenerateAudio: boolean; + showReturnLastFrame: boolean; + showSeed: boolean; +} + +export interface VideoToolDefaults { + mode: VideoToolMode; + ratio: string; + duration: number; + resolution?: VideoToolResolution; + generateAudio?: boolean; + fps?: number; +} + +export interface VideoToolCapabilityContract { + schemaVersion: "v1"; + source: "provider-default" | "model-capabilities"; + providerType: string; + modelId?: string; + modeOptions: VideoToolModeOption[]; + ratioOptions: string[]; + durationOptions: number[]; + resolutionOptions: VideoToolResolution[]; + ui: VideoToolUiToggles; + defaults: VideoToolDefaults; +}