fix: align runway video params with model capabilities
This commit is contained in:
Your Name
parent
fc88bbb3d0
commit
dbfc8e97d2
@ -8922,6 +8922,10 @@ export async function POST(request: Request) {
|
||||
videoParams: effectiveVideoParams,
|
||||
});
|
||||
|
||||
if (result?.finishReason === "error") {
|
||||
throw new Error(result.text || "视频生成失败");
|
||||
}
|
||||
|
||||
videoMarkdown = result.text || "视频生成失败";
|
||||
tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || "";
|
||||
}
|
||||
|
||||
@ -51,6 +51,7 @@ import {
|
||||
isImageGenerationModelDescriptor,
|
||||
isVideoModelDescriptor,
|
||||
} from "@/lib/ai/model-classification";
|
||||
import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability";
|
||||
import type { Attachment, ChatMessage } from "@/lib/types";
|
||||
import type {
|
||||
AgentPreset,
|
||||
@ -893,24 +894,14 @@ function normalizeVideoParamsForProvider(
|
||||
): VideoGenerationParams {
|
||||
if (capability) {
|
||||
const modeOptions = capability.modeOptions || [];
|
||||
const ratioOptions = capability.ratioOptions || [];
|
||||
const durationOptions = capability.durationOptions || [];
|
||||
const resolutionOptions = capability.resolutionOptions || [];
|
||||
const modeSet = new Set(modeOptions.map((item) => item.mode));
|
||||
const ratioSet = new Set(ratioOptions);
|
||||
const durationSet = new Set(durationOptions);
|
||||
const resolutionSet = new Set(resolutionOptions);
|
||||
|
||||
const fallbackMode = capability.defaults?.mode || modeOptions[0]?.mode || "text-to-video";
|
||||
const fallbackRatio = capability.defaults?.ratio || ratioOptions[0] || "16:9";
|
||||
const fallbackDuration = capability.defaults?.duration || durationOptions[0] || 5;
|
||||
const fallbackResolution = capability.defaults?.resolution || resolutionOptions[0] || "720p";
|
||||
|
||||
const merged = {
|
||||
mode: fallbackMode,
|
||||
resolution: fallbackResolution as VideoGenerationParams["resolution"],
|
||||
ratio: fallbackRatio as VideoGenerationParams["ratio"],
|
||||
duration: Number(fallbackDuration) || 5,
|
||||
resolution: (capability.defaults?.resolution || "720p") as VideoGenerationParams["resolution"],
|
||||
ratio: (capability.defaults?.ratio || "16:9") as VideoGenerationParams["ratio"],
|
||||
duration: Number(capability.defaults?.duration) || 5,
|
||||
generateAudio: capability.defaults?.generateAudio ?? false,
|
||||
watermark: false,
|
||||
cameraFixed: false,
|
||||
@ -922,16 +913,39 @@ function normalizeVideoParamsForProvider(
|
||||
if (!modeSet.has(merged.mode)) {
|
||||
merged.mode = fallbackMode as VideoGenerationParams["mode"];
|
||||
}
|
||||
|
||||
const scopedOptions = getVideoToolOptionsForMode(capability, merged.mode);
|
||||
const ratioOptions = scopedOptions.ratioOptions || [];
|
||||
const durationOptions = scopedOptions.durationOptions || [];
|
||||
const resolutionOptions = scopedOptions.resolutionOptions || [];
|
||||
const ratioSet = new Set(ratioOptions);
|
||||
const durationSet = new Set(durationOptions);
|
||||
const resolutionSet = new Set(resolutionOptions);
|
||||
const fallbackRatio =
|
||||
scopedOptions.defaults?.ratio || ratioOptions[0] || capability.defaults?.ratio || "16:9";
|
||||
const fallbackDuration =
|
||||
Number(scopedOptions.defaults?.duration) ||
|
||||
durationOptions[0] ||
|
||||
Number(capability.defaults?.duration) ||
|
||||
5;
|
||||
const fallbackResolution =
|
||||
scopedOptions.defaults?.resolution ||
|
||||
resolutionOptions[0] ||
|
||||
capability.defaults?.resolution ||
|
||||
"720p";
|
||||
|
||||
if (!ratioSet.has(merged.ratio)) {
|
||||
merged.ratio = fallbackRatio as VideoGenerationParams["ratio"];
|
||||
}
|
||||
if (!durationSet.has(Number(merged.duration))) {
|
||||
merged.duration = Number(fallbackDuration) || 5;
|
||||
merged.duration = fallbackDuration;
|
||||
} else {
|
||||
merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5;
|
||||
merged.duration = Number(merged.duration) || fallbackDuration;
|
||||
}
|
||||
if (resolutionOptions.length > 0 && !resolutionSet.has(merged.resolution)) {
|
||||
merged.resolution = fallbackResolution as VideoGenerationParams["resolution"];
|
||||
} else if (resolutionOptions.length === 0) {
|
||||
merged.resolution = fallbackResolution as VideoGenerationParams["resolution"];
|
||||
}
|
||||
if (!capability.ui.showGenerateAudio) {
|
||||
merged.generateAudio = false;
|
||||
|
||||
@ -32,6 +32,7 @@ import {
|
||||
isImageGenerationModelDescriptor,
|
||||
isVideoModelDescriptor,
|
||||
} from "@/lib/ai/model-classification";
|
||||
import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability";
|
||||
import { fetcher, fetchWithErrorHandlers, generateUUID } from "@/lib/utils";
|
||||
import { Artifact } from "@/components/artifact";
|
||||
import { ProtocolDebugPanel } from "@/components/protocol-debug-panel";
|
||||
@ -85,24 +86,14 @@ function normalizeVideoParamsForProvider(
|
||||
): VideoGenerationParams {
|
||||
if (capability) {
|
||||
const modeOptions = capability.modeOptions || [];
|
||||
const ratioOptions = capability.ratioOptions || [];
|
||||
const durationOptions = capability.durationOptions || [];
|
||||
const resolutionOptions = capability.resolutionOptions || [];
|
||||
const modeSet = new Set(modeOptions.map((item) => item.mode));
|
||||
const ratioSet = new Set(ratioOptions);
|
||||
const durationSet = new Set(durationOptions);
|
||||
const resolutionSet = new Set(resolutionOptions);
|
||||
|
||||
const fallbackMode = capability.defaults?.mode || modeOptions[0]?.mode || "text-to-video";
|
||||
const fallbackRatio = capability.defaults?.ratio || ratioOptions[0] || "16:9";
|
||||
const fallbackDuration = capability.defaults?.duration || durationOptions[0] || 5;
|
||||
const fallbackResolution = capability.defaults?.resolution || resolutionOptions[0] || "720p";
|
||||
|
||||
const merged = {
|
||||
mode: fallbackMode,
|
||||
resolution: fallbackResolution as VideoGenerationParams["resolution"],
|
||||
ratio: fallbackRatio as VideoGenerationParams["ratio"],
|
||||
duration: Number(fallbackDuration) || 5,
|
||||
resolution: (capability.defaults?.resolution || "720p") as VideoGenerationParams["resolution"],
|
||||
ratio: (capability.defaults?.ratio || "16:9") as VideoGenerationParams["ratio"],
|
||||
duration: Number(capability.defaults?.duration) || 5,
|
||||
generateAudio: capability.defaults?.generateAudio ?? false,
|
||||
watermark: false,
|
||||
cameraFixed: false,
|
||||
@ -114,15 +105,38 @@ function normalizeVideoParamsForProvider(
|
||||
if (!modeSet.has(merged.mode)) {
|
||||
merged.mode = fallbackMode as VideoGenerationParams["mode"];
|
||||
}
|
||||
|
||||
const scopedOptions = getVideoToolOptionsForMode(capability, merged.mode);
|
||||
const ratioOptions = scopedOptions.ratioOptions || [];
|
||||
const durationOptions = scopedOptions.durationOptions || [];
|
||||
const resolutionOptions = scopedOptions.resolutionOptions || [];
|
||||
const ratioSet = new Set(ratioOptions);
|
||||
const durationSet = new Set(durationOptions);
|
||||
const resolutionSet = new Set(resolutionOptions);
|
||||
const fallbackRatio =
|
||||
scopedOptions.defaults?.ratio || ratioOptions[0] || capability.defaults?.ratio || "16:9";
|
||||
const fallbackDuration =
|
||||
Number(scopedOptions.defaults?.duration) ||
|
||||
durationOptions[0] ||
|
||||
Number(capability.defaults?.duration) ||
|
||||
5;
|
||||
const fallbackResolution =
|
||||
scopedOptions.defaults?.resolution ||
|
||||
resolutionOptions[0] ||
|
||||
capability.defaults?.resolution ||
|
||||
"720p";
|
||||
|
||||
if (!ratioSet.has(merged.ratio)) {
|
||||
merged.ratio = fallbackRatio as VideoGenerationParams["ratio"];
|
||||
}
|
||||
if (!durationSet.has(Number(merged.duration))) {
|
||||
merged.duration = Number(fallbackDuration) || 5;
|
||||
merged.duration = fallbackDuration;
|
||||
} else {
|
||||
merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5;
|
||||
merged.duration = Number(merged.duration) || fallbackDuration;
|
||||
}
|
||||
if (!resolutionSet.has(merged.resolution)) {
|
||||
if (resolutionOptions.length > 0 && !resolutionSet.has(merged.resolution)) {
|
||||
merged.resolution = fallbackResolution as VideoGenerationParams["resolution"];
|
||||
} else if (resolutionOptions.length === 0) {
|
||||
merged.resolution = fallbackResolution as VideoGenerationParams["resolution"];
|
||||
}
|
||||
|
||||
|
||||
@ -40,6 +40,7 @@ import {
|
||||
Volume2,
|
||||
VolumeX,
|
||||
} from "lucide-react";
|
||||
import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { VideoToolCapabilityContract } from "@/types/video-tool-capability";
|
||||
|
||||
@ -172,21 +173,13 @@ function normalizeVideoParamsByProvider(
|
||||
): VideoGenerationParams {
|
||||
if (capability) {
|
||||
const modes = capability.modeOptions.map((item) => item.mode);
|
||||
const ratios = capability.ratioOptions;
|
||||
const durations = capability.durationOptions;
|
||||
const resolutions = capability.resolutionOptions;
|
||||
|
||||
const fallbackMode = capability.defaults.mode || modes[0] || "text-to-video";
|
||||
const fallbackRatio = capability.defaults.ratio || ratios[0] || "16:9";
|
||||
const fallbackDuration = capability.defaults.duration || durations[0] || 5;
|
||||
const fallbackResolution =
|
||||
capability.defaults.resolution || resolutions[0] || "720p";
|
||||
|
||||
const merged = {
|
||||
mode: fallbackMode,
|
||||
resolution: fallbackResolution as VideoGenerationParams["resolution"],
|
||||
ratio: fallbackRatio as VideoGenerationParams["ratio"],
|
||||
duration: Number(fallbackDuration) || 5,
|
||||
resolution: (capability.defaults.resolution || "720p") as VideoGenerationParams["resolution"],
|
||||
ratio: (capability.defaults.ratio || "16:9") as VideoGenerationParams["ratio"],
|
||||
duration: Number(capability.defaults.duration) || 5,
|
||||
generateAudio: capability.defaults.generateAudio ?? false,
|
||||
watermark: false,
|
||||
cameraFixed: false,
|
||||
@ -198,20 +191,39 @@ function normalizeVideoParamsByProvider(
|
||||
if (!modes.includes(merged.mode)) {
|
||||
merged.mode = fallbackMode as VideoGenerationParams["mode"];
|
||||
}
|
||||
|
||||
const scopedOptions = getVideoToolOptionsForMode(capability, merged.mode);
|
||||
const ratios = scopedOptions.ratioOptions;
|
||||
const durations = scopedOptions.durationOptions;
|
||||
const resolutions = scopedOptions.resolutionOptions;
|
||||
const fallbackRatio =
|
||||
scopedOptions.defaults?.ratio || ratios[0] || capability.defaults.ratio || "16:9";
|
||||
const fallbackDuration =
|
||||
Number(scopedOptions.defaults?.duration) || durations[0] || Number(capability.defaults.duration) || 5;
|
||||
const fallbackResolution =
|
||||
scopedOptions.defaults?.resolution ||
|
||||
resolutions[0] ||
|
||||
capability.defaults.resolution ||
|
||||
"720p";
|
||||
|
||||
if (!ratios.includes(merged.ratio)) {
|
||||
merged.ratio = fallbackRatio as VideoGenerationParams["ratio"];
|
||||
}
|
||||
if (!durations.includes(Number(merged.duration))) {
|
||||
merged.duration = Number(fallbackDuration) || 5;
|
||||
merged.duration = fallbackDuration;
|
||||
} else {
|
||||
merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5;
|
||||
merged.duration = Number(merged.duration) || fallbackDuration;
|
||||
}
|
||||
if (resolutions.length > 0 && !resolutions.includes(merged.resolution)) {
|
||||
merged.resolution = fallbackResolution as VideoGenerationParams["resolution"];
|
||||
} else if (resolutions.length === 0) {
|
||||
merged.resolution = fallbackResolution as VideoGenerationParams["resolution"];
|
||||
}
|
||||
|
||||
if (!capability.ui.showGenerateAudio) {
|
||||
merged.generateAudio = false;
|
||||
} else if (typeof merged.generateAudio !== "boolean") {
|
||||
merged.generateAudio = scopedOptions.defaults?.generateAudio ?? capability.defaults.generateAudio ?? false;
|
||||
}
|
||||
if (!capability.ui.showFps) {
|
||||
merged.fps = capability.defaults.fps || 24;
|
||||
@ -290,6 +302,7 @@ export function VideoParamsBar({
|
||||
}: VideoParamsBarProps) {
|
||||
const isFalVideo = providerType === "fal-video";
|
||||
const isRuxaVideo = providerType === "ruxa-video";
|
||||
|
||||
const capabilityModeOptions = capabilities?.modeOptions?.length
|
||||
? capabilities.modeOptions.map((item) => ({
|
||||
value: item.mode,
|
||||
@ -301,30 +314,6 @@ export function VideoParamsBar({
|
||||
: isRuxaVideo
|
||||
? RUXA_MODE_OPTIONS
|
||||
: STANDARD_MODE_OPTIONS);
|
||||
const capabilityRatioOptions = capabilities?.ratioOptions?.length
|
||||
? capabilities.ratioOptions.map((value) => ({ value, label: value }))
|
||||
: null;
|
||||
const ratioOptions = capabilityRatioOptions || (isFalVideo
|
||||
? FAL_RATIO_OPTIONS
|
||||
: isRuxaVideo
|
||||
? RUXA_RATIO_OPTIONS
|
||||
: RATIO_OPTIONS);
|
||||
const durationOptions = capabilities?.durationOptions?.length
|
||||
? capabilities.durationOptions
|
||||
: (isFalVideo
|
||||
? FAL_DURATION_OPTIONS
|
||||
: isRuxaVideo
|
||||
? RUXA_DURATION_OPTIONS
|
||||
: STANDARD_DURATION_OPTIONS);
|
||||
const resolutionOptions = capabilities?.resolutionOptions?.length
|
||||
? capabilities.resolutionOptions.map((value) => ({
|
||||
value,
|
||||
label: value.toUpperCase(),
|
||||
}))
|
||||
: RESOLUTION_OPTIONS;
|
||||
const showResolutionControl = capabilities
|
||||
? capabilities.ui.showResolution && resolutionOptions.length > 0
|
||||
: !isFalVideo && !isRuxaVideo;
|
||||
const showGenerateAudioControl = capabilities
|
||||
? capabilities.ui.showGenerateAudio
|
||||
: isFalVideo;
|
||||
@ -339,6 +328,32 @@ export function VideoParamsBar({
|
||||
normalizeVideoParamsByProvider(initialParams, providerType, capabilities)
|
||||
);
|
||||
|
||||
const scopedCapabilityOptions = getVideoToolOptionsForMode(capabilities, params.mode);
|
||||
const capabilityRatioOptions = scopedCapabilityOptions.ratioOptions.length
|
||||
? scopedCapabilityOptions.ratioOptions.map((value) => ({ value, label: value }))
|
||||
: null;
|
||||
const ratioOptions = capabilityRatioOptions || (isFalVideo
|
||||
? FAL_RATIO_OPTIONS
|
||||
: isRuxaVideo
|
||||
? RUXA_RATIO_OPTIONS
|
||||
: RATIO_OPTIONS);
|
||||
const durationOptions = scopedCapabilityOptions.durationOptions.length
|
||||
? scopedCapabilityOptions.durationOptions
|
||||
: (isFalVideo
|
||||
? FAL_DURATION_OPTIONS
|
||||
: isRuxaVideo
|
||||
? RUXA_DURATION_OPTIONS
|
||||
: STANDARD_DURATION_OPTIONS);
|
||||
const resolutionOptions = scopedCapabilityOptions.resolutionOptions.length
|
||||
? scopedCapabilityOptions.resolutionOptions.map((value) => ({
|
||||
value,
|
||||
label: value.toUpperCase(),
|
||||
}))
|
||||
: RESOLUTION_OPTIONS;
|
||||
const showResolutionControl = capabilities
|
||||
? capabilities.ui.showResolution && scopedCapabilityOptions.resolutionOptions.length > 0
|
||||
: !isFalVideo && !isRuxaVideo;
|
||||
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
@ -412,7 +427,11 @@ export function VideoParamsBar({
|
||||
key: K,
|
||||
value: VideoGenerationParams[K]
|
||||
) => {
|
||||
const newParams = { ...params, [key]: value };
|
||||
const newParams = normalizeVideoParamsByProvider(
|
||||
{ ...params, [key]: value },
|
||||
providerType,
|
||||
capabilities
|
||||
);
|
||||
setParams(newParams);
|
||||
onParamsChange?.(newParams);
|
||||
};
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
import "server-only";
|
||||
|
||||
import sharp from "sharp";
|
||||
import {
|
||||
getRunwayVideoModeDefinition,
|
||||
isRunwayAudioSupported,
|
||||
isRunwaySeedSupported,
|
||||
normalizeRunwayDuration,
|
||||
resolveRunwayApiRatio,
|
||||
} from "@/lib/ai/runway-video-models";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { inferRunwayModelKind } from "@/lib/ai/model-classification";
|
||||
|
||||
@ -21,6 +28,7 @@ interface RunwayVideoParams {
|
||||
ratio?: string;
|
||||
duration?: number;
|
||||
seed?: number;
|
||||
generateAudio?: boolean;
|
||||
}
|
||||
|
||||
interface UploadedImage {
|
||||
@ -109,20 +117,6 @@ function normalizeImageResolution(aspectRatio?: string, imageSize?: string) {
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeVideoRatio(ratio?: string) {
|
||||
const normalized = String(ratio || "").trim();
|
||||
if (["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"].includes(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
return "16:9";
|
||||
}
|
||||
|
||||
function normalizeVideoDuration(duration?: number) {
|
||||
if (!Number.isFinite(duration)) return 5;
|
||||
const normalized = Math.round(Number(duration));
|
||||
return normalized >= 8 ? 10 : 5;
|
||||
}
|
||||
|
||||
function normalizeSeed(seed?: number) {
|
||||
if (!Number.isFinite(seed)) return undefined;
|
||||
return Math.max(0, Math.min(4294967295, Math.round(Number(seed))));
|
||||
@ -348,6 +342,23 @@ async function createTask(baseURL: string, apiKey: string, path: string, body: R
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
console.error("[Runway] Task request failed:", {
|
||||
path,
|
||||
status: response.status,
|
||||
bodySummary: {
|
||||
model: body.model,
|
||||
ratio: body.ratio,
|
||||
duration: body.duration,
|
||||
hasPromptImage: Boolean(body.promptImage),
|
||||
promptImageCount: Array.isArray(body.promptImage) ? body.promptImage.length : body.promptImage ? 1 : 0,
|
||||
hasReferenceImages: Array.isArray(body.referenceImages) ? body.referenceImages.length : 0,
|
||||
hasPromptText: typeof body.promptText === "string" && body.promptText.length > 0,
|
||||
promptTextLength: typeof body.promptText === "string" ? body.promptText.length : 0,
|
||||
audio: body.audio,
|
||||
seed: typeof body.seed === "number" ? body.seed : undefined,
|
||||
},
|
||||
response: payload,
|
||||
});
|
||||
const message =
|
||||
typeof payload?.error === "string"
|
||||
? payload.error
|
||||
@ -461,6 +472,43 @@ function getRunwayVideoEndpoint(modelId: string, hasPromptImage: boolean) {
|
||||
return "/v1/text_to_video";
|
||||
}
|
||||
|
||||
function normalizeVideoMode(
|
||||
mode: RunwayVideoParams["mode"] | undefined,
|
||||
hasPromptImage: boolean
|
||||
): NonNullable<RunwayVideoParams["mode"]> {
|
||||
if (
|
||||
mode === "text-to-video" ||
|
||||
mode === "first-frame" ||
|
||||
mode === "first-last-frame" ||
|
||||
mode === "reference-images"
|
||||
) {
|
||||
return mode;
|
||||
}
|
||||
|
||||
return hasPromptImage ? "first-frame" : "text-to-video";
|
||||
}
|
||||
|
||||
function buildRunwayPromptImagePayload(params: {
|
||||
modelId: string;
|
||||
mode: NonNullable<RunwayVideoParams["mode"]>;
|
||||
imageUrls: string[];
|
||||
}) {
|
||||
const resolved = getRunwayVideoModeDefinition(params.modelId, params.mode);
|
||||
if (!resolved || params.imageUrls.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const limitedImageUrls = params.imageUrls.slice(0, Math.max(1, resolved.definition.imageMax));
|
||||
if (params.mode === "first-last-frame" && limitedImageUrls.length >= 2) {
|
||||
return [
|
||||
{ uri: limitedImageUrls[0], position: "first" as const },
|
||||
{ uri: limitedImageUrls[1], position: "last" as const },
|
||||
];
|
||||
}
|
||||
|
||||
return limitedImageUrls[0];
|
||||
}
|
||||
|
||||
function buildReferenceImages(imageUrls: string[]) {
|
||||
return imageUrls.slice(0, 3).map((uri, index) => ({
|
||||
uri,
|
||||
@ -518,7 +566,10 @@ export function createRunway({ apiKey, baseURL }: RunwayConfig) {
|
||||
};
|
||||
}
|
||||
|
||||
const videoMode = options.videoParams?.mode || (normalizedImageUrls[0] ? "first-frame" : "text-to-video");
|
||||
const videoMode = normalizeVideoMode(
|
||||
options.videoParams?.mode,
|
||||
Boolean(normalizedImageUrls[0])
|
||||
);
|
||||
if (
|
||||
(videoMode === "first-frame" || videoMode === "first-last-frame" || videoMode === "reference-images") &&
|
||||
!normalizedImageUrls[0]
|
||||
@ -526,16 +577,51 @@ export function createRunway({ apiKey, baseURL }: RunwayConfig) {
|
||||
throw new Error("Runway 图生视频至少需要 1 张参考图");
|
||||
}
|
||||
|
||||
const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(normalizedImageUrls[0]));
|
||||
const promptImage = buildRunwayPromptImagePayload({
|
||||
modelId: activeModelId,
|
||||
mode: videoMode,
|
||||
imageUrls: normalizedImageUrls,
|
||||
});
|
||||
const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(promptImage));
|
||||
const ratio = resolveRunwayApiRatio({
|
||||
modelId: activeModelId,
|
||||
mode: videoMode,
|
||||
ratio: options.videoParams?.ratio,
|
||||
});
|
||||
const duration = normalizeRunwayDuration({
|
||||
modelId: activeModelId,
|
||||
mode: videoMode,
|
||||
duration: options.videoParams?.duration,
|
||||
});
|
||||
const seed = isRunwaySeedSupported(activeModelId, videoMode)
|
||||
? normalizeSeed(options.videoParams?.seed)
|
||||
: undefined;
|
||||
const audio =
|
||||
isRunwayAudioSupported(activeModelId, videoMode) &&
|
||||
typeof options.videoParams?.generateAudio === "boolean"
|
||||
? options.videoParams.generateAudio
|
||||
: undefined;
|
||||
|
||||
console.log("[Runway] Video request config:", {
|
||||
model: activeModelId,
|
||||
mode: videoMode,
|
||||
endpoint,
|
||||
ratio,
|
||||
duration,
|
||||
hasPromptImage: Boolean(promptImage),
|
||||
promptImageCount: Array.isArray(promptImage) ? promptImage.length : promptImage ? 1 : 0,
|
||||
audio,
|
||||
seedEnabled: seed !== undefined,
|
||||
});
|
||||
|
||||
const taskId = await createTask(baseURL, apiKey, endpoint, {
|
||||
model: activeModelId,
|
||||
promptText: prompt,
|
||||
ratio: normalizeVideoRatio(options.videoParams?.ratio),
|
||||
duration: normalizeVideoDuration(options.videoParams?.duration),
|
||||
...(normalizedImageUrls[0] ? { promptImage: normalizedImageUrls[0] } : {}),
|
||||
...(normalizeSeed(options.videoParams?.seed) !== undefined
|
||||
? { seed: normalizeSeed(options.videoParams?.seed) }
|
||||
: {}),
|
||||
ratio,
|
||||
duration,
|
||||
...(promptImage ? { promptImage } : {}),
|
||||
...(seed !== undefined ? { seed } : {}),
|
||||
...(typeof audio === "boolean" ? { audio } : {}),
|
||||
});
|
||||
|
||||
const task = await waitForTaskOutput(baseURL, apiKey, taskId);
|
||||
|
||||
482
lib/ai/runway-video-models.ts
Normal file
482
lib/ai/runway-video-models.ts
Normal file
@ -0,0 +1,482 @@
|
||||
import type {
|
||||
VideoToolDefaults,
|
||||
VideoToolMode,
|
||||
VideoToolModeConstraint,
|
||||
VideoToolUiToggles,
|
||||
} from "@/types/video-tool-capability";
|
||||
|
||||
type RunwayModeDefinition = VideoToolModeConstraint & {
|
||||
imageMin: number;
|
||||
imageMax: number;
|
||||
apiRatios: Record<string, string>;
|
||||
supportsAudio?: boolean;
|
||||
supportsSeed?: boolean;
|
||||
};
|
||||
|
||||
type RunwayVideoModelDefinition = {
|
||||
defaults: VideoToolDefaults;
|
||||
ui: VideoToolUiToggles;
|
||||
modes: Partial<Record<VideoToolMode, RunwayModeDefinition>>;
|
||||
};
|
||||
|
||||
const COMMON_IMAGE_TO_VIDEO_RATIOS = {
|
||||
"21:9": "1584:672",
|
||||
"16:9": "1280:720",
|
||||
"4:3": "1104:832",
|
||||
"1:1": "960:960",
|
||||
"3:4": "832:1104",
|
||||
"9:16": "720:1280",
|
||||
} as const;
|
||||
|
||||
const PORTRAIT_LANDSCAPE_RATIOS = {
|
||||
"16:9": "1280:720",
|
||||
"9:16": "720:1280",
|
||||
} as const;
|
||||
|
||||
const GEN3A_RATIOS = {
|
||||
"16:9": "1280:768",
|
||||
"9:16": "768:1280",
|
||||
} as const;
|
||||
|
||||
const VEO_RATIOS = {
|
||||
"16:9": "1280:720",
|
||||
"9:16": "720:1280",
|
||||
} as const;
|
||||
|
||||
const DURATIONS_2_TO_10 = Array.from({ length: 9 }, (_, index) => index + 2);
|
||||
const DURATIONS_5_OR_10 = [5, 10];
|
||||
const DURATIONS_4_6_8 = [4, 6, 8];
|
||||
|
||||
function buildModeDefinition(params: {
|
||||
imageMin: number;
|
||||
imageMax: number;
|
||||
ratioMap: Record<string, string>;
|
||||
durationOptions: number[];
|
||||
defaults?: Partial<VideoToolDefaults>;
|
||||
supportsAudio?: boolean;
|
||||
supportsSeed?: boolean;
|
||||
}): RunwayModeDefinition {
|
||||
return {
|
||||
imageMin: params.imageMin,
|
||||
imageMax: params.imageMax,
|
||||
ratioOptions: Object.keys(params.ratioMap),
|
||||
durationOptions: [...params.durationOptions],
|
||||
resolutionOptions: [],
|
||||
defaults: params.defaults,
|
||||
apiRatios: { ...params.ratioMap },
|
||||
supportsAudio: params.supportsAudio,
|
||||
supportsSeed: params.supportsSeed,
|
||||
};
|
||||
}
|
||||
|
||||
const RUNWAY_VIDEO_MODEL_DEFINITIONS: Array<[string, RunwayVideoModelDefinition]> = [
|
||||
[
|
||||
"gen4.5",
|
||||
{
|
||||
defaults: {
|
||||
mode: "text-to-video",
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: false,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: true,
|
||||
},
|
||||
modes: {
|
||||
"text-to-video": buildModeDefinition({
|
||||
imageMin: 0,
|
||||
imageMax: 0,
|
||||
ratioMap: PORTRAIT_LANDSCAPE_RATIOS,
|
||||
durationOptions: DURATIONS_2_TO_10,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
},
|
||||
supportsSeed: true,
|
||||
}),
|
||||
"first-frame": buildModeDefinition({
|
||||
imageMin: 1,
|
||||
imageMax: 1,
|
||||
ratioMap: COMMON_IMAGE_TO_VIDEO_RATIOS,
|
||||
durationOptions: DURATIONS_2_TO_10,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
},
|
||||
supportsSeed: true,
|
||||
}),
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
"gen4_turbo",
|
||||
{
|
||||
defaults: {
|
||||
mode: "first-frame",
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: false,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: true,
|
||||
},
|
||||
modes: {
|
||||
"first-frame": buildModeDefinition({
|
||||
imageMin: 1,
|
||||
imageMax: 1,
|
||||
ratioMap: COMMON_IMAGE_TO_VIDEO_RATIOS,
|
||||
durationOptions: DURATIONS_5_OR_10,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
},
|
||||
supportsSeed: true,
|
||||
}),
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
"gen3a_turbo",
|
||||
{
|
||||
defaults: {
|
||||
mode: "first-frame",
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: false,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: true,
|
||||
},
|
||||
modes: {
|
||||
"first-frame": buildModeDefinition({
|
||||
imageMin: 1,
|
||||
imageMax: 1,
|
||||
ratioMap: GEN3A_RATIOS,
|
||||
durationOptions: DURATIONS_5_OR_10,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
},
|
||||
supportsSeed: true,
|
||||
}),
|
||||
"first-last-frame": buildModeDefinition({
|
||||
imageMin: 2,
|
||||
imageMax: 2,
|
||||
ratioMap: GEN3A_RATIOS,
|
||||
durationOptions: DURATIONS_5_OR_10,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
},
|
||||
supportsSeed: true,
|
||||
}),
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
"veo3.1",
|
||||
{
|
||||
defaults: {
|
||||
mode: "text-to-video",
|
||||
ratio: "16:9",
|
||||
duration: 4,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: true,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: false,
|
||||
},
|
||||
modes: {
|
||||
"text-to-video": buildModeDefinition({
|
||||
imageMin: 0,
|
||||
imageMax: 0,
|
||||
ratioMap: VEO_RATIOS,
|
||||
durationOptions: DURATIONS_4_6_8,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 4,
|
||||
generateAudio: false,
|
||||
},
|
||||
supportsAudio: true,
|
||||
supportsSeed: false,
|
||||
}),
|
||||
"first-frame": buildModeDefinition({
|
||||
imageMin: 1,
|
||||
imageMax: 1,
|
||||
ratioMap: VEO_RATIOS,
|
||||
durationOptions: DURATIONS_4_6_8,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 4,
|
||||
generateAudio: false,
|
||||
},
|
||||
supportsAudio: true,
|
||||
supportsSeed: false,
|
||||
}),
|
||||
"first-last-frame": buildModeDefinition({
|
||||
imageMin: 2,
|
||||
imageMax: 2,
|
||||
ratioMap: VEO_RATIOS,
|
||||
durationOptions: DURATIONS_4_6_8,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 4,
|
||||
generateAudio: false,
|
||||
},
|
||||
supportsAudio: true,
|
||||
supportsSeed: false,
|
||||
}),
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
"veo3.1_fast",
|
||||
{
|
||||
defaults: {
|
||||
mode: "text-to-video",
|
||||
ratio: "16:9",
|
||||
duration: 4,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: true,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: false,
|
||||
},
|
||||
modes: {
|
||||
"text-to-video": buildModeDefinition({
|
||||
imageMin: 0,
|
||||
imageMax: 0,
|
||||
ratioMap: VEO_RATIOS,
|
||||
durationOptions: DURATIONS_4_6_8,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 4,
|
||||
generateAudio: false,
|
||||
},
|
||||
supportsAudio: true,
|
||||
supportsSeed: false,
|
||||
}),
|
||||
"first-frame": buildModeDefinition({
|
||||
imageMin: 1,
|
||||
imageMax: 1,
|
||||
ratioMap: VEO_RATIOS,
|
||||
durationOptions: DURATIONS_4_6_8,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 4,
|
||||
generateAudio: false,
|
||||
},
|
||||
supportsAudio: true,
|
||||
supportsSeed: false,
|
||||
}),
|
||||
"first-last-frame": buildModeDefinition({
|
||||
imageMin: 2,
|
||||
imageMax: 2,
|
||||
ratioMap: VEO_RATIOS,
|
||||
durationOptions: DURATIONS_4_6_8,
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 4,
|
||||
generateAudio: false,
|
||||
},
|
||||
supportsAudio: true,
|
||||
supportsSeed: false,
|
||||
}),
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
"veo3",
|
||||
{
|
||||
defaults: {
|
||||
mode: "text-to-video",
|
||||
ratio: "16:9",
|
||||
duration: 8,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: false,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: false,
|
||||
},
|
||||
modes: {
|
||||
"text-to-video": buildModeDefinition({
|
||||
imageMin: 0,
|
||||
imageMax: 0,
|
||||
ratioMap: VEO_RATIOS,
|
||||
durationOptions: [8],
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 8,
|
||||
},
|
||||
supportsSeed: false,
|
||||
}),
|
||||
"first-frame": buildModeDefinition({
|
||||
imageMin: 1,
|
||||
imageMax: 1,
|
||||
ratioMap: VEO_RATIOS,
|
||||
durationOptions: [8],
|
||||
defaults: {
|
||||
ratio: "16:9",
|
||||
duration: 8,
|
||||
},
|
||||
supportsSeed: false,
|
||||
}),
|
||||
},
|
||||
},
|
||||
],
|
||||
];
|
||||
|
||||
function normalizeModelId(modelId?: string | null) {
|
||||
return String(modelId || "").trim().toLowerCase();
|
||||
}
|
||||
|
||||
export function getRunwayVideoModelDefinition(modelId?: string | null) {
|
||||
const normalizedModelId = normalizeModelId(modelId);
|
||||
if (!normalizedModelId) return null;
|
||||
|
||||
for (const [key, definition] of RUNWAY_VIDEO_MODEL_DEFINITIONS) {
|
||||
if (
|
||||
normalizedModelId === key ||
|
||||
normalizedModelId.startsWith(`${key}_`) ||
|
||||
normalizedModelId.startsWith(`${key}:`) ||
|
||||
normalizedModelId.startsWith(`${key}-`)
|
||||
) {
|
||||
return definition;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function getRunwayVideoModeDefinition(
|
||||
modelId?: string | null,
|
||||
mode?: VideoToolMode | null
|
||||
) {
|
||||
const definition = getRunwayVideoModelDefinition(modelId);
|
||||
if (!definition) return null;
|
||||
|
||||
const availableModes = Object.entries(definition.modes).filter(
|
||||
(entry): entry is [VideoToolMode, RunwayModeDefinition] => Boolean(entry[1])
|
||||
);
|
||||
if (availableModes.length === 0) return null;
|
||||
|
||||
if (mode && definition.modes[mode]) {
|
||||
return {
|
||||
mode,
|
||||
definition: definition.modes[mode]!,
|
||||
model: definition,
|
||||
};
|
||||
}
|
||||
|
||||
const fallbackMode = definition.defaults.mode;
|
||||
if (definition.modes[fallbackMode]) {
|
||||
return {
|
||||
mode: fallbackMode,
|
||||
definition: definition.modes[fallbackMode]!,
|
||||
model: definition,
|
||||
};
|
||||
}
|
||||
|
||||
const [firstMode, firstDefinition] = availableModes[0];
|
||||
return {
|
||||
mode: firstMode,
|
||||
definition: firstDefinition,
|
||||
model: definition,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveRunwayApiRatio(params: {
|
||||
modelId?: string | null;
|
||||
mode?: VideoToolMode | null;
|
||||
ratio?: string | null;
|
||||
}) {
|
||||
const resolved = getRunwayVideoModeDefinition(params.modelId, params.mode);
|
||||
if (!resolved) {
|
||||
return params.ratio ? String(params.ratio).trim() : "1280:720";
|
||||
}
|
||||
|
||||
const normalizedRatio = String(params.ratio || "").trim();
|
||||
if (normalizedRatio && resolved.definition.apiRatios[normalizedRatio]) {
|
||||
return resolved.definition.apiRatios[normalizedRatio];
|
||||
}
|
||||
|
||||
const fallbackRatio =
|
||||
resolved.definition.defaults?.ratio ||
|
||||
resolved.model.defaults.ratio ||
|
||||
Object.keys(resolved.definition.apiRatios)[0] ||
|
||||
"16:9";
|
||||
|
||||
return resolved.definition.apiRatios[fallbackRatio] || "1280:720";
|
||||
}
|
||||
|
||||
export function normalizeRunwayDuration(params: {
|
||||
modelId?: string | null;
|
||||
mode?: VideoToolMode | null;
|
||||
duration?: number | null;
|
||||
}) {
|
||||
const resolved = getRunwayVideoModeDefinition(params.modelId, params.mode);
|
||||
const allowedDurations = resolved?.definition.durationOptions || [5];
|
||||
const fallbackDuration =
|
||||
Number(resolved?.definition.defaults?.duration) ||
|
||||
Number(resolved?.model.defaults.duration) ||
|
||||
allowedDurations[0] ||
|
||||
5;
|
||||
|
||||
if (!Number.isFinite(params.duration)) {
|
||||
return fallbackDuration;
|
||||
}
|
||||
|
||||
const normalizedDuration = Math.round(Number(params.duration));
|
||||
if (allowedDurations.includes(normalizedDuration)) {
|
||||
return normalizedDuration;
|
||||
}
|
||||
|
||||
return allowedDurations.reduce((closest, current) => {
|
||||
return Math.abs(current - normalizedDuration) < Math.abs(closest - normalizedDuration)
|
||||
? current
|
||||
: closest;
|
||||
}, fallbackDuration);
|
||||
}
|
||||
|
||||
export function isRunwayAudioSupported(modelId?: string | null, mode?: VideoToolMode | null) {
|
||||
const resolved = getRunwayVideoModeDefinition(modelId, mode);
|
||||
return Boolean(resolved?.definition.supportsAudio);
|
||||
}
|
||||
|
||||
export function isRunwaySeedSupported(modelId?: string | null, mode?: VideoToolMode | null) {
|
||||
const resolved = getRunwayVideoModeDefinition(modelId, mode);
|
||||
return Boolean(resolved?.definition.supportsSeed);
|
||||
}
|
||||
@ -1,7 +1,10 @@
|
||||
import type { VideoModelCapabilities } from "@/types/ai";
|
||||
import { getRunwayVideoModelDefinition } from "@/lib/ai/runway-video-models";
|
||||
import type {
|
||||
VideoToolCapabilityContract,
|
||||
VideoToolDefaults,
|
||||
VideoToolMode,
|
||||
VideoToolModeConstraint,
|
||||
VideoToolModeOption,
|
||||
VideoToolResolution,
|
||||
} from "@/types/video-tool-capability";
|
||||
@ -25,6 +28,10 @@ const PORTRAIT_RATIOS = ["16:9", "9:16", "1:1"];
|
||||
const STANDARD_RESOLUTIONS: VideoToolResolution[] = ["480p", "720p", "1080p"];
|
||||
const STANDARD_DURATIONS = [5, 10];
|
||||
|
||||
type ParsedVideoCapabilities = Partial<VideoModelCapabilities> & {
|
||||
constraints?: Partial<Record<VideoToolMode, VideoToolModeConstraint>>;
|
||||
};
|
||||
|
||||
function asObject(value: unknown) {
|
||||
return value && typeof value === "object" ? (value as Record<string, any>) : null;
|
||||
}
|
||||
@ -65,7 +72,64 @@ function normalizeModeOptions(
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeVideoCapabilities(rawCapabilities: unknown): Partial<VideoModelCapabilities> {
|
||||
function normalizeModeConstraints(
|
||||
rawConstraints: unknown
|
||||
): Partial<Record<VideoToolMode, VideoToolModeConstraint>> | undefined {
|
||||
const raw = asObject(rawConstraints);
|
||||
if (!raw) return undefined;
|
||||
|
||||
const modeConstraints: Partial<Record<VideoToolMode, VideoToolModeConstraint>> = {};
|
||||
|
||||
for (const mode of MODE_ORDER) {
|
||||
const entry = asObject(raw[mode]);
|
||||
if (!entry) continue;
|
||||
|
||||
const ratioOptions = Array.isArray(entry.ratioOptions)
|
||||
? entry.ratioOptions.map((item) => String(item || "").trim()).filter(Boolean)
|
||||
: undefined;
|
||||
const durationOptions = Array.isArray(entry.durationOptions)
|
||||
? entry.durationOptions
|
||||
.map((item) => Number(item))
|
||||
.filter((item) => Number.isFinite(item))
|
||||
.map((item) => Math.round(item))
|
||||
: undefined;
|
||||
const resolutionOptions = Array.isArray(entry.resolutionOptions)
|
||||
? entry.resolutionOptions.filter(
|
||||
(item): item is VideoToolResolution =>
|
||||
item === "480p" || item === "720p" || item === "1080p"
|
||||
)
|
||||
: undefined;
|
||||
const defaults = asObject(entry.defaults);
|
||||
|
||||
modeConstraints[mode] = {
|
||||
ratioOptions: ratioOptions?.length ? Array.from(new Set(ratioOptions)) : undefined,
|
||||
durationOptions: durationOptions?.length ? Array.from(new Set(durationOptions)) : undefined,
|
||||
resolutionOptions: resolutionOptions?.length ? Array.from(new Set(resolutionOptions)) : undefined,
|
||||
defaults: defaults
|
||||
? {
|
||||
mode,
|
||||
ratio: typeof defaults.ratio === "string" ? defaults.ratio : undefined,
|
||||
duration: Number.isFinite(Number(defaults.duration))
|
||||
? Math.round(Number(defaults.duration))
|
||||
: undefined,
|
||||
resolution:
|
||||
defaults.resolution === "480p" ||
|
||||
defaults.resolution === "720p" ||
|
||||
defaults.resolution === "1080p"
|
||||
? defaults.resolution
|
||||
: undefined,
|
||||
generateAudio:
|
||||
typeof defaults.generateAudio === "boolean" ? defaults.generateAudio : undefined,
|
||||
fps: Number.isFinite(Number(defaults.fps)) ? Math.round(Number(defaults.fps)) : undefined,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return Object.keys(modeConstraints).length > 0 ? modeConstraints : undefined;
|
||||
}
|
||||
|
||||
function normalizeVideoCapabilities(rawCapabilities: unknown): ParsedVideoCapabilities {
|
||||
const raw = asObject(rawCapabilities);
|
||||
if (!raw) return {};
|
||||
|
||||
@ -108,10 +172,101 @@ function normalizeVideoCapabilities(rawCapabilities: unknown): Partial<VideoMode
|
||||
supportReturnLastFrame:
|
||||
typeof raw.supportReturnLastFrame === "boolean" ? raw.supportReturnLastFrame : undefined,
|
||||
supportSeed: typeof raw.supportSeed === "boolean" ? raw.supportSeed : undefined,
|
||||
constraints: normalizeModeConstraints(raw.modeConstraints || raw.constraints),
|
||||
};
|
||||
}
|
||||
|
||||
function createProviderDefaultCapability(providerType: string): VideoToolCapabilityContract {
|
||||
function createRunwayDefaultCapability(modelId?: string | null): VideoToolCapabilityContract {
|
||||
const definition = getRunwayVideoModelDefinition(modelId);
|
||||
|
||||
if (!definition) {
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
source: "provider-default",
|
||||
providerType: "runway",
|
||||
modelId: modelId || undefined,
|
||||
modeOptions: normalizeModeOptions(["text-to-video", "first-frame"]),
|
||||
ratioOptions: [...STANDARD_RATIOS],
|
||||
durationOptions: [5, 10],
|
||||
resolutionOptions: [],
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: false,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: true,
|
||||
},
|
||||
defaults: {
|
||||
mode: "text-to-video",
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const modeEntries = MODE_ORDER.map((mode) => {
|
||||
const modeDefinition = definition.modes[mode];
|
||||
if (!modeDefinition) return null;
|
||||
return {
|
||||
mode,
|
||||
label: MODE_LABELS[mode],
|
||||
imageMin: modeDefinition.imageMin,
|
||||
imageMax: modeDefinition.imageMax,
|
||||
} satisfies VideoToolModeOption;
|
||||
}).filter((item): item is VideoToolModeOption => Boolean(item));
|
||||
|
||||
const modeConstraints = MODE_ORDER.reduce<Partial<Record<VideoToolMode, VideoToolModeConstraint>>>(
|
||||
(acc, mode) => {
|
||||
const modeDefinition = definition.modes[mode];
|
||||
if (!modeDefinition) return acc;
|
||||
acc[mode] = {
|
||||
ratioOptions: modeDefinition.ratioOptions,
|
||||
durationOptions: modeDefinition.durationOptions,
|
||||
resolutionOptions: modeDefinition.resolutionOptions || [],
|
||||
defaults: modeDefinition.defaults,
|
||||
};
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
|
||||
const ratioOptions = Array.from(
|
||||
new Set(
|
||||
Object.values(definition.modes)
|
||||
.flatMap((modeDefinition) => modeDefinition?.ratioOptions || [])
|
||||
.filter(Boolean)
|
||||
)
|
||||
);
|
||||
const durationOptions = Array.from(
|
||||
new Set(
|
||||
Object.values(definition.modes)
|
||||
.flatMap((modeDefinition) => modeDefinition?.durationOptions || [])
|
||||
.filter((item): item is number => Number.isFinite(item))
|
||||
)
|
||||
).sort((a, b) => a - b);
|
||||
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
source: "provider-default",
|
||||
providerType: "runway",
|
||||
modelId: modelId || undefined,
|
||||
modeOptions: modeEntries,
|
||||
ratioOptions,
|
||||
durationOptions,
|
||||
resolutionOptions: [],
|
||||
modeConstraints,
|
||||
ui: { ...definition.ui },
|
||||
defaults: { ...definition.defaults },
|
||||
};
|
||||
}
|
||||
|
||||
function createProviderDefaultCapability(
|
||||
providerType: string,
|
||||
modelId?: string | null
|
||||
): VideoToolCapabilityContract {
|
||||
if (providerType === "fal-video") {
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
@ -194,30 +349,7 @@ function createProviderDefaultCapability(providerType: string): VideoToolCapabil
|
||||
}
|
||||
|
||||
if (providerType === "runway") {
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
source: "provider-default",
|
||||
providerType,
|
||||
modeOptions: normalizeModeOptions(["text-to-video", "first-frame"]),
|
||||
ratioOptions: [...STANDARD_RATIOS],
|
||||
durationOptions: [5, 10],
|
||||
resolutionOptions: [],
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: false,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: true,
|
||||
},
|
||||
defaults: {
|
||||
mode: "text-to-video",
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
};
|
||||
return createRunwayDefaultCapability(modelId);
|
||||
}
|
||||
|
||||
return {
|
||||
@ -271,6 +403,65 @@ function pickPreferredMode(modeOptions: VideoToolModeOption[], fallback: VideoTo
|
||||
return modeOptions[0]?.mode || fallback;
|
||||
}
|
||||
|
||||
function pickPreferredResolution(resolutionOptions: VideoToolResolution[]) {
|
||||
return resolutionOptions.includes("720p") ? "720p" : resolutionOptions[0] || undefined;
|
||||
}
|
||||
|
||||
function getModeConstraintOptions(
|
||||
modeConstraints: Partial<Record<VideoToolMode, VideoToolModeConstraint>> | undefined,
|
||||
mode: VideoToolMode | undefined,
|
||||
defaults: VideoToolDefaults
|
||||
) {
|
||||
const constraint = mode ? modeConstraints?.[mode] : undefined;
|
||||
const ratioOptions = constraint?.ratioOptions?.length ? constraint.ratioOptions : [];
|
||||
const durationOptions = constraint?.durationOptions?.length ? constraint.durationOptions : [];
|
||||
const resolutionOptions = constraint?.resolutionOptions?.length
|
||||
? constraint.resolutionOptions
|
||||
: [];
|
||||
|
||||
return {
|
||||
ratioOptions,
|
||||
durationOptions,
|
||||
resolutionOptions,
|
||||
defaults: {
|
||||
...defaults,
|
||||
...(constraint?.defaults || {}),
|
||||
} as VideoToolDefaults,
|
||||
};
|
||||
}
|
||||
|
||||
export function getVideoToolOptionsForMode(
|
||||
capability: VideoToolCapabilityContract | null | undefined,
|
||||
mode?: VideoToolMode | null
|
||||
) {
|
||||
if (!capability) {
|
||||
return {
|
||||
ratioOptions: [] as string[],
|
||||
durationOptions: [] as number[],
|
||||
resolutionOptions: [] as VideoToolResolution[],
|
||||
defaults: null as VideoToolDefaults | null,
|
||||
};
|
||||
}
|
||||
|
||||
const fallbackMode =
|
||||
(mode && capability.modeOptions.some((item) => item.mode === mode) ? mode : undefined) ||
|
||||
capability.defaults.mode ||
|
||||
capability.modeOptions[0]?.mode;
|
||||
|
||||
const scoped = getModeConstraintOptions(capability.modeConstraints, fallbackMode, capability.defaults);
|
||||
|
||||
return {
|
||||
ratioOptions: scoped.ratioOptions.length ? scoped.ratioOptions : capability.ratioOptions,
|
||||
durationOptions: scoped.durationOptions.length
|
||||
? scoped.durationOptions
|
||||
: capability.durationOptions,
|
||||
resolutionOptions: scoped.resolutionOptions.length
|
||||
? scoped.resolutionOptions
|
||||
: capability.resolutionOptions,
|
||||
defaults: scoped.defaults,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildVideoToolCapability(params: {
|
||||
providerType?: string | null;
|
||||
modelId?: string | null;
|
||||
@ -278,7 +469,7 @@ export function buildVideoToolCapability(params: {
|
||||
}): VideoToolCapabilityContract | null {
|
||||
const providerType = String(params.providerType || "").trim();
|
||||
if (!providerType) return null;
|
||||
const base = createProviderDefaultCapability(providerType);
|
||||
const base = createProviderDefaultCapability(providerType, params.modelId);
|
||||
|
||||
if (providerType === "fal-video" || providerType === "qianwen-video" || providerType === "ruxa-video") {
|
||||
return {
|
||||
@ -292,7 +483,11 @@ export function buildVideoToolCapability(params: {
|
||||
Boolean(parsed.supportedModes?.length) ||
|
||||
Boolean(parsed.supportedRatios?.length) ||
|
||||
Boolean(parsed.supportedResolutions?.length) ||
|
||||
Boolean(parsed.durationRange);
|
||||
Boolean(parsed.durationRange) ||
|
||||
Boolean(
|
||||
parsed.constraints &&
|
||||
Object.values(parsed.constraints).some((constraint) => Boolean(constraint))
|
||||
);
|
||||
|
||||
if (!hasModelCapability) {
|
||||
return {
|
||||
@ -304,6 +499,8 @@ export function buildVideoToolCapability(params: {
|
||||
const supportedModes =
|
||||
parsed.supportedModes && parsed.supportedModes.length > 0
|
||||
? parsed.supportedModes
|
||||
: parsed.constraints && Object.keys(parsed.constraints).length > 0
|
||||
? MODE_ORDER.filter((mode) => Boolean(parsed.constraints?.[mode]))
|
||||
: base.modeOptions.map((item) => item.mode);
|
||||
|
||||
const modeOptions = normalizeModeOptions(supportedModes, parsed.maxReferenceImages || 4);
|
||||
@ -318,6 +515,51 @@ export function buildVideoToolCapability(params: {
|
||||
const durationOptions = parsed.durationRange
|
||||
? normalizeDurations(parsed.durationRange.min, parsed.durationRange.max)
|
||||
: base.durationOptions;
|
||||
const baseModeConstraints = base.modeConstraints || {};
|
||||
const nextModeConstraints =
|
||||
parsed.constraints && Object.keys(parsed.constraints).length > 0
|
||||
? MODE_ORDER.reduce<Partial<Record<VideoToolMode, VideoToolModeConstraint>>>((acc, mode) => {
|
||||
const baseConstraint = baseModeConstraints[mode];
|
||||
const parsedConstraint = parsed.constraints?.[mode];
|
||||
if (!baseConstraint && !parsedConstraint) return acc;
|
||||
acc[mode] = {
|
||||
ratioOptions:
|
||||
parsedConstraint?.ratioOptions?.length
|
||||
? parsedConstraint.ratioOptions
|
||||
: baseConstraint?.ratioOptions,
|
||||
durationOptions:
|
||||
parsedConstraint?.durationOptions?.length
|
||||
? parsedConstraint.durationOptions
|
||||
: parsedConstraint
|
||||
? durationOptions
|
||||
: baseConstraint?.durationOptions,
|
||||
resolutionOptions:
|
||||
parsedConstraint?.resolutionOptions?.length
|
||||
? parsedConstraint.resolutionOptions
|
||||
: baseConstraint?.resolutionOptions,
|
||||
defaults: {
|
||||
...(baseConstraint?.defaults || {}),
|
||||
...(parsedConstraint?.defaults || {}),
|
||||
},
|
||||
};
|
||||
return acc;
|
||||
}, {})
|
||||
: base.modeConstraints;
|
||||
const defaultMode = pickPreferredMode(modeOptions, base.defaults.mode);
|
||||
const modeScopedDefaults = getModeConstraintOptions(
|
||||
nextModeConstraints,
|
||||
defaultMode,
|
||||
base.defaults
|
||||
);
|
||||
const effectiveRatioOptions = modeScopedDefaults.ratioOptions.length
|
||||
? modeScopedDefaults.ratioOptions
|
||||
: ratioOptions;
|
||||
const effectiveDurationOptions = modeScopedDefaults.durationOptions.length
|
||||
? modeScopedDefaults.durationOptions
|
||||
: durationOptions;
|
||||
const effectiveResolutionOptions = modeScopedDefaults.resolutionOptions.length
|
||||
? modeScopedDefaults.resolutionOptions
|
||||
: resolutionOptions;
|
||||
|
||||
return {
|
||||
...base,
|
||||
@ -327,6 +569,7 @@ export function buildVideoToolCapability(params: {
|
||||
ratioOptions,
|
||||
resolutionOptions,
|
||||
durationOptions,
|
||||
modeConstraints: nextModeConstraints,
|
||||
ui: {
|
||||
...base.ui,
|
||||
showReturnLastFrame:
|
||||
@ -338,12 +581,11 @@ export function buildVideoToolCapability(params: {
|
||||
},
|
||||
defaults: {
|
||||
...base.defaults,
|
||||
mode: pickPreferredMode(modeOptions, base.defaults.mode),
|
||||
ratio: pickPreferredRatio(ratioOptions),
|
||||
duration: pickPreferredDuration(durationOptions),
|
||||
resolution: resolutionOptions.includes("720p")
|
||||
? "720p"
|
||||
: resolutionOptions[0] || undefined,
|
||||
...modeScopedDefaults.defaults,
|
||||
mode: defaultMode,
|
||||
ratio: pickPreferredRatio(effectiveRatioOptions),
|
||||
duration: pickPreferredDuration(effectiveDurationOptions),
|
||||
resolution: pickPreferredResolution(effectiveResolutionOptions),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@ -31,6 +31,13 @@ export interface VideoToolDefaults {
|
||||
fps?: number;
|
||||
}
|
||||
|
||||
export interface VideoToolModeConstraint {
|
||||
ratioOptions?: string[];
|
||||
durationOptions?: number[];
|
||||
resolutionOptions?: VideoToolResolution[];
|
||||
defaults?: Partial<VideoToolDefaults>;
|
||||
}
|
||||
|
||||
export interface VideoToolCapabilityContract {
|
||||
schemaVersion: "v1";
|
||||
source: "provider-default" | "model-capabilities";
|
||||
@ -40,6 +47,7 @@ export interface VideoToolCapabilityContract {
|
||||
ratioOptions: string[];
|
||||
durationOptions: number[];
|
||||
resolutionOptions: VideoToolResolution[];
|
||||
modeConstraints?: Partial<Record<VideoToolMode, VideoToolModeConstraint>>;
|
||||
ui: VideoToolUiToggles;
|
||||
defaults: VideoToolDefaults;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user