fix: match video canvas models to real capabilities and remove simulated processing
This commit is contained in:
134
parent
0c1a2e6fee
commit
60b7900bf9
@ -1,4 +1,5 @@
|
||||
"use client";
|
||||
import { filterCanvasModels } from "@/lib/video-canvas/model-matching";
|
||||
|
||||
import Link from "next/link";
|
||||
import { ArrowLeft, Plus, Trash2 } from "lucide-react";
|
||||
@ -24,10 +25,6 @@ import {
|
||||
} from "@/components/ui/select";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import {
|
||||
isImageGenerationModelDescriptor,
|
||||
isVideoModelDescriptor,
|
||||
} from "@/lib/ai/model-classification";
|
||||
import { fetcher } from "@/lib/utils";
|
||||
import {
|
||||
getVideoCanvasFunctionBillingCopy,
|
||||
@ -87,56 +84,23 @@ function getModelOptionLabel(model: AiModelOption) {
|
||||
return `${name}${id}`;
|
||||
}
|
||||
|
||||
function isImageModelOption(model: AiModelOption) {
|
||||
return isImageGenerationModelDescriptor({
|
||||
providerType: model.providerType,
|
||||
modelId: model.modelId,
|
||||
capabilities: model.capabilities,
|
||||
supportsImageGen: model.supportsImageGen,
|
||||
});
|
||||
}
|
||||
|
||||
function isVideoModelOption(model: AiModelOption) {
|
||||
return isVideoModelDescriptor({
|
||||
providerType: model.providerType,
|
||||
modelId: model.modelId,
|
||||
capabilities: model.capabilities,
|
||||
supportsImageGen: model.supportsImageGen,
|
||||
});
|
||||
}
|
||||
|
||||
function filterModelsForFunctionKind(
|
||||
models: AiModelOption[],
|
||||
kind: VideoCanvasFunctionDefinition["kind"],
|
||||
) {
|
||||
const enabled = models.filter((model) => model.isEnabled !== false && getModelOptionValue(model));
|
||||
let filtered = enabled;
|
||||
if (kind === "image" || kind === "image-edit") {
|
||||
filtered = enabled.filter(isImageModelOption);
|
||||
} else if (kind === "video") {
|
||||
filtered = enabled.filter(isVideoModelOption);
|
||||
} else if (kind === "multimodal") {
|
||||
filtered = enabled.filter(
|
||||
(model) => model.isMultimodal || model.supportsVision || (!isImageModelOption(model) && !isVideoModelOption(model)),
|
||||
);
|
||||
}
|
||||
return filtered.length ? filtered : enabled;
|
||||
}
|
||||
|
||||
function ModelSelect({
|
||||
allowInherit = false,
|
||||
kind,
|
||||
functionKey,
|
||||
models,
|
||||
onChange,
|
||||
value,
|
||||
}: {
|
||||
allowInherit?: boolean;
|
||||
kind: VideoCanvasFunctionDefinition["kind"];
|
||||
functionKey?: string;
|
||||
models: AiModelOption[];
|
||||
onChange: (value: string) => void;
|
||||
value: string;
|
||||
}) {
|
||||
const filteredModels = filterModelsForFunctionKind(models, kind);
|
||||
const modelKind = functionKey === "audio-to-video" ? "audio-video" : kind === "image" || kind === "image-edit" || kind === "video" || kind === "multimodal" ? kind : "text";
|
||||
const filteredModels = filterCanvasModels(models, modelKind);
|
||||
const normalizedValue = value?.trim() || (allowInherit ? INHERIT_MODEL_VALUE : "auto");
|
||||
const hasCurrent =
|
||||
normalizedValue === "auto" ||
|
||||
@ -851,13 +815,14 @@ export function VideoCanvasSettingsClient() {
|
||||
<div className="grid gap-4 md:grid-cols-1">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`model-${item.key}`}>模型覆盖</Label>
|
||||
<ModelSelect
|
||||
{["canvas-generate-image", "canvas-generate-video", "canvas-workflow-analyze", "pick-object"].includes(item.route) ? <ModelSelect
|
||||
allowInherit
|
||||
functionKey={item.key}
|
||||
kind={item.kind}
|
||||
models={modelOptions}
|
||||
onChange={(value) => updateFunction(item.key, { model: value })}
|
||||
value={item.model || ""}
|
||||
/>
|
||||
/> : <p className="text-sm text-muted-foreground">{item.route === "local" ? "本地处理，无需生成模型" : "使用此功能的专用服务接口"}</p>}
|
||||
</div>
|
||||
<div className="rounded-lg border border-dashed border-border/70 bg-muted/20 px-3 py-2">
|
||||
<div className="text-xs font-medium text-foreground">
|
||||
|
||||
@ -102,8 +102,8 @@ export async function GET(request: Request) {
|
||||
},
|
||||
{
|
||||
id: "local-e2e-video-model",
|
||||
modelId: "kling-3-local",
|
||||
displayName: "可灵 3.0",
|
||||
modelId: "veo-3.1-fast",
|
||||
displayName: "Veo 3.1 Fast（本地布局测试）",
|
||||
avatarUrl: null,
|
||||
description: "本地视频画布测试视频模型",
|
||||
supportsVision: true,
|
||||
@ -111,18 +111,18 @@ export async function GET(request: Request) {
|
||||
isMultimodal: true,
|
||||
isDefault: false,
|
||||
providerName: "Local E2E",
|
||||
providerType: "local",
|
||||
providerType: "vip98k",
|
||||
isSubModel: false,
|
||||
parentModelId: null,
|
||||
capabilities: { video: true },
|
||||
capabilities: { kind: "video", supportedModes: ["text-to-video", "first-frame", "first-last-frame", "reference-images"], supportedRatios:["16:9"], supportedResolutions:["720p","1080p"], durationOptions:[8], maxReferenceImages:3 },
|
||||
videoToolCapability: {
|
||||
supported: true,
|
||||
durations: [5, 10],
|
||||
durations: [8],
|
||||
resolutions: ["720p", "1080p"],
|
||||
},
|
||||
imageGenConfig: {
|
||||
creditCostPerVideo: 20,
|
||||
videoDurationOptions: [5, 10],
|
||||
videoDurationOptions: [8],
|
||||
},
|
||||
sortOrder: 1,
|
||||
selectable: true,
|
||||
|
||||
@ -31,6 +31,7 @@ function normalizeBillingFeature(
|
||||
|
||||
function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest {
|
||||
return {
|
||||
allowFailover: typeof body.allowFailover === "boolean" ? body.allowFailover : undefined,
|
||||
prompt: String(body.prompt || "").trim(),
|
||||
negativePrompt:
|
||||
typeof body.negativePrompt === "string" ? body.negativePrompt : undefined,
|
||||
|
||||
@ -10,6 +10,8 @@ interface GenerateVideoRequest {
|
||||
prompt?: string;
|
||||
mode?: VideoGenerationMode;
|
||||
images?: string[];
|
||||
audioUrls?: string[];
|
||||
generateAudio?: boolean;
|
||||
resolution?: VideoResolution;
|
||||
ratio?: VideoRatio;
|
||||
duration?: number;
|
||||
@ -47,12 +49,12 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
const inputImages = Array.isArray(body.images)
|
||||
? body.images.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 4)
|
||||
? body.images.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 8)
|
||||
: [];
|
||||
const { kept: reachableImages, dropped: unreachableImages } =
|
||||
await filterReachableRemoteUrls(inputImages, { timeoutMs: 3000 });
|
||||
|
||||
if (inputImages.length > 0 && reachableImages.length === 0) {
|
||||
if (unreachableImages.length > 0) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "参考图片链接不可访问，请重新上传后再试" },
|
||||
{ status: 400 }
|
||||
@ -63,6 +65,8 @@ export async function POST(request: Request) {
|
||||
const result = await executeDirectVideoGeneration({
|
||||
prompt,
|
||||
mode,
|
||||
audioUrls: Array.isArray(body.audioUrls) ? body.audioUrls.filter(url => typeof url === "string" && /^https?:\/\//i.test(url)).slice(0, 3) : undefined,
|
||||
generateAudio: body.generateAudio,
|
||||
images: reachableImages,
|
||||
resolution: body.resolution,
|
||||
ratio: body.ratio,
|
||||
|
||||
39
app/api/canvas/video/process/route.ts
Normal file
39
app/api/canvas/video/process/route.ts
Normal file
@ -0,0 +1,39 @@
|
||||
import {auth} from '@/app/(auth)/auth';
|
||||
import {NextResponse} from 'next/server';
|
||||
import {z} from 'zod';
|
||||
import {mkdtemp,readFile,rm} from 'node:fs/promises';
|
||||
import {tmpdir} from 'node:os';
|
||||
import {join} from 'node:path';
|
||||
import {randomUUID} from 'node:crypto';
|
||||
import {uploadFile} from '@/lib/storage/service';
|
||||
import {downloadCanvasMedia} from '@/lib/video-canvas/download-media';
|
||||
import {processVideoFile} from '@/lib/video-canvas/media-processing';
|
||||
export const runtime='nodejs';
|
||||
export const maxDuration=300;
|
||||
const running=new Set<string>();
|
||||
const schema=z.object({operation:z.enum(['crop','audio-video']),url:z.string().url(),rect:z.object({x:z.number(),y:z.number(),width:z.number(),height:z.number()}).optional()});
|
||||
export async function POST(request:Request) {
|
||||
const session=await auth();
|
||||
if(!session?.user?.id) return NextResponse.json({success:false,error:'未登录'},{status:401});
|
||||
const parsed=schema.safeParse(await request.json().catch(()=>null));
|
||||
if(!parsed.success) return NextResponse.json({success:false,error:'视频处理参数无效'},{status:400});
|
||||
if(running.has(session.user.id)||running.size>=2) return NextResponse.json({success:false,error:'视频处理中，请稍后再提交'},{status:429});
|
||||
running.add(session.user.id);
|
||||
let dir='';
|
||||
try {
|
||||
dir=await mkdtemp(join(tmpdir(),'canvas-video-'));
|
||||
const input=join(dir,'input');
|
||||
await downloadCanvasMedia(parsed.data.url,input);
|
||||
const save=async(name:string)=>{const buffer=await readFile(join(dir,name));return (await uploadFile(`canvas-video-${randomUUID()}-${name}`,new Uint8Array(buffer).buffer)).url;};
|
||||
if(parsed.data.operation==='crop') {
|
||||
const metadata=await processVideoFile(input,join(dir,'crop.mp4'),'crop',parsed.data.rect);
|
||||
return NextResponse.json({success:true,data:{url:await save('crop.mp4'),...metadata}});
|
||||
}
|
||||
const metadata=await processVideoFile(input,join(dir,'audio.m4a'),'extract-audio');
|
||||
await processVideoFile(input,join(dir,'silent.mp4'),'silent-video');
|
||||
return NextResponse.json({success:true,data:{audioUrl:await save('audio.m4a'),url:await save('silent.mp4'),...metadata}});
|
||||
} catch(error) {
|
||||
const message=error instanceof Error ? error.message : '视频处理失败';
|
||||
return NextResponse.json({success:false,error:message.startsWith('Command failed')?'视频处理失败，请确认文件完整且格式受支持':message},{status:400});
|
||||
} finally {running.delete(session.user.id);if(dir)await rm(dir,{recursive:true,force:true});}
|
||||
}
|
||||
@ -1,3 +1,4 @@
|
||||
import { isCanvasLanguageModel, isCanvasVisionModel } from "@/lib/video-canvas/model-matching";
|
||||
import { generateText } from "ai";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
@ -776,6 +777,11 @@ function buildStoryScriptPrompt(params: {
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
}
|
||||
|
||||
const body = await request.json().catch(() => ({}));
|
||||
const parsed = requestSchema.safeParse(body);
|
||||
if (!parsed.success) {
|
||||
@ -800,50 +806,27 @@ export async function POST(request: Request) {
|
||||
);
|
||||
}
|
||||
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
}
|
||||
|
||||
const [allModels, agentModel] = await Promise.all([
|
||||
getEnabledAiModelsWithProvider(false),
|
||||
getAgentAiModelWithProvider(),
|
||||
]);
|
||||
const compatibleModels = allModels.filter(
|
||||
(item) => !item.model.isSubModel && item.model.isEnabled
|
||||
);
|
||||
|
||||
const preferredModel =
|
||||
preferredModelRef
|
||||
? compatibleModels.find(
|
||||
(item) =>
|
||||
item.model.id === preferredModelRef ||
|
||||
item.model.modelId === preferredModelRef,
|
||||
)
|
||||
: null;
|
||||
|
||||
const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
|
||||
|
||||
let selectedModel =
|
||||
preferredModel && (!hasVisionInputs || supportsVision(preferredModel.model))
|
||||
? preferredModel
|
||||
: hasVisionInputs &&
|
||||
agentModel &&
|
||||
supportsVision(agentModel.model)
|
||||
? agentModel
|
||||
: hasVisionInputs
|
||||
? compatibleModels.find((item) => supportsVision(item.model))
|
||||
: agentModel || compatibleModels[0];
|
||||
|
||||
if (!selectedModel) {
|
||||
selectedModel = compatibleModels[0];
|
||||
const compatibleModels = allModels.filter((item) => {
|
||||
const descriptor = {...item.model, providerType: item.provider.type};
|
||||
return !item.model.isSubModel && item.model.isEnabled &&
|
||||
(hasVisionInputs ? isCanvasVisionModel(descriptor) : isCanvasLanguageModel(descriptor));
|
||||
});
|
||||
const requestedRef = preferredModelRef === "auto" ? "" : preferredModelRef;
|
||||
const preferredModel = requestedRef ? compatibleModels.find(
|
||||
item => item.model.id === requestedRef || item.model.modelId === requestedRef
|
||||
) : undefined;
|
||||
if (requestedRef && !preferredModel) {
|
||||
return NextResponse.json({ success: false, error: "所选模型不可用或不支持当前分析输入，请重新选择分析模型" }, { status: 400 });
|
||||
}
|
||||
|
||||
const selectedModel = preferredModel || compatibleModels.find(item => item.model.id === agentModel?.model.id) || compatibleModels[0];
|
||||
if (!selectedModel) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "当前站点没有可用的分析模型" },
|
||||
{ status: 500 }
|
||||
);
|
||||
return NextResponse.json({ success: false, error: hasVisionInputs ? "当前没有可用的视觉理解模型" : "当前没有可用的文字分析模型" }, { status: 400 });
|
||||
}
|
||||
|
||||
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
|
||||
/* include utility classes in streamdown */
|
||||
@source '../node_modules/streamdown/dist/index.js';
|
||||
@source '../components/canvas/**/*.tsx';
|
||||
@source not '../services/pascal-editor';
|
||||
|
||||
/* custom variant for setting dark mode programmatically */
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
60
docs/video-canvas-parity-2026-09-16.md
Normal file
60
docs/video-canvas-parity-2026-09-16.md
Normal file
@ -0,0 +1,60 @@
|
||||
# 视频画布模型匹配与逐项对照（2026-09-16）
|
||||
|
||||
参考：[LibTV 画布](https://www.liblib.tv/canvas)。以登录后的 1280×720 画布为界面对照基准。用户最新要求：模型接通即可，不继续真实生成测试。因此下表区分接口与参数核验、界面核验、实际生成，不能把未生成的功能标为产物验收通过，也不把局部对齐称为整站像素级复刻完成。
|
||||
|
||||
## 模型结论
|
||||
|
||||
初始生产登记 413 个模型，模型和服务商同时启用 23 个，真正的视频模型启用数为 0。前端菜单硬编码模型名称，后端把 Agnes 文字模型当成视频模型；无匹配模型时还会把图片、文字和 3D 模型全部列出。
|
||||
|
||||
本轮改为按真实模型 ID、适配器、输入模式筛选。管理员默认与节点覆盖都使用现有模型；用户手动指定不可用模型时明确报错。图片编辑按参考图能力筛选，分析节点只接受语言/视觉理解模型，专用图像工具沿用自己的服务接口，不提供无效的通用模型覆盖。
|
||||
|
||||
| 类型 | 匹配结果 | 参数 / 限制 |
|
||||
|---|---|---|
|
||||
| 文生、首帧、首尾帧、多参考视频 | VIP98K `veo-3.1-fast` | 当前目录为 8 秒、16:9、720P/1080P；首尾帧 2 图，多参考最多 3 图 |
|
||||
| 图片生成与编辑 | 现有 GPT Image 2.5 默认，其他兼容已启用图片模型可选 | 依据模型返回能力，手动选择不静默换模型 |
|
||||
| 脚本、视频分析、提示词优化翻译、元素识别 | 现有 Qwen3 VL Plus 默认，其他兼容模型可选 | 排除图片、视频、3D、世界生成模型 |
|
||||
| 音频驱动视频 | AtlasCloud 适配器已传递 `reference_audios`，当前无可用启用线路 | 普通 Veo 不冒充音频驱动能力 |
|
||||
| 视频超清、字幕擦除、人声/背景音分离 | 当前目录没有确认可用的专用线路 | 明确提示未接通，已移除计时后返回原素材的模拟成功 |
|
||||
| 视频画面裁剪、音轨提取与静音视频 | 48 服务器本地 FFmpeg | 无需模型；125 MiB、10 分钟处理上限，与生成结果下载上限无关 |
|
||||
|
||||
用户要求停止真实测试之前已获得的证据：AtlasCloud Seedance Mini 返回 402；旧 VIP98K Seedance 路由返回 404；当前 VIP98K Veo task 接口完成过一个任务。此后没有再提交真实模型生成。其他模式按当前公开 schema、请求契约和界面检查，不宣称分别生成成功。
|
||||
|
||||
## 逐项记录
|
||||
|
||||
| 功能 | 完成的改动 / 核验 | 验证状态 |
|
||||
|---|---|---|
|
||||
| 节点模型菜单 | 删除固定 Seedance 等名称；显示真实模型及模式；无匹配项不混入其他类型 | 单测、浏览器菜单通过 |
|
||||
| 参数联动 | 根据模型与模式限定比例、清晰度、离散时长；Veo 无虚构 4K 或 5 秒选项 | 浏览器确认只有 16:9、720P/1080P、8 秒 |
|
||||
| 文生视频 | 统一现有服务商选择与任务调用；自动选择也能正确进入 VIP98K 适配器 | 既有接通证据；不再新生成 |
|
||||
| 首帧 / 首尾帧 / 多参考 | 按 schema 补入可选 `file_urls`；区分 frame/ingredient；校验数量与模式 | 契约单测通过，不做效果验收 |
|
||||
| 音频生视频 | 音频实际进入请求；筛选支持音频参考的适配器，无兼容模型明确拒绝 | 单测通过，线路未接通 |
|
||||
| 全能参考入口 | 由旧首帧专用编辑器改为统一视频节点编辑器，支持模式和模型切换 | 浏览器确认参考图与 8 秒参数 |
|
||||
| 脚本 / 角色场景提取 | 语言/视觉模型严格筛选，指定模型错误不静默替换；鉴权先于视频帧提取 | 类型检查与选择单测通过 |
|
||||
| 视频分析 / 提示词优化翻译 | 同上；替换生产配置中的失效文字模型引用 | 不发起真实分析 |
|
||||
| 三视图、四宫格、九宫格、25 宫格 | 参考图编辑链路与独立提示词核对，模型覆盖传入生成请求 | 接线核验，不做效果验收 |
|
||||
| 角色脸部三视图、角色设定图、场景设定图、产品设定图 | 按当前参考站补入四项预设，支持管理员提示词及模型配置 | 代码与类型检查；不点击生成 |
|
||||
| 九宫格菜单 | 核对 11 项内容，采用 14px 文案、39.25px 行高与可滚动浮层，避免旧缩放导致菜单过小 | 参考 DOM 尺寸已记录 |
|
||||
| 光影校正、画面推演、打光、重绘、擦除 | 保留现有图片编辑入口；复用兼容模型，显式选择不切换 | 请求链路核对，不发起生成 |
|
||||
| 多角度 | 现有 fal Qwen Multiple Angles 专用路由 | 接线核对，不额外调用 |
|
||||
| 图片高清、扩图、抠图、图层分离 | 沿用对应专用接口；管理员不再显示无效通用模型选择 | 接线核对，不额外调用 |
|
||||
| 图片裁剪、宫格切分 | 本地图片处理，无需分配生成模型 | 既有处理逻辑保留 |
|
||||
| 视频画面裁剪 | 拖动框选，按真实源视频像素编码新文件，返回新 URL | 本地合成短片裁成 160×120，通过 |
|
||||
| 音视频分离 | 从原视频提取 AAC 音轨及静音视频，移除示例音频替代分支 | 本地合成短片验证通过 |
|
||||
| 视频超清、字幕擦除、人声分离 | 移除虚假成功；保留明确未接通提示 | 当前模型目录不足，未标为可用 |
|
||||
| 空画布入口 | 四卡片每项 240×56、984px 行宽、8px 间隔、42px 图标；保留本站品牌 | 本地 1280×720 截图核对 |
|
||||
| 底部工具栏 | 补拖动画布按钮、空格临时拖动；48px 高度与参考对齐 | 浏览器显示确认 |
|
||||
| 新增节点 / 工作流定位 | 避免新节点互相覆盖；工作流按当前窗口居中缩放 | 代码和本地界面核对 |
|
||||
| 视频编辑器 | 保持固定可读尺寸，模型参数随节点保存，位置限制在窗口内 | 本地界面核对 |
|
||||
| 积分显示 | 删除固定积分文案，修复缺少会员配置时误显示“免费” | 会员状态读取核对 |
|
||||
| 素材、角色、特效、运镜、历史 | 保留现有持久化与引用链路，参数进入统一视频生成入口 | 未逐项实测完整业务闭环 |
|
||||
|
||||
## 尚未达到像素级一致的部分
|
||||
|
||||
参考站当前新增的人像质感、工具箱模板、分享与 Agent 顶栏等与本站仍有差异。本次没有把这些差异记为完成，也没有复制参考站最新模型名称充当本站已接入模型。专用视频能力缺少可用供应商线路，不能靠改菜单补齐。
|
||||
|
||||
## 检查与发布
|
||||
|
||||
- 模型匹配、VIP98K 参考图与恢复相关测试：48 项通过，无真实外部生成请求。
|
||||
- TypeScript 检查通过；本地 FFmpeg 合成短片完成画面裁剪、音轨提取、静音视频三项验证。
|
||||
- 旧 `model-classification.test.ts` 两项测试在未修改基线同样失败，未混入本轮修复。
|
||||
- 发布状态与提交信息在上线验收后追加。
|
||||
@ -302,7 +302,12 @@ export function createAtlasCloudVideoRequestBody(params: {
|
||||
images: params.referenceImages,
|
||||
});
|
||||
|
||||
if (params.options.referenceAudios?.length) {
|
||||
if (!runtime.propertyNames.includes("reference_audios")) throw new Error("当前视频模型不支持音频参考");
|
||||
body.reference_audios = params.options.referenceAudios;
|
||||
}
|
||||
const optionalParams: Array<[string, unknown]> = [
|
||||
["generate_audio", params.options.generateAudio],
|
||||
["seed", params.options.seed],
|
||||
["watermark", params.options.watermark],
|
||||
["camera_fixed", params.options.cameraFixed],
|
||||
|
||||
@ -15,6 +15,7 @@ import {
|
||||
import { getDefaultStorageConfig, getSystemSetting } from "@/lib/db/queries";
|
||||
import {
|
||||
bindVip98kImages,
|
||||
getVip98kImageBindingKeys,
|
||||
hasVerifiedVip98kOpenAiImageArrayContract,
|
||||
normalizeVip98kInvocationHeaders,
|
||||
resolveVip98kImageTaskSize,
|
||||
@ -1461,8 +1462,8 @@ function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation |
|
||||
hasNativeAspectRatio = true;
|
||||
}
|
||||
}
|
||||
if (duration && ("duration" in body || !invocation)) body.duration = duration;
|
||||
if (duration && "duration_seconds" in body) body.duration_seconds = duration;
|
||||
if (duration && invocationSupportsBodyField(invocation, "duration")) body.duration = duration;
|
||||
if (duration && invocationSupportsBodyField(invocation, "duration_seconds")) body.duration_seconds = duration;
|
||||
if (resolution) {
|
||||
if (invocationSupportsBodyField(invocation, "resolution")) {
|
||||
body.resolution = resolution;
|
||||
@ -1522,6 +1523,18 @@ function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation |
|
||||
if ("generate_audio" in body || !invocation) body.generate_audio = videoParams.generateAudio;
|
||||
}
|
||||
|
||||
// Reference fields can be optional in the schema and absent from its text-only example.
|
||||
if (params.imageUrls.length) {
|
||||
const keys = getVip98kImageBindingKeys(invocation);
|
||||
const existing = keys.filter(key => key in body);
|
||||
const chosen = existing.length ? existing : keys.includes("file_urls") ? ["file_urls"] : keys.slice(0, 1);
|
||||
for (const key of chosen) if (!(key in body)) body[key] = [];
|
||||
}
|
||||
if (params.kind === "video" && params.modelId === "veo-3.1-fast" && invocation && invocationSupportsBodyField(invocation, "mode_image")) {
|
||||
const imageMode = videoParams.mode === "reference-images" ? "ingredient" : "frame";
|
||||
body.metadata = { ...(toRecord(body.metadata) || {}), mode_image: imageMode };
|
||||
delete body.mode_image;
|
||||
}
|
||||
const imageBindingKeys = bindVip98kImages(body, params.imageUrls);
|
||||
if (params.imageUrls.length > 0 && !invocation && imageBindingKeys.length === 0) {
|
||||
body.images = params.imageUrls;
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
* 支持火山引擎视频生成的多种模式
|
||||
*/
|
||||
|
||||
import { getCanvasVideoModes, isExecutableCanvasVideoModel, supportsCanvasAudioVideo, validateCanvasVideoInputs, getCanvasVideoParameters } from "@/lib/video-canvas/model-matching";
|
||||
import { tool } from "ai";
|
||||
import { z } from "zod";
|
||||
import { doGenerateStream } from "@/lib/ai/providers/volcano-video";
|
||||
@ -24,6 +25,8 @@ export interface ExecuteDirectVideoGenerationParams {
|
||||
prompt: string;
|
||||
mode: VideoGenerationMode;
|
||||
images?: string[];
|
||||
audioUrls?: string[];
|
||||
generateAudio?: boolean;
|
||||
resolution?: "480p" | "720p" | "1080p";
|
||||
ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive";
|
||||
duration?: number;
|
||||
@ -68,9 +71,8 @@ function extractVideoUrlFromGeneratedText(value: unknown) {
|
||||
}
|
||||
|
||||
function getSupportedModes(caps: Partial<VideoModelCapabilities> | undefined, modelId: string): VideoGenerationMode[] {
|
||||
const modes = (caps?.supportedModes || []) as VideoGenerationMode[];
|
||||
if (Array.isArray(modes) && modes.length > 0) {
|
||||
return modes;
|
||||
if (Array.isArray(caps?.supportedModes)) {
|
||||
return getCanvasVideoModes({modelId, capabilities: {...caps, kind: "video"}});
|
||||
}
|
||||
return inferSupportedModesFromModelId(modelId);
|
||||
}
|
||||
@ -80,7 +82,7 @@ function withConfiguredVideoCapabilities(
|
||||
imageGenConfig?: { videoDurationOptions?: number[] | null } | null
|
||||
): VideoModelCapabilities {
|
||||
const baseCapabilities = {
|
||||
supportedModes: ["text-to-video", "first-frame", "first-last-frame", "reference-images"],
|
||||
supportedModes: [],
|
||||
supportedResolutions: ["480p", "720p", "1080p"],
|
||||
supportedRatios: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"],
|
||||
durationRange: { min: 2, max: 12 },
|
||||
@ -114,7 +116,7 @@ function isAgnesVideoModel(item: any) {
|
||||
const modelId = String(item?.model?.modelId || "").toLowerCase();
|
||||
return (
|
||||
item?.provider?.type === "custom" &&
|
||||
(provider === "agnes-ai" || (kind === "video" && modelId.startsWith("agnes-video")))
|
||||
(kind === "video" && (provider === "agnes-ai" || modelId.startsWith("agnes-video")))
|
||||
);
|
||||
}
|
||||
|
||||
@ -145,12 +147,7 @@ function isVip98kVideoModel(item: any) {
|
||||
}
|
||||
|
||||
function isDirectExecutableVideoModel(item: any) {
|
||||
return (
|
||||
isVolcanoVideoModel(item) ||
|
||||
isAgnesVideoModel(item) ||
|
||||
isAtlasCloudVideoModel(item) ||
|
||||
isVip98kVideoModel(item)
|
||||
);
|
||||
return isExecutableCanvasVideoModel({...item.model, providerType: item.provider?.type});
|
||||
}
|
||||
|
||||
/**
|
||||
@ -209,7 +206,9 @@ async function selectBestSubModel(
|
||||
const supportedResolutions = caps.supportedResolutions || ["480p", "720p", "1080p"];
|
||||
const supportsResolution = !params.resolution || supportedResolutions.includes(params.resolution);
|
||||
|
||||
return supportsMode && supportsResolution;
|
||||
if (params.referenceAudios?.length && !supportsCanvasAudioVideo({...m.model,providerType:m.provider.type})) return false;
|
||||
if (!isDirectExecutableVideoModel(m) || !supportsMode || !supportsResolution) return false;
|
||||
try { validateParamsAgainstCapabilities({...params, mode}, caps); return true; } catch { return false; }
|
||||
});
|
||||
|
||||
if (compatibleModels.length === 0) {
|
||||
@ -276,7 +275,9 @@ async function getCompatibleModel(
|
||||
m.model.capabilities as VideoModelCapabilities,
|
||||
(m.model as any).imageGenConfig
|
||||
);
|
||||
return getSupportedModes(caps, m.model.modelId).includes(mode);
|
||||
if (!getSupportedModes(caps, m.model.modelId).includes(mode)) return false;
|
||||
if (params?.referenceAudios?.length && !supportsCanvasAudioVideo({...m.model, providerType:m.provider.type})) return false;
|
||||
try { validateParamsAgainstCapabilities({...params, mode}, caps); return true; } catch { return false; }
|
||||
});
|
||||
|
||||
if (compatibleModels.length === 0) {
|
||||
@ -312,6 +313,10 @@ function validateParamsAgainstCapabilities(
|
||||
params: Partial<VideoGenerationOptions>,
|
||||
capabilities: VideoModelCapabilities
|
||||
): void {
|
||||
const scoped = getCanvasVideoParameters({ capabilities }, params.mode);
|
||||
if (params.ratio && scoped.ratios.length && !scoped.ratios.includes(params.ratio)) throw new Error(`当前模式不支持 ${params.ratio} 比例`);
|
||||
if (params.resolution && scoped.resolutions.length && !scoped.resolutions.includes(params.resolution)) throw new Error(`当前模式不支持 ${params.resolution} 分辨率`);
|
||||
if (params.duration && scoped.durations.length && !scoped.durations.includes(params.duration)) throw new Error(`当前模式视频时长仅支持 ${scoped.durations.join(" / ")} 秒`);
|
||||
// 检查分辨率
|
||||
if (
|
||||
params.resolution &&
|
||||
@ -385,6 +390,10 @@ function buildVideoOptions(
|
||||
watermark: params.watermark ?? true,
|
||||
cameraFixed: params.cameraFixed ?? false,
|
||||
fps: 24,
|
||||
seed: params.seed,
|
||||
returnLastFrame: params.returnLastFrame,
|
||||
referenceAudios: params.referenceAudios,
|
||||
generateAudio: params.generateAudio,
|
||||
};
|
||||
|
||||
// 根据模式处理图片输入
|
||||
@ -394,7 +403,7 @@ function buildVideoOptions(
|
||||
options.firstFrameImage = images[0];
|
||||
options.lastFrameImage = images[1];
|
||||
} else if (mode === "reference-images" && images && images.length > 0) {
|
||||
options.referenceImages = images.slice(0, 4);
|
||||
options.referenceImages = images;
|
||||
}
|
||||
|
||||
return options;
|
||||
@ -402,7 +411,7 @@ function buildVideoOptions(
|
||||
|
||||
async function resolvePreferredVideoParentModelId(preferredModelRef?: string) {
|
||||
const normalizedRef = String(preferredModelRef || "").trim();
|
||||
if (!normalizedRef) return undefined;
|
||||
if (!normalizedRef || normalizedRef === "auto") return undefined;
|
||||
|
||||
const models = await getEnabledAiModelsWithProvider();
|
||||
const matched = models.find(
|
||||
@ -411,10 +420,7 @@ async function resolvePreferredVideoParentModelId(preferredModelRef?: string) {
|
||||
(item.model.id === normalizedRef || item.model.modelId === normalizedRef),
|
||||
);
|
||||
|
||||
if (!matched) return undefined;
|
||||
if (matched.model.isSubModel && matched.model.parentModelId) {
|
||||
return matched.model.parentModelId;
|
||||
}
|
||||
if (!matched) throw new Error("所选视频模型不可用，请重新选择视频模型");
|
||||
return matched.model.id;
|
||||
}
|
||||
|
||||
@ -608,6 +614,7 @@ async function executeVip98kVideoGeneration(params: {
|
||||
generationParams: Partial<VideoGenerationOptions>;
|
||||
}): Promise<ExecuteDirectVideoGenerationResult> {
|
||||
const selected = params.selected.model;
|
||||
params.generationParams.duration ??= params.selected.capabilities.durationOptions?.[0] || params.selected.capabilities.durationRange?.min || 5;
|
||||
const languageModel = await getLanguageModelById(selected.model.id);
|
||||
if (!languageModel || typeof (languageModel as any).doGenerate !== "function") {
|
||||
throw new Error("所选 VIP98K 视频模型初始化失败");
|
||||
@ -637,6 +644,7 @@ async function executeVip98kVideoGeneration(params: {
|
||||
cameraFixed: params.generationParams.cameraFixed,
|
||||
seed: params.generationParams.seed,
|
||||
returnLastFrame: params.generationParams.returnLastFrame,
|
||||
generateAudio: params.generationParams.generateAudio,
|
||||
},
|
||||
});
|
||||
if (result?.finishReason === "error") {
|
||||
@ -673,37 +681,16 @@ export async function executeDirectVideoGeneration(
|
||||
const preferredParentModelId = await resolvePreferredVideoParentModelId(
|
||||
params.preferredModelRef,
|
||||
);
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const fallbackParentModel =
|
||||
allModels.find(
|
||||
(item) =>
|
||||
isVolcanoVideoModel(item) &&
|
||||
!item.model.isSubModel &&
|
||||
item.model.modelId === "volcano-video-master",
|
||||
) ||
|
||||
allModels.find(
|
||||
(item) =>
|
||||
isDirectExecutableVideoModel(item) &&
|
||||
!item.model.isSubModel,
|
||||
);
|
||||
|
||||
const parentModelId =
|
||||
preferredParentModelId ||
|
||||
fallbackParentModel?.model.id ||
|
||||
allModels.find(
|
||||
(item) =>
|
||||
isDirectExecutableVideoModel(item) &&
|
||||
item.model.isEnabled &&
|
||||
item.model.isSubModel &&
|
||||
item.model.parentModelId,
|
||||
)?.model.parentModelId ||
|
||||
undefined;
|
||||
validateCanvasVideoInputs(params.mode, params.images);
|
||||
const parentModelId = preferredParentModelId;
|
||||
|
||||
const generationParams = {
|
||||
referenceAudios: params.audioUrls,
|
||||
generateAudio: params.generateAudio,
|
||||
mode: params.mode,
|
||||
resolution: params.resolution || "720p",
|
||||
ratio: params.ratio || "16:9",
|
||||
duration: params.duration || 5,
|
||||
duration: params.duration,
|
||||
watermark: params.watermark ?? true,
|
||||
cameraFixed: params.cameraFixed ?? false,
|
||||
seed: params.seed,
|
||||
@ -717,6 +704,7 @@ export async function executeDirectVideoGeneration(
|
||||
generationParams,
|
||||
);
|
||||
if (preferredVip98kModel) {
|
||||
if (params.audioUrls?.length) throw new Error("所选模型不支持音频参考，请选择音频生视频模型");
|
||||
return executeVip98kVideoGeneration({
|
||||
request: params,
|
||||
prompt,
|
||||
@ -725,39 +713,18 @@ export async function executeDirectVideoGeneration(
|
||||
});
|
||||
}
|
||||
|
||||
let selectedModelConfig: { model: any; capabilities: VideoModelCapabilities };
|
||||
try {
|
||||
selectedModelConfig = await getCompatibleModel(
|
||||
params.mode,
|
||||
parentModelId,
|
||||
generationParams,
|
||||
);
|
||||
} catch (preferredModelError) {
|
||||
if (parentModelId) {
|
||||
if (
|
||||
preferredModelError instanceof Error &&
|
||||
/不支持|没有模型支持/.test(preferredModelError.message)
|
||||
) {
|
||||
throw preferredModelError;
|
||||
}
|
||||
console.warn(
|
||||
"[Generate Video Tool] Preferred parent model selection failed, fallback to global compatible model:",
|
||||
preferredModelError,
|
||||
);
|
||||
selectedModelConfig = await getCompatibleModel(
|
||||
params.mode,
|
||||
undefined,
|
||||
generationParams,
|
||||
);
|
||||
} else {
|
||||
throw preferredModelError;
|
||||
}
|
||||
}
|
||||
const selectedModelConfig = await getCompatibleModel(params.mode, parentModelId, generationParams);
|
||||
|
||||
const { model, capabilities } = selectedModelConfig;
|
||||
if (params.audioUrls?.length && !supportsCanvasAudioVideo({...model.model,providerType:model.provider.type})) throw new Error("当前没有可用的音频生视频模型");
|
||||
generationParams.duration ??= capabilities.durationOptions?.[0] || capabilities.durationRange?.min || 5;
|
||||
|
||||
validateParamsAgainstCapabilities(generationParams, capabilities);
|
||||
|
||||
if (isVip98kVideoModel(model)) {
|
||||
return executeVip98kVideoGeneration({request: params, prompt, selected: selectedModelConfig, generationParams});
|
||||
}
|
||||
|
||||
const options = buildVideoOptions(
|
||||
params.mode,
|
||||
params.images,
|
||||
|
||||
@ -572,7 +572,7 @@ function getSupportedVideoResolutions(model: Vip98kCatalogModel) {
|
||||
|
||||
function getDurations(model: Vip98kCatalogModel) {
|
||||
return parseNumberArray(
|
||||
model.durations || model.durationOptions || model.duration_options,
|
||||
model.durations || model.durationOptions || model.duration_options || toRecord((model as any).duration)?.allowed_seconds,
|
||||
);
|
||||
}
|
||||
|
||||
@ -831,16 +831,21 @@ function createVideoPreset(
|
||||
parseNumber(model.defaultDuration || model.default_duration) ||
|
||||
durations[durations.length - 1] ||
|
||||
8;
|
||||
const supportsReferenceImages =
|
||||
capabilities.includes("image-to-video") ||
|
||||
inputModes.includes("url") ||
|
||||
inputModes.includes("file");
|
||||
const invocation = getInvocationCandidates(model)[0];
|
||||
const schema = toRecord(toRecord(invocation?.input_schema)?.properties) || {};
|
||||
const keys = new Set(getInvocationRequestKeys(model));
|
||||
const imageModes = parseStringArray(toRecord(schema.mode_image)?.enum);
|
||||
const supportsReferenceImages = capabilities.includes("image-to-video") || keys.has("file-urls") || keys.has("image") || keys.has("image-url");
|
||||
const supportsFramePair = imageModes.includes("frame");
|
||||
const supportsIngredients = imageModes.includes("ingredient");
|
||||
const supportedModes = [
|
||||
...(capabilities.includes("text-to-video") || capabilities.length === 0
|
||||
? ["text-to-video"]
|
||||
: []),
|
||||
...(supportsReferenceImages ? ["image-to-video", "first-frame"] : []),
|
||||
...(capabilities.includes("text-to-video") || keys.has("prompt") ? ["text-to-video"] : []),
|
||||
...(supportsReferenceImages ? ["first-frame"] : []),
|
||||
...(supportsFramePair ? ["first-last-frame"] : []),
|
||||
...(supportsIngredients ? ["reference-images"] : []),
|
||||
];
|
||||
const ratios = parseStringArray(toRecord(schema.aspect_ratio)?.enum);
|
||||
const maxReferenceImages = Number(toRecord(schema.file_urls)?.maxItems) || (supportsReferenceImages ? 1 : 0);
|
||||
|
||||
return {
|
||||
modelId,
|
||||
@ -860,7 +865,7 @@ function createVideoPreset(
|
||||
inputModes,
|
||||
outputModes,
|
||||
supportedResolutions,
|
||||
supportedRatios: [...DEFAULT_VIDEO_RATIOS],
|
||||
supportedRatios: ratios.length ? ratios : [...DEFAULT_VIDEO_RATIOS],
|
||||
durationOptions: durations,
|
||||
durationRange: durations.length
|
||||
? { min: durations[0], max: durations[durations.length - 1] }
|
||||
@ -868,7 +873,7 @@ function createVideoPreset(
|
||||
defaultDuration,
|
||||
supportWatermark: false,
|
||||
supportCameraFixed: false,
|
||||
maxReferenceImages: supportsReferenceImages ? 4 : 0,
|
||||
maxReferenceImages,
|
||||
vip98k: getVip98kMetadata(model, "video"),
|
||||
},
|
||||
defaultParams: {
|
||||
|
||||
@ -219,9 +219,10 @@ function invocationTargetsModel(
|
||||
export function getVip98kImageBindingKeys(invocation: Vip98kTaskInvocation | null) {
|
||||
if (!invocation) return [];
|
||||
const body = invocation.requestExample || {};
|
||||
const properties = (invocation.inputSchema?.properties || {}) as Record<string, unknown>;
|
||||
return [
|
||||
...IMAGE_ARRAY_INPUT_KEYS.filter((key) => key in body),
|
||||
...IMAGE_SINGLE_INPUT_KEYS.filter((key) => key in body),
|
||||
...IMAGE_ARRAY_INPUT_KEYS.filter((key) => key in body || key in properties),
|
||||
...IMAGE_SINGLE_INPUT_KEYS.filter((key) => key in body || key in properties),
|
||||
...(Array.isArray(body.video_clips) ? ["video_clips" as const] : []),
|
||||
];
|
||||
}
|
||||
|
||||
@ -4907,6 +4907,7 @@ export async function generateCanvasImage(
|
||||
: imageModels.find((m) => m.model.isDefault) || imageModels[0];
|
||||
|
||||
if (!selectedModel) {
|
||||
if (model && model !== "auto" && request.allowFailover === false) throw new Error("指定图片模型不存在或未启用，请重新选择模型");
|
||||
selectedModel = imageModels[0];
|
||||
}
|
||||
|
||||
|
||||
@ -95,6 +95,16 @@ export type VideoCanvasFunctionConfig = {
|
||||
};
|
||||
|
||||
const defaultFunctions: VideoCanvasFunctionDefinition[] = [
|
||||
...[
|
||||
["character-face-three-view", "角色脸部三视图", "以参考图片的角色为唯一身份依据，生成横向排布的正面、左侧面、右侧面脸部三视图。保持五官、肤色、发型、饰品和年龄一致，头肩构图，纯净背景，三个角度大小一致。"],
|
||||
["character-design-sheet", "角色设定图", "根据参考角色生成完整角色设定图，包含正面、侧面、背面全身及服饰配件细节。所有视图保持同一身份、体型和服装材质，以干净背景和清楚分区呈现。"],
|
||||
["scene-design-sheet", "场景设定图", "根据参考场景生成场景设定图，包含场景全貌、相邻机位视角、关键陈设与材质细节。保持空间布局、建筑结构、光照时段与美术风格一致。"],
|
||||
["product-design-sheet", "产品设定图", "根据参考产品生成产品设定图，包含正面、侧面、背面和关键结构细节。保持外形、数量、配色、材质、比例及已有标识，不增添不存在的部件，统一干净背景与柔和光照。"],
|
||||
].map(([key, label, prompt]): VideoCanvasFunctionDefinition => ({
|
||||
key, label, prompt: `${prompt}输出单张 16:9 图片，不添加文字或水印。`,
|
||||
kind: "image-edit", route: "canvas-generate-image", enabled: true,
|
||||
description: `${label}，使用支持参考图的现有图片编辑模型。`,
|
||||
})),
|
||||
{
|
||||
key: "story-beat-grid-4",
|
||||
label: "剧情推演四宫格",
|
||||
@ -331,7 +341,7 @@ const defaultFunctions: VideoCanvasFunctionDefinition[] = [
|
||||
route: "canvas-generate-video",
|
||||
enabled: true,
|
||||
prompt: "根据上传音频生成对应场景画面，镜头语言、节奏、音乐匹配情绪变化，电影级质感。",
|
||||
description: "官方为音频+参考图到视频；当前先接视频生成提示词和参考图，音频节拍解析接口后续补充。",
|
||||
description: "音频与参考图一同传入支持音频参考的视频模型；没有兼容模型时明确提示，不降级为普通图生视频。",
|
||||
requiresBackend: true,
|
||||
},
|
||||
];
|
||||
|
||||
39
lib/video-canvas/download-media.ts
Normal file
39
lib/video-canvas/download-media.ts
Normal file
@ -0,0 +1,39 @@
|
||||
import {lookup} from 'node:dns/promises';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import {createWriteStream} from 'node:fs';
|
||||
import {pipeline} from 'node:stream/promises';
|
||||
import {Transform} from 'node:stream';
|
||||
|
||||
const MAX_BYTES=125*1024*1024;
|
||||
export function isPublicMediaIpv4(address:string) {
|
||||
const p=address.split('.').map(Number);
|
||||
return p.length===4 && p.every(v=>Number.isInteger(v)&&v>=0&&v<=255) &&
|
||||
![0,10,127].includes(p[0]) && p[0]<224 &&
|
||||
!(p[0]===169&&p[1]===254) && !(p[0]===172&&p[1]>=16&&p[1]<=31) &&
|
||||
!(p[0]===192&&(p[1]===168||p[1]===0)) && !(p[0]===100&&p[1]>=64&&p[1]<=127) &&
|
||||
!(p[0]===198&&(p[1]===18||p[1]===19));
|
||||
}
|
||||
export async function downloadCanvasMedia(url:string,output:string,redirects=0):Promise<void> {
|
||||
const target=new URL(url);
|
||||
if(!['http:','https:'].includes(target.protocol)||target.username||target.password||redirects>4) throw new Error('视频地址无效');
|
||||
const addresses=await lookup(target.hostname,{family:4,all:true});
|
||||
if(!addresses.length||addresses.some(v=>!isPublicMediaIpv4(v.address))) throw new Error('视频地址不可访问');
|
||||
// Pin the validated address while preserving Host and TLS server name.
|
||||
const response=await new Promise<http.IncomingMessage>((resolve,reject)=>{
|
||||
const request=(target.protocol==='https:'?https:http).get({
|
||||
protocol:target.protocol,hostname:addresses[0].address,port:target.port||undefined,
|
||||
path:target.pathname+target.search,headers:{Host:target.host},servername:target.hostname,
|
||||
signal:AbortSignal.timeout(90000),
|
||||
},resolve);
|
||||
request.on('error',reject);
|
||||
});
|
||||
if([301,302,303,307,308].includes(response.statusCode||0)&&response.headers.location) {
|
||||
response.destroy();
|
||||
return downloadCanvasMedia(new URL(response.headers.location,target).href,output,redirects+1);
|
||||
}
|
||||
if(response.statusCode!==200||Number(response.headers['content-length']||0)>MAX_BYTES) {response.destroy();throw new Error('视频下载失败或超过 125 MiB 处理上限');}
|
||||
let bytes=0;
|
||||
const limit=new Transform({transform(chunk,_encoding,callback){bytes+=chunk.length;callback(bytes>MAX_BYTES?new Error('视频超过 125 MiB 处理上限'):null,chunk);}});
|
||||
await pipeline(response,limit,createWriteStream(output));
|
||||
}
|
||||
37
lib/video-canvas/media-processing.ts
Normal file
37
lib/video-canvas/media-processing.ts
Normal file
@ -0,0 +1,37 @@
|
||||
import {execFile} from 'node:child_process';
|
||||
import {promisify} from 'node:util';
|
||||
|
||||
export type MediaRect = {x:number;y:number;width:number;height:number};
|
||||
export function toVideoPixelRect(rect:MediaRect,width:number,height:number) {
|
||||
if (![rect.x,rect.y,rect.width,rect.height,width,height].every(Number.isFinite) || rect.x<0 || rect.y<0 || rect.width<=0 || rect.height<=0 || rect.x+rect.width>1.001 || rect.y+rect.height>1.001) throw new Error('裁剪区域无效');
|
||||
const even=(n:number)=>Math.floor(n/2)*2;
|
||||
const x=even(rect.x*width), y=even(rect.y*height);
|
||||
const w=Math.min(even(rect.width*width),even(width-x)), h=Math.min(even(rect.height*height),even(height-y));
|
||||
if(w<2||h<2) throw new Error('裁剪区域太小');
|
||||
return {x,y,width:w,height:h};
|
||||
}
|
||||
const run=promisify(execFile);
|
||||
const inputSafety=['-protocol_whitelist','file','-format_whitelist','mov,matroska,webm,avi,mpeg,mpegts,flv,ogg'];
|
||||
export async function processVideoFile(input:string,output:string,operation:'crop'|'extract-audio'|'silent-video',rect?:MediaRect) {
|
||||
const {stdout}=await run('ffprobe',['-v','error',...inputSafety,'-show_streams','-show_format','-of','json',input],{timeout:15000,maxBuffer:1024*1024});
|
||||
const info=JSON.parse(stdout), video=info.streams?.find((s:any)=>s.codec_type==='video');
|
||||
if(!video) throw new Error('素材中没有视频画面');
|
||||
if (!(Number(video.width)>0 && Number(video.height)>0) || Number(video.width)*Number(video.height)>33554432) throw new Error('视频尺寸不受支持，请使用 8K 以内的视频');
|
||||
const duration=Number(info.format?.duration || 0);
|
||||
if(!Number.isFinite(duration)||duration<=0||duration>600) throw new Error('本地视频处理支持 10 分钟以内的视频');
|
||||
let args:string[];
|
||||
let dimensions={width:Number(video.width),height:Number(video.height)};
|
||||
if(operation==='extract-audio') {
|
||||
if(!info.streams.some((s:any)=>s.codec_type==='audio')) throw new Error('原视频不包含音轨');
|
||||
args=['-map','0:a:0','-vn','-c:a','aac','-b:a','192k'];
|
||||
} else if(operation==='silent-video') {
|
||||
args=['-map','0:v:0','-an','-c:v','copy','-movflags','+faststart'];
|
||||
} else {
|
||||
if(!rect) throw new Error('请选择裁剪区域');
|
||||
const crop=toVideoPixelRect(rect,dimensions.width,dimensions.height);
|
||||
dimensions=crop;
|
||||
args=['-map','0:v:0','-map','0:a?','-vf',`crop=${crop.width}:${crop.height}:${crop.x}:${crop.y}`,'-c:v','libx264','-preset','veryfast','-crf','20','-pix_fmt','yuv420p','-c:a','aac','-movflags','+faststart'];
|
||||
}
|
||||
await run('ffmpeg',['-nostdin','-y','-v','error','-threads','2','-filter_threads','1',...inputSafety,'-i',input,...args,'-threads','2',output],{timeout:180000,maxBuffer:1024*1024});
|
||||
return {duration,...dimensions};
|
||||
}
|
||||
109
lib/video-canvas/model-matching.ts
Normal file
109
lib/video-canvas/model-matching.ts
Normal file
@ -0,0 +1,109 @@
|
||||
import {
|
||||
is3DModelDescriptor,
|
||||
isImageGenerationModelDescriptor,
|
||||
isVideoModelDescriptor,
|
||||
} from "@/lib/ai/model-classification";
|
||||
import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability";
|
||||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||
import type { VideoGenerationMode } from "@/types/ai";
|
||||
|
||||
export type CanvasModelKind = "image" | "image-edit" | "video" | "audio-video" | "multimodal" | "text" | "any";
|
||||
export type CanvasModelDescriptor = {
|
||||
id?: string;
|
||||
modelId?: string | null;
|
||||
displayName?: string | null;
|
||||
providerType?: string | null;
|
||||
supportsVision?: boolean | null;
|
||||
supportsImageGen?: boolean | null;
|
||||
isMultimodal?: boolean | null;
|
||||
isEnabled?: boolean;
|
||||
selectable?: boolean;
|
||||
capabilities?: unknown;
|
||||
defaultParams?: unknown;
|
||||
imageGenConfig?: unknown;
|
||||
};
|
||||
|
||||
function record(value: unknown): Record<string, any> {
|
||||
return value && typeof value === "object" ? value as Record<string, any> : {};
|
||||
}
|
||||
|
||||
export function isCanvasLanguageModel(model: CanvasModelDescriptor) {
|
||||
const caps = record(model.capabilities);
|
||||
return !is3DModelDescriptor(model) && !isImageGenerationModelDescriptor(model) &&
|
||||
!isVideoModelDescriptor(model) && !/^(audio|speech|tts|asr|embedding|rerank|image-tool|video-tool|world|model-3d)$/i.test(String(caps.kind || caps.mediaType || "")) &&
|
||||
!/world[-_]generation|text-to-3d|image-to-3d/i.test(model.modelId || "");
|
||||
}
|
||||
|
||||
export function isCanvasVisionModel(model: CanvasModelDescriptor) {
|
||||
return isCanvasLanguageModel(model) && Boolean(
|
||||
model.supportsVision || model.isMultimodal || isAiModelMultimodal(model)
|
||||
);
|
||||
}
|
||||
|
||||
export function getCanvasVideoModes(model: CanvasModelDescriptor): VideoGenerationMode[] {
|
||||
if (!isVideoModelDescriptor(model)) return [];
|
||||
const raw = record(model.capabilities).supportedModes;
|
||||
if (!Array.isArray(raw)) return [];
|
||||
const aliases: Record<string, VideoGenerationMode> = {
|
||||
"text-to-video": "text-to-video", "text2video": "text-to-video",
|
||||
"image-to-video": "first-frame", "image2video": "first-frame", "first-frame": "first-frame",
|
||||
"first-last-frame": "first-last-frame", "frames2video": "first-last-frame",
|
||||
"reference-images": "reference-images", "reference-to-video": "reference-images",
|
||||
};
|
||||
return [...new Set(raw.map(value => aliases[String(value)]).filter(Boolean))];
|
||||
}
|
||||
|
||||
// Only adapters actually handled by the canvas video endpoint are selectable here.
|
||||
export function isExecutableCanvasVideoModel(model: CanvasModelDescriptor) {
|
||||
if (!isVideoModelDescriptor(model)) return false;
|
||||
if (["volcano-video", "doubao-video", "atlascloud", "vip98k"].includes(model.providerType || "")) return true;
|
||||
const caps = record(model.capabilities);
|
||||
return model.providerType === "custom" && caps.kind === "video" &&
|
||||
(caps.provider === "agnes-ai" || /^agnes-video/i.test(model.modelId || ""));
|
||||
}
|
||||
|
||||
export function supportsCanvasAudioVideo(model: CanvasModelDescriptor) {
|
||||
const runtime = record(record(model.imageGenConfig).atlasCloud);
|
||||
return isExecutableCanvasVideoModel(model) && model.providerType === "atlascloud" &&
|
||||
Array.isArray(runtime.propertyNames) && runtime.propertyNames.includes("reference_audios");
|
||||
}
|
||||
|
||||
export function filterCanvasModels<T extends CanvasModelDescriptor>(
|
||||
models: T[], kind: CanvasModelKind, mode?: VideoGenerationMode,
|
||||
) {
|
||||
return models.filter(model => {
|
||||
if (model.isEnabled === false || model.selectable === false) return false;
|
||||
if (kind === "audio-video") return supportsCanvasAudioVideo(model);
|
||||
if (kind === "video") return isExecutableCanvasVideoModel(model) && getCanvasVideoModes(model).length > 0 &&
|
||||
(!mode || getCanvasVideoModes(model).includes(mode));
|
||||
if (kind === "image" || kind === "image-edit") return isImageGenerationModelDescriptor(model) &&
|
||||
(kind !== "image-edit" || !modelLacksReferenceImageInputSupport(model, {type: model.providerType}));
|
||||
if (kind === "multimodal") return isCanvasVisionModel(model);
|
||||
if (kind === "text") return isCanvasLanguageModel(model);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export function getCanvasVideoParameters(model: CanvasModelDescriptor | undefined, mode?: VideoGenerationMode) {
|
||||
const caps = record(model?.capabilities);
|
||||
const scoped = record(record(caps.modeConstraints || caps.constraints)[mode || ""]);
|
||||
const strings = (value: unknown) => Array.isArray(value) ? [...new Set(value.map(String).filter(Boolean))] : [];
|
||||
const durationRange = record(caps.durationRange);
|
||||
const min = Number(durationRange.min), max = Number(durationRange.max);
|
||||
const configured = record((model as CanvasModelDescriptor & {imageGenConfig?: unknown})?.imageGenConfig).videoDurationOptions;
|
||||
const durations = Array.isArray(scoped.durationOptions) ? scoped.durationOptions : Array.isArray(configured) && configured.length ? configured : caps.durationOptions;
|
||||
return {
|
||||
ratios: strings(scoped.ratioOptions || caps.supportedRatios),
|
||||
resolutions: strings(scoped.resolutionOptions || caps.supportedResolutions),
|
||||
durations: Array.isArray(durations) ? [...new Set(durations.map(Number).filter(n => Number.isFinite(n) && n > 0))].sort((a,b) => a-b) :
|
||||
min > 0 && max >= min && max-min <= 120 ? Array.from({length: max-min+1}, (_,i) => min+i) : [],
|
||||
supportsAudio: caps.supportGenerateAudio === true || caps.supportsAudio === true || caps.generateAudio === true || strings(record(record(model?.imageGenConfig).atlasCloud).propertyNames).includes("generate_audio"),
|
||||
};
|
||||
}
|
||||
|
||||
export function validateCanvasVideoInputs(mode: VideoGenerationMode, images: string[] = []) {
|
||||
if (mode === "text-to-video" && images.length !== 0) throw new Error("文生视频不能包含参考图片，请切换生成模式");
|
||||
if (mode === "first-frame" && images.length !== 1) throw new Error("首帧生视频需要恰好一张参考图片");
|
||||
if (mode === "first-last-frame" && images.length !== 2) throw new Error("首尾帧生视频需要恰好两张参考图片");
|
||||
if (mode === "reference-images" && images.length === 0) throw new Error("参考生视频需要至少一张参考图片");
|
||||
}
|
||||
@ -3,6 +3,7 @@ import { defineConfig } from "@playwright/test";
|
||||
export default defineConfig({
|
||||
testDir: "./tests",
|
||||
testMatch: [
|
||||
/routes\/video-canvas-model-matching\.test\.ts/,
|
||||
/routes\/canvas-chat-helpers\.test\.ts/,
|
||||
/routes\/agent-main-ai-template-entry\.test\.ts/,
|
||||
/routes\/canvas-agent-operations\.test\.ts/,
|
||||
|
||||
@ -397,6 +397,7 @@ export const imageGenerationService = {
|
||||
|
||||
// AI 生成图片
|
||||
async generateImage(params: {
|
||||
allowFailover?: boolean;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
model?: string;
|
||||
@ -1151,6 +1152,8 @@ export const videoGenerationService = {
|
||||
prompt: string;
|
||||
mode?: VideoGenerationMode;
|
||||
images?: string[];
|
||||
audioUrls?: string[];
|
||||
generateAudio?: boolean;
|
||||
resolution?: VideoResolution;
|
||||
ratio?: VideoRatio;
|
||||
duration?: number;
|
||||
|
||||
71
tests/routes/video-canvas-model-matching.test.ts
Normal file
71
tests/routes/video-canvas-model-matching.test.ts
Normal file
@ -0,0 +1,71 @@
|
||||
import {test, expect} from '@playwright/test';
|
||||
import {filterCanvasModels, getCanvasVideoModes, getCanvasVideoParameters, validateCanvasVideoInputs} from '@/lib/video-canvas/model-matching';
|
||||
|
||||
const language = {id: 'text', modelId: 'agnes-1.5-flash', providerType:'custom', supportsVision:true, capabilities:{kind:'text', provider:'agnes-ai'}};
|
||||
const image = {id:'image', modelId:'gpt-image-2', providerType:'vip98k', supportsImageGen:true, supportsVision:true, capabilities:{kind:'image'}};
|
||||
const threeD = {id:'3d', modelId:'tencent/hunyuan3d-pro/image-to-3d', providerType:'atlascloud', supportsVision:true, capabilities:{kind:'3d'}};
|
||||
const video = {id:'video', modelId:'bytedance/seedance-2.0-mini/image-to-video', providerType:'atlascloud', capabilities:{kind:'video', supportedModes:['first-frame','first-last-frame'], supportedResolutions:['720p','1080p'], supportedRatios:['16:9','9:16'], durationOptions:[5,10]}};
|
||||
|
||||
test('never offers text, image or 3D models when video is unavailable', () => {
|
||||
expect(filterCanvasModels([language,image,threeD], 'video')).toEqual([]);
|
||||
});
|
||||
test('vision analysis excludes image generation and 3D models even when supportsVision is true', () => {
|
||||
expect(filterCanvasModels([language,image,threeD,video], 'multimodal')).toEqual([language]);
|
||||
});
|
||||
test('video choices honor enabled status, executable adapter and exact input mode', () => {
|
||||
expect(filterCanvasModels([video], 'video','text-to-video')).toEqual([]);
|
||||
expect(filterCanvasModels([video], 'video','first-last-frame')).toEqual([video]);
|
||||
expect(filterCanvasModels([{...video,isEnabled:false}], 'video')).toEqual([]);
|
||||
expect(filterCanvasModels([{...video,providerType:'unimplemented'}], 'video')).toEqual([]);
|
||||
});
|
||||
test('normalizes provider mode aliases without inventing unknown capabilities', () => {
|
||||
expect(getCanvasVideoModes({...video, capabilities:{kind:'video',supportedModes:['image-to-video','reference-to-video','made-up']}})).toEqual(['first-frame','reference-images']);
|
||||
expect(getCanvasVideoModes({...video,capabilities:{kind:'video'}})).toEqual([]);
|
||||
});
|
||||
test('image editing rejects a model that does not accept references', () => {
|
||||
expect(filterCanvasModels([{...image,supportsVision:false}], 'image-edit')).toEqual([]);
|
||||
expect(filterCanvasModels([image], 'image-edit')).toEqual([image]);
|
||||
});
|
||||
test('mode parameter constraints override generic choices and never add 4K', () => {
|
||||
const options = getCanvasVideoParameters({...video,capabilities:{...video.capabilities,modeConstraints:{'first-last-frame':{durationOptions:[5],resolutionOptions:['720p'],ratioOptions:['16:9']}}}},'first-last-frame');
|
||||
expect(options).toMatchObject({durations:[5],resolutions:['720p'],ratios:['16:9']});
|
||||
expect(getCanvasVideoParameters(undefined).resolutions).toEqual([]);
|
||||
});
|
||||
test('preserves configured discrete duration choices', () => {
|
||||
expect(getCanvasVideoParameters(video).durations).toEqual([5,10]);
|
||||
});
|
||||
test('rejects missing tail frame and references silently attached to text-to-video', () => {
|
||||
expect(()=>validateCanvasVideoInputs('first-last-frame',['head'])).toThrow('两张');
|
||||
expect(()=>validateCanvasVideoInputs('text-to-video',['head'])).toThrow('切换');
|
||||
expect(()=>validateCanvasVideoInputs('reference-images',[])).toThrow('至少');
|
||||
expect(()=>validateCanvasVideoInputs('first-frame',['head','tail'])).toThrow('一张');
|
||||
expect(()=>validateCanvasVideoInputs('first-last-frame',['head','tail'])).not.toThrow();
|
||||
});
|
||||
|
||||
import {buildVip98kModelPresets} from '@/lib/ai/vip98k-models';
|
||||
import {getVip98kImageBindingKeys, selectVip98kTaskInvocation} from '@/lib/ai/vip98k-task-invocation';
|
||||
test('reads task schema rather than a text-only example for Veo frame and ingredient inputs', () => {
|
||||
const model:any = {id:'veo-3.1-fast',type:'video',status:'published',available:true,capabilities:['media.tasks'],input_modes:['prompt','url'],output_modes:['video'],supported_resolutions:['720p','1080p'],duration:{allowed_seconds:[8]},invocation:{kind:'task',path:'/extensions/v1/tasks',method:'POST',request_example:{model:'veo-3.1-fast',prompt:'test'},input_schema:{properties:{prompt:{type:'string'},mode_image:{enum:['frame','ingredient']},file_urls:{type:'array',maxItems:3},aspect_ratio:{enum:['16:9']}}}}};
|
||||
const preset = buildVip98kModelPresets([model])[0];
|
||||
expect(preset.capabilities).toMatchObject({supportedModes:['text-to-video','first-frame','first-last-frame','reference-images'],durationOptions:[8],supportedRatios:['16:9'],maxReferenceImages:3});
|
||||
const invocation:any = (preset.capabilities.vip98k as any).invocation;
|
||||
expect(getVip98kImageBindingKeys(invocation)).toContain('file_urls');
|
||||
expect(selectVip98kTaskInvocation({modelId:model.id,configuredInvocation:invocation,imageCount:2})?.path).toBe('/extensions/v1/tasks');
|
||||
});
|
||||
|
||||
import {toVideoPixelRect} from '@/lib/video-canvas/media-processing';
|
||||
import {isPublicMediaIpv4} from '@/lib/video-canvas/download-media';
|
||||
test('video crop preserves selected geometry with even output dimensions', () => {
|
||||
expect(toVideoPixelRect({x:.1,y:.1,width:.8,height:.8},1920,1080)).toEqual({x:192,y:108,width:1536,height:864});
|
||||
expect(()=>toVideoPixelRect({x:.9,y:0,width:.8,height:1},1920,1080)).toThrow();
|
||||
expect(()=>toVideoPixelRect({x:NaN,y:0,width:1,height:1},1920,1080)).toThrow();
|
||||
});
|
||||
test('video downloads reject private, metadata and loopback networks', () => {
|
||||
for(const ip of ['127.0.0.1','10.0.0.1','169.254.169.254','172.16.0.1','192.168.1.1','100.64.0.1','0.0.0.0','224.0.0.1']) expect(isPublicMediaIpv4(ip)).toBe(false);
|
||||
expect(isPublicMediaIpv4('8.8.8.8')).toBe(true);
|
||||
});
|
||||
test('audio-driven video requires a real audio reference contract', () => {
|
||||
expect(filterCanvasModels([video], 'audio-video')).toEqual([]);
|
||||
const audioVideo={...video,imageGenConfig:{atlasCloud:{propertyNames:['reference_audios']}}};
|
||||
expect(filterCanvasModels([audioVideo], 'audio-video')).toEqual([audioVideo]);
|
||||
});
|
||||
@ -68,7 +68,9 @@ export interface VideoGenerationOptions {
|
||||
// 图片输入（可选）
|
||||
firstFrameImage?: string; // 首帧图片 URL/Base64
|
||||
lastFrameImage?: string; // 尾帧图片 URL/Base64
|
||||
referenceImages?: string[]; // 参考图数组（1-4张）
|
||||
referenceImages?: string[]; // 按所选模型限制参考图数量
|
||||
referenceAudios?: string[];
|
||||
generateAudio?: boolean;
|
||||
|
||||
// 视频参数
|
||||
resolution?: VideoResolution;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user