172 lines
7.4 KiB
TypeScript
172 lines
7.4 KiB
TypeScript
import { z } from "zod";
|
||
|
||
export const videoAgentSettingsSchema = z.object({
|
||
modelRef: z.string().max(180).optional(),
|
||
imageModelRef: z.string().max(180).optional(),
|
||
mode: z
|
||
.enum(["auto", "text-to-video", "first-frame", "first-last-frame", "reference-images"])
|
||
.default("auto"),
|
||
duration: z.number().int().min(1).max(120).default(5),
|
||
ratio: z.enum(["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"]).default("16:9"),
|
||
resolution: z.enum(["480p", "720p", "1080p"]).default("720p"),
|
||
});
|
||
export type VideoAgentSettings = z.infer<typeof videoAgentSettingsSchema>;
|
||
export type VideoTaskRequirements = {
|
||
expectsVideo: boolean;
|
||
expectsScript: boolean;
|
||
expectedVideoCount?: number;
|
||
duration?: number;
|
||
ratio?: string;
|
||
};
|
||
export type VideoRuntimeNode = {
|
||
id: string;
|
||
kind: string;
|
||
title: string;
|
||
subtitle?: string;
|
||
inputType?: "image" | "video" | "audio" | "text";
|
||
x: number;
|
||
y: number;
|
||
width: number;
|
||
height: number;
|
||
accent: string;
|
||
draftValues?: Record<string, unknown>;
|
||
};
|
||
export function videoNodeUrl(node: VideoRuntimeNode) {
|
||
const draft = node.draftValues || {};
|
||
return (
|
||
[draft.mediaUrl, ...(Array.isArray(draft.url) ? draft.url : []), draft.sourceUrl].find(
|
||
(value): value is string => typeof value === "string" && /^https?:\/\//i.test(value),
|
||
) || ""
|
||
);
|
||
}
|
||
export function videoNodeType(node: VideoRuntimeNode) {
|
||
return String(
|
||
node.draftValues?.mediaType ||
|
||
node.inputType ||
|
||
(node.kind === "text" || node.kind === "script" ? "text" : ""),
|
||
);
|
||
}
|
||
export function inferVideoTaskRequirements(text: string): VideoTaskRequirements {
|
||
const analysisOnly =
|
||
/(?:只|仅)(?:要|需|做|用)?[^，。；]{0,12}(?:分析|脚本|分镜|方案|提示词)|不要(?:生成|制作|执行)[^，。；]{0,4}视频/.test(
|
||
text,
|
||
);
|
||
const countMatch = text.match(
|
||
/([一二两三四五六七八九十]|\d+)\s*(?:条|段|个|支)\s*(?:\d+[:：]\d+\s*)?(?:成片|短片|视频)/,
|
||
);
|
||
const counts: Record<string, number> = {
|
||
一: 1,
|
||
二: 2,
|
||
两: 2,
|
||
三: 3,
|
||
四: 4,
|
||
五: 5,
|
||
六: 6,
|
||
七: 7,
|
||
八: 8,
|
||
九: 9,
|
||
十: 10,
|
||
};
|
||
const duration = text.match(/(\d+)\s*秒/);
|
||
const ratio = text.match(/(?:^|[^\d])(\d+)\s*[:：]\s*(\d+)(?!\d)/);
|
||
return {
|
||
expectsVideo:
|
||
!analysisOnly &&
|
||
/(?:生成|制作|创作|拍|做|再来|继续|来|出).{0,35}(?:视频|成片|短片)|(?:视频|成片|短片).{0,12}(?:生成|制作)|生成.{0,10}动效|动起来/.test(
|
||
text,
|
||
),
|
||
expectsScript:
|
||
/(?:只要|只需|仅要).{0,8}(?:脚本|分镜)|(?:写|生成|制作|编写|先|做).{0,12}(?:脚本|分镜)|分镜.{0,8}(?:脚本|方案)/.test(
|
||
text,
|
||
),
|
||
...(countMatch ? { expectedVideoCount: counts[countMatch[1]] || Number(countMatch[1]) } : {}),
|
||
...(duration ? { duration: Number(duration[1]) } : {}),
|
||
...(ratio ? { ratio: `${ratio[1]}:${ratio[2]}` } : {}),
|
||
};
|
||
}
|
||
export function resolveVideoMode(mode: VideoAgentSettings["mode"], referenceCount: number) {
|
||
const resolved =
|
||
mode === "auto"
|
||
? referenceCount >= 3
|
||
? "reference-images"
|
||
: referenceCount === 2
|
||
? "first-last-frame"
|
||
: referenceCount === 1
|
||
? "first-frame"
|
||
: "text-to-video"
|
||
: mode;
|
||
const valid =
|
||
resolved === "text-to-video"
|
||
? referenceCount === 0
|
||
: resolved === "first-frame"
|
||
? referenceCount === 1
|
||
: resolved === "first-last-frame"
|
||
? referenceCount === 2
|
||
: referenceCount >= 1 && referenceCount <= 4;
|
||
if (!valid)
|
||
throw new Error("参考图片数量与视频模式不匹配，请检查首帧、尾帧或参考图。不会自动丢弃参考图。");
|
||
return resolved;
|
||
}
|
||
export function restoreVideoRuntimeNodes(data: unknown): VideoRuntimeNode[] {
|
||
const nodes = (
|
||
data as {
|
||
nodes?: Array<{ id?: string; data?: { metadata?: { liblibCloneNode?: VideoRuntimeNode } } }>;
|
||
} | null
|
||
)?.nodes;
|
||
return (nodes || []).flatMap((node) => {
|
||
const clone = node.data?.metadata?.liblibCloneNode;
|
||
return clone && clone.id === node.id ? [clone] : [];
|
||
});
|
||
}
|
||
export function serializeVideoRuntimeNode(node: VideoRuntimeNode) {
|
||
return {
|
||
id: node.id,
|
||
type: "workflow",
|
||
position: { x: node.x, y: node.y },
|
||
size: { width: node.width, height: node.height },
|
||
data: {
|
||
title: node.title,
|
||
description: node.subtitle || "",
|
||
status: "success",
|
||
templateType: `libtv-${node.kind}`,
|
||
metadata: { liblibCloneNode: node },
|
||
},
|
||
connections: [],
|
||
createdAt: new Date().toISOString(),
|
||
updatedAt: new Date().toISOString(),
|
||
};
|
||
}
|
||
export function videoRuntimeSystemPrompt(
|
||
objective: string,
|
||
summary: string,
|
||
supplements: string[],
|
||
context: unknown,
|
||
) {
|
||
return [
|
||
"先用 listVideoCanvasCapabilities 查询全部已授权功能。runVideoCanvasFunction 执行配置预设、图像编辑和素材分析；editVideoCanvas 编辑节点和连线；openVideoCanvasFeature 打开真实面板。权限与功能后端是否已实现、工具栏是否显示独立。",
|
||
"视频截帧、剪辑、裁剪、提取完整音轨和静音使用 processVideoMedia 真实处理；人声与背景音分离没有真实后端，不能用完整音轨冒充分离结果。",
|
||
"未接后端、模拟去字幕/分离/裁剪的界面入口不能声称已处理媒体。文件选择和付费、发布提交需用户操作。",
|
||
"你是本站视频画布 Agent，使用提供的工具连续完成视频创作。先 planCanvasTask 再 readCanvas，随后按依赖执行。",
|
||
"读取真实节点、连接关系和内容定位素材，不猜测 ID。用户指定首尾帧时 referenceNodeIds 的顺序必须是首帧、尾帧。",
|
||
"分析图片或视频用 analyzeVideoMaterials；脚本分镜用 writeVideoScript 写入画布；关键帧用 generateVideoKeyframe；成片用 generateVideoShot。不同镜头用不同工具调用。",
|
||
"沿用用户的视频模型、模式、时长、比例和分辨率。缺少参考帧或关键信息时先询问。不要用虚构素材或擅自切换模式。",
|
||
"writeVideoScript 的每行包含 sceneDescription、imagePrompt、motionPrompt、duration，后续关键帧和视频必须基于这些实际分镜。",
|
||
"当前工具生成独立视频镜头，不具备多片段拼接或音轨合成能力。用户要求这些操作时明确说明，不把多个片段声称为一条已剪辑成片。",
|
||
"不要把方案、排队、脚本或关键帧声称为视频完成。视频工具只有返回实际 video 节点才算完成。保留全部原素材和已有结果。",
|
||
"视频涉及费用，结果不明时停止核对，不盲目重试。用户补充从下一步应用，不假称正在生成的镜头已经改变。",
|
||
"工具失败或验收问题要如实告知。支持分析和脚本任务，不要用户只要脚本却擅自生成视频。最终简短说明交付内容。",
|
||
`原始任务：${objective}`,
|
||
`后续要求（最新优先）：${JSON.stringify(supplements)}`,
|
||
`画布上下文与用户参数：${JSON.stringify(context)}`,
|
||
summary ? `已压缩历史（事实参考）：${summary}` : "",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
/** Browser saves may race with Agent appends: retain task-owned nodes and their connections. */
|
||
export function mergeVideoAgentResults<T extends { id: string }>(local: T[], remote: T[]) {
|
||
const ids = new Set(local.map((node) => node.id));
|
||
return [...local, ...remote.filter((node) => !ids.has(node.id))];
|
||
}
|