ai-chatbot/lib/video-canvas/toolbox-workflows.ts

427 lines
20 KiB
TypeScript

export type VideoCanvasToolboxCategory = {
enabled: boolean;
id: string;
label: string;
};
export type VideoCanvasToolboxTemplate = {
categoryId: string;
coverUrl: string;
enabled: boolean;
id: string;
name: string;
prompt: string;
referenceUrl: string;
resultUrl: string;
};
export type VideoCanvasToolboxLayout = {
groupHeight: number;
groupWidth: number;
placementOffsetX: number;
placementOffsetY: number;
prompt: { height: number; width: number; x: number; y: number };
reference: { height: number; width: number; x: number; y: number };
result: { height: number; width: number; x: number; y: number };
};
export type VideoCanvasToolboxConfig = {
categories: VideoCanvasToolboxCategory[];
enabled: boolean;
helpText: string;
helpUrl: string;
layout: VideoCanvasToolboxLayout;
templates: VideoCanvasToolboxTemplate[];
};
const DEFAULT_RESULT_VIDEO_URL =
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/script-video-reference.mp4";
function sourceUrl(url: string) {
return url.split("?")[0] || url;
}
function template(
id: string,
categoryId: string,
name: string,
coverUrl: string,
prompt: string,
): VideoCanvasToolboxTemplate {
return {
categoryId,
coverUrl,
enabled: true,
id,
name,
prompt,
referenceUrl: sourceUrl(coverUrl),
resultUrl: DEFAULT_RESULT_VIDEO_URL,
};
}
const MY_TOOLBOX_TEMPLATES: VideoCanvasToolboxTemplate[] = [
template(
"left-arc-slide-primary",
"my-toolbox",
"【预设】左弧滑行",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/794154946370bcfd0574b5beb68c95f4d4d57c88.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"电影级人物环绕镜头。镜头从人物正前方开始,沿左侧平滑弧线移动到侧后方;两位人物持续转头追随镜头并保持眼神交流。保持人物身份、服装、木质室内环境和高级时装质感稳定,动作连贯,面部清晰,不发生肢体变形或背景跳变。",
),
template(
"phone-pop-product",
"my-toolbox",
"【预设】电商手机弹出效果",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/4205e8f1b9d5449c9c134a55ab23e475/3f86729b953d5e3f78cf314a20541ef7195786072400aa6af9d33a2dc2405cf6.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"手机产品从画面中心快速弹出并轻微旋转,在高光扫过后稳定定格。保持产品外形、品牌细节和材质一致,背景干净,镜头节奏利落。",
),
template(
"coffee-cup-entry",
"my-toolbox",
"【预设】咖啡杯出场",
"https://libtv-res.liblib.art/upload-images/beda8103d1a8414aa15aa0eb1db95a07/c8c9444584348de6209bee45d9f95a0c2c69932f.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"咖啡杯从桌面侧方平滑滑入,杯口蒸汽缓慢上升,柔和暖光勾勒杯身和液面,最终停在画面中心形成品牌静物镜头。",
),
template(
"orbit-360-showcase",
"my-toolbox",
"【预设】360旋转展示",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/498ebfb6bbf47d17fb8d5449b8eb8029df39cc9f.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"镜头以恒定半径围绕主体完成一圈稳定环绕,速度均匀,主体始终居中,完整展示各个角度并保持材质、结构和光照连续。",
),
template(
"robot-arm-pov-primary",
"my-toolbox",
"【预设】机械臂视角",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/4d251222f8ebdc2cf634cc589fc909c8e69b5185.jpg?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"第一人称镜头沿机械臂运动路径穿行,在金属结构和产品之间形成明显视差,运动精确流畅,保持工业材质和空间结构稳定。",
),
template(
"live-2d-character",
"my-toolbox",
"【预设】Live 2D",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/882cc4bc76db85989fa961fed616cb3ebe201a0c.jpg?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"将单张角色图转为轻量角色动画:自然眨眼、轻微微笑、呼吸和小幅身体摆动。严格保持五官、发型、服装和画风一致。",
),
template(
"pupil-zoom",
"my-toolbox",
"【预设】瞳孔拉近",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/59a02980eda985744952c15bcfd8697bffab14aa.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"镜头从人物近景快速推进到瞳孔极特写,在眼睛反光中显现下一场景并完成自然转场,保持眼部结构和人物身份稳定。",
),
template(
"bird-dissolve",
"my-toolbox",
"【预设】飞鸟解体",
"https://libtv-res.liblib.art/upload-images/beda8103d1a8414aa15aa0eb1db95a07/6676b12a15100018ca7648c85287e007ceb9f9f2.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"主体从边缘开始分解为成群飞鸟,鸟群沿运动方向散开后重新聚合为完整主体,轮廓转换清晰且背景连续。",
),
template(
"break-out-of-box",
"my-toolbox",
"【预设】破盒而出",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/beda8103d1a8414aa15aa0eb1db95a07/84979fa89cac130f874f49e7286a51e13fbccc95a19657ba13fcd7287fedb50a.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"产品冲破包装盒进入画面,碎片向外飞散并快速减速,主体在中心稳定定格,保持包装、产品和品牌细节一致。",
),
template(
"product-hero-entry",
"my-toolbox",
"【预设】商品震撼登场",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/b43d4ec927a2766382d82e95c4020a5e7c3b08aa.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"商品伴随冲击波和局部高光快速进入画面,镜头轻微推进后定格,突出产品轮廓、材质和主视觉信息。",
),
template(
"right-arc-slide",
"my-toolbox",
"【预设】右弧滑行",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/794154946370bcfd0574b5beb68c95f4d4d57c88.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"镜头从人物正前方沿右侧做平滑大弧线运动,人物目光持续追随镜头,保持身份、服装、场景和光线稳定。",
),
template(
"left-arc-slide-secondary",
"my-toolbox",
"【预设】左弧滑行",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/794154946370bcfd0574b5beb68c95f4d4d57c88.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"镜头从人物正前方沿左侧做平滑大弧线运动,人物自然转头保持眼神交流,背景空间和人物外观连续。",
),
template(
"inverted-space",
"my-toolbox",
"【预设】颠倒空间",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/8852de28b54eb90134a11fa526426babdd85b1d3.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"场景围绕主体逐步翻转至上下颠倒,主体保持重心和朝向稳定,空间旋转连续,避免结构断裂和画面跳变。",
),
template(
"anti-gravity-float",
"my-toolbox",
"【预设】反重力漂浮",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/beda8103d1a8414aa15aa0eb1db95a07/abc353b7b28311027d4bf4427531f3025cb1a5161f929b7d414e44ebb9c3d9b8.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"主体和周围小物件缓慢脱离地面悬浮,运动带有轻微惯性和旋转,光影及空间关系保持真实连续。",
),
template(
"particle-dissolve",
"my-toolbox",
"【预设】粒子融解",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/beda8103d1a8414aa15aa0eb1db95a07/95be6a97f91ab6c16081f260cbaaa1df3d823a5ec037b8cc6a970ef05cbe4001.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"主体由外向内融解为细小发光粒子,粒子顺风扩散并逐渐消失,保留主体轮廓演变和背景连续性。",
),
template(
"travel-zoom-in",
"my-toolbox",
"【预设】旅拍转场 zoom in",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/beda8103d1a8414aa15aa0eb1db95a07/71e6713e72ade5531b4eb131e1faf18b1cf27737e9a5fed18378a429503b8034.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"镜头快速推近前景遮挡物,在遮挡充满画面时切换到新地点并继续向前运动,形成连贯旅拍转场。",
),
template(
"travel-zoom-out",
"my-toolbox",
"【预设】旅拍转场 zoom out",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/beda8103d1a8414aa15aa0eb1db95a07/29c1f23b7d8bfa35267e3ddc11f094e63a1c6d5089f883e1b2c9134dd36ef10a.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"镜头从新场景局部快速拉远,在拉远过程中完成地点切换,速度曲线顺滑并保持主体位置衔接。",
),
template(
"travel-rotate-right",
"my-toolbox",
"【预设】旅拍转场 向右旋转",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/beda8103d1a8414aa15aa0eb1db95a07/b49297c4637a024c010501407619e90e5d2c08c9c6600fb2ce52878e1daf71f0.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"手持镜头快速向右旋转,利用运动模糊在旋转中点切换地点,并在新场景逐渐稳定。",
),
template(
"travel-rotate-left",
"my-toolbox",
"【预设】旅拍转场 向左旋转",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/beda8103d1a8414aa15aa0eb1db95a07/005550129798804a81e92fa8382cdb4f0e77615dc64491358ae166380dc8839e.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"手持镜头快速向左旋转,利用运动模糊自然切换地点,结束时回到稳定构图并保持主体连续。",
),
template(
"travel-growth",
"my-toolbox",
"【预设】旅拍转场 生长",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/beda8103d1a8414aa15aa0eb1db95a07/8ad3b1b28b7406aadcfbacf827b7b1ce6b5570ee1434c187091db52b1ff78dd8.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"植物从前景快速生长并覆盖画面,在叶片遮挡中切换到下一地点,随后叶片展开露出完整新场景。",
),
template(
"hero-perspective",
"my-toolbox",
"【预设】英雄视角",
"https://libtv-res.liblib.art/upload-images/be52904b1a974e01af26ccd4fe36b4c0/9cf12958ec85cc870011c017834fe894c13932ad.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"低机位广角镜头缓慢推近主体,强化前景透视和人物气势,保持面部、服装和环境结构稳定。",
),
template(
"ai-fashion-model",
"my-toolbox",
"【预设】AI模特服饰动态展示",
"https://libtv-res.liblib.art/upload-images/9c9ca63a3fd948e8abb9ab96ade97321/a676b7a777d24d0b2bbdc7509109546c9ae3d2d7.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"模特自然转身并展示服装正面、侧面和背面,镜头轻微环绕,严格保持服饰版型、纹理、颜色和人物身份一致。",
),
template(
"robot-arm-pov-secondary",
"my-toolbox",
"【预设】机械臂视角",
"https://libtv-res.liblib.art/upload-images/beda8103d1a8414aa15aa0eb1db95a07/bf72676a252b5397d06fd6b6734f3ede26a1d121.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"镜头模拟机械臂的精确移动轨迹,在主体周围做平滑推拉和转向,突出工业空间、材质细节与机械节奏。",
),
template(
"master-storyboard-dark",
"my-toolbox",
"【预设】大师分镜九宫格-经典暗调",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/9c9ca63a3fd948e8abb9ab96ade97321/082931e3f634ac1b7e90f79fa9d8e66b12a596d0fbe7d1d4ce9ff04acdba8203.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"根据同一角色和场景生成严格 3x3 电影分镜,覆盖远景、中景、近景、俯拍、仰拍和细节镜头,统一经典暗调光影并保持人物一致。",
),
template(
"interior-renovation-preview",
"my-toolbox",
"【预设】AI室内装修效果预览",
"https://libtv-res.liblib.art/sd-gen-save-img/genius_playground/image/9c9ca63a3fd948e8abb9ab96ade97321/fba8fa5f7e53e796c26216a91601bca675051c01b554dc0c8b0dfa25ea94d3da.png?x-oss-process=image/resize,w_280,m_lfit/format,webp/ignore-error,1",
"保持房间结构、门窗和透视不变,平滑展示装修前后材质、家具和灯光变化,形成真实连贯的室内改造预览。",
),
];
const CLASSIC_SCENE_TEMPLATES: VideoCanvasToolboxTemplate[] = [
template(
"king-of-comedy-gunfight",
"stephen-chow",
"喜剧之王-枪战",
"https://libtv-res.liblib.art/upload-images/70a305c50c704a778db114468830617b/aae0d0cafc4fd2e682e72b15d2334ddeebb6871c.webp",
"复刻室内喜剧枪战名场面的节奏与走位,使用快速反应镜头、夸张停顿和清晰动作衔接,保持人物身份及空间连续。",
),
template(
"king-of-comedy-mystery-tycoon",
"stephen-chow",
"喜剧之王-神秘富豪",
"https://libtv-res.liblib.art/upload-images/70a305c50c704a778db114468830617b/e09663b853f2273d628526168663cbef93a1d785.webp",
"以神秘富豪出场为核心,镜头先隐藏身份再逐步揭示人物,利用近景反应和喜剧停顿强化反差。",
),
template(
"god-of-cookery-contest",
"stephen-chow",
"食神-食神大赛",
"https://libtv-res.liblib.art/upload-images/70a305c50c704a778db114468830617b/372645626796f75c4dd9574111e61c9c0497054e.webp",
"复刻烹饪大赛的高强度节奏,组合食材特写、快速操作、评委反应和成品揭晓镜头,保持场景和人物连续。",
),
template(
"god-of-cookery-beef-ball",
"stephen-chow",
"食神-撒尿牛丸",
"https://libtv-res.liblib.art/upload-images/70a305c50c704a778db114468830617b/7f0a71b7fe580c0c04c3ddd1ad22608fde42b915.webp",
"围绕弹性牛丸的夸张展示设计喜剧镜头,使用高速动作、反应特写和定格强调,保持食物外观和场景一致。",
),
template(
"cj7-fix-fan",
"stephen-chow",
"七仔-修风扇",
"https://libtv-res.liblib.art/upload-images/70a305c50c704a778db114468830617b/b2249423467d4a9a18c03c4d4aff7927e696fcc3.webp",
"以角色修理风扇的连续动作为主线,结合道具细节、人物反应和轻喜剧停顿,保持角色造型及室内空间稳定。",
),
];
export const defaultVideoCanvasToolboxConfig: VideoCanvasToolboxConfig = {
categories: [
{ enabled: true, id: "my-toolbox", label: "我的工具箱" },
{ enabled: true, id: "stephen-chow", label: "周星驰经典名场面" },
],
enabled: true,
helpText: "使用工具箱模板加速创作,快速构建你的专属工具箱。",
helpUrl:
"https://liblibai.feishu.cn/wiki/Loxfw6XHziYRk0kKzdjcFfp9nhb#share-EGsydYnauomw6rxz7SAc313Bnfh",
layout: {
groupHeight: 658,
groupWidth: 1493,
placementOffsetX: -175,
placementOffsetY: -175,
prompt: { height: 200, width: 350, x: 525, y: 45 },
reference: { height: 525, width: 350, x: 40, y: 40 },
result: { height: 525, width: 350, x: 1104, y: 93 },
},
templates: [...MY_TOOLBOX_TEMPLATES, ...CLASSIC_SCENE_TEMPLATES],
};
export function getVideoCanvasToolboxGroupOrigin({
layout,
viewport,
viewportHeight,
viewportWidth,
}: {
layout: VideoCanvasToolboxLayout;
viewport: { x: number; y: number; zoom: number };
viewportHeight: number;
viewportWidth: number;
}) {
const zoom = Number.isFinite(viewport.zoom) && viewport.zoom > 0 ? viewport.zoom : 1;
return {
x: (viewportWidth / 2 + layout.placementOffsetX - viewport.x) / zoom,
y: (viewportHeight / 2 + layout.placementOffsetY - viewport.y) / zoom,
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function boundedNumber(value: unknown, fallback: number, min: number, max: number) {
const parsed = Number(value);
return Number.isFinite(parsed) ? Math.max(min, Math.min(max, parsed)) : fallback;
}
function normalizeLayoutItem(
value: unknown,
fallback: VideoCanvasToolboxLayout["reference"],
) {
const incoming = isRecord(value) ? value : {};
return {
height: boundedNumber(incoming.height, fallback.height, 40, 4000),
width: boundedNumber(incoming.width, fallback.width, 40, 4000),
x: boundedNumber(incoming.x, fallback.x, -10_000, 10_000),
y: boundedNumber(incoming.y, fallback.y, -10_000, 10_000),
};
}
export function normalizeVideoCanvasToolboxConfig(
value: unknown,
): VideoCanvasToolboxConfig {
if (!isRecord(value)) return defaultVideoCanvasToolboxConfig;
const incomingCategories = Array.isArray(value.categories) ? value.categories : [];
const categories = incomingCategories
.slice(0, 20)
.map((item, index) => {
const incoming = isRecord(item) ? item : {};
return {
enabled: incoming.enabled !== false,
id: String(incoming.id || `toolbox-category-${index + 1}`).trim().slice(0, 100),
label: String(incoming.label || `分类 ${index + 1}`).trim().slice(0, 100),
};
})
.filter((item) => item.id && item.label);
const resolvedCategories = categories.length
? categories
: defaultVideoCanvasToolboxConfig.categories;
const categoryIds = new Set(resolvedCategories.map((item) => item.id));
const firstCategoryId = resolvedCategories[0]?.id || "my-toolbox";
const incomingTemplates = Array.isArray(value.templates) ? value.templates : [];
const templates = incomingTemplates
.slice(0, 200)
.map((item, index) => {
const incoming = isRecord(item) ? item : {};
const coverUrl = String(incoming.coverUrl || "").trim().slice(0, 4000);
const categoryId = String(incoming.categoryId || firstCategoryId).trim();
return {
categoryId: categoryIds.has(categoryId) ? categoryId : firstCategoryId,
coverUrl,
enabled: incoming.enabled !== false,
id: String(incoming.id || `toolbox-template-${index + 1}`).trim().slice(0, 120),
name: String(incoming.name || `模板 ${index + 1}`).trim().slice(0, 200),
prompt: String(incoming.prompt || "").trim().slice(0, 20_000),
referenceUrl: String(incoming.referenceUrl || sourceUrl(coverUrl))
.trim()
.slice(0, 4000),
resultUrl: String(incoming.resultUrl || DEFAULT_RESULT_VIDEO_URL)
.trim()
.slice(0, 4000),
};
})
.filter((item) => item.id && item.name && item.coverUrl);
const incomingLayout = isRecord(value.layout) ? value.layout : {};
const fallbackLayout = defaultVideoCanvasToolboxConfig.layout;
return {
categories: resolvedCategories,
enabled: value.enabled !== false,
helpText:
String(value.helpText || "").trim().slice(0, 500) ||
defaultVideoCanvasToolboxConfig.helpText,
helpUrl:
String(value.helpUrl || "").trim().slice(0, 2000) ||
defaultVideoCanvasToolboxConfig.helpUrl,
layout: {
groupHeight: boundedNumber(
incomingLayout.groupHeight,
fallbackLayout.groupHeight,
100,
8000,
),
groupWidth: boundedNumber(
incomingLayout.groupWidth,
fallbackLayout.groupWidth,
100,
8000,
),
placementOffsetX: boundedNumber(
incomingLayout.placementOffsetX,
fallbackLayout.placementOffsetX,
-4000,
4000,
),
placementOffsetY: boundedNumber(
incomingLayout.placementOffsetY,
fallbackLayout.placementOffsetY,
-4000,
4000,
),
prompt: normalizeLayoutItem(incomingLayout.prompt, fallbackLayout.prompt),
reference: normalizeLayoutItem(incomingLayout.reference, fallbackLayout.reference),
result: normalizeLayoutItem(incomingLayout.result, fallbackLayout.result),
},
templates: templates.length ? templates : defaultVideoCanvasToolboxConfig.templates,
};
}