Update video canvas home and Aliyun analysis flow
This commit is contained in:
134
parent
e97296a9e8
commit
439af60c2a
@ -1,79 +1,75 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { redirect } from "next/navigation";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { db } from "@/lib/db";
|
||||
import { canvasProject } from "@/lib/db/schema";
|
||||
import { and, desc, eq } from "drizzle-orm";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { VideoCanvasHome, type VideoCanvasHomeProject } from "@/components/canvas/VideoCanvasHome";
|
||||
import { db } from "@/lib/db";
|
||||
import { getSiteSettings } from "@/lib/db/queries";
|
||||
import { canvasProject } from "@/lib/db/schema";
|
||||
|
||||
type VideoCanvasRedirectPageProps = {
|
||||
searchParams?: Promise<Record<string, string | string[] | undefined>>;
|
||||
};
|
||||
type Props = { searchParams?: Promise<Record<string, string | string[] | undefined>> };
|
||||
|
||||
function getSearchParam(
|
||||
params: Record<string, string | string[] | undefined>,
|
||||
key: string
|
||||
) {
|
||||
const value = params[key];
|
||||
function first(value: string | string[] | undefined) {
|
||||
return Array.isArray(value) ? value[0] : value;
|
||||
}
|
||||
|
||||
function buildVideoCanvasUrl(
|
||||
canvasId: string,
|
||||
params: Record<string, string | string[] | undefined>
|
||||
) {
|
||||
const query = new URLSearchParams();
|
||||
query.set("mode", "video");
|
||||
query.set("workspaceType", "video");
|
||||
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
if (
|
||||
key === "new" ||
|
||||
key === "mode" ||
|
||||
key === "workspaceType" ||
|
||||
value === undefined
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((item) => query.append(key, item));
|
||||
continue;
|
||||
}
|
||||
query.set(key, value);
|
||||
function canvasHref(id: string, incoming: Record<string, string | string[] | undefined>) {
|
||||
const params = new URLSearchParams({ mode: "video", workspaceType: "video" });
|
||||
for (const [key, value] of Object.entries(incoming)) {
|
||||
if (["new", "mode", "workspaceType"].includes(key) || value === undefined) continue;
|
||||
for (const item of Array.isArray(value) ? value : [value]) params.append(key, item);
|
||||
}
|
||||
|
||||
return `/canvas/${canvasId}?${query.toString()}`;
|
||||
return `/canvas/${id}?${params.toString()}`;
|
||||
}
|
||||
|
||||
async function getLatestVideoCanvasId(userId: string) {
|
||||
const [latestProject] = await db
|
||||
.select({ id: canvasProject.id })
|
||||
.from(canvasProject)
|
||||
.where(
|
||||
and(
|
||||
eq(canvasProject.userId, userId),
|
||||
eq(canvasProject.workspaceType, "video")
|
||||
)
|
||||
)
|
||||
.orderBy(desc(canvasProject.updatedAt))
|
||||
.limit(1);
|
||||
|
||||
return latestProject?.id || null;
|
||||
function projectPreview(thumbnail: string | null, canvasData: unknown) {
|
||||
const nodes = canvasData && typeof canvasData === "object" && "nodes" in canvasData
|
||||
? (canvasData as { nodes?: unknown }).nodes : null;
|
||||
const candidates = [thumbnail];
|
||||
if (Array.isArray(nodes)) for (const node of [...nodes].reverse()) {
|
||||
if (!node || typeof node !== "object") continue;
|
||||
const item = node as { inputType?: string; draftValues?: Record<string, unknown> };
|
||||
const draft = item.draftValues || {};
|
||||
candidates.push(
|
||||
String(draft.poster || draft.previewImageUrl || draft.thumbnailUrl || draft.resultImage || (item.inputType === "image" ? draft.mediaUrl || (Array.isArray(draft.url) ? draft.url[0] : "") : "") || ""),
|
||||
);
|
||||
}
|
||||
return candidates.find(value => typeof value === "string" && (value.startsWith("https://") || value.startsWith("/"))) || null;
|
||||
}
|
||||
|
||||
export default async function VideoCanvasRedirectPage({
|
||||
searchParams,
|
||||
}: VideoCanvasRedirectPageProps) {
|
||||
const params = (await searchParams) || {};
|
||||
const shouldCreateNew = getSearchParam(params, "new") === "1";
|
||||
|
||||
if (shouldCreateNew) {
|
||||
redirect(buildVideoCanvasUrl(randomUUID(), params));
|
||||
}
|
||||
|
||||
export default async function VideoCanvasHomePage({ searchParams }: Props) {
|
||||
const session = await auth();
|
||||
const latestCanvasId = session?.user?.id
|
||||
? await getLatestVideoCanvasId(session.user.id).catch(() => null)
|
||||
: null;
|
||||
if (!session?.user?.id) redirect("/login?callbackUrl=%2Fvideo-canvas");
|
||||
|
||||
redirect(buildVideoCanvasUrl(latestCanvasId || randomUUID(), params));
|
||||
const params = (await searchParams) || {};
|
||||
if (first(params.new) === "1") {
|
||||
const [created] = await db.insert(canvasProject).values({
|
||||
userId: session.user.id,
|
||||
title: "未命名视频画布",
|
||||
workspaceType: "video",
|
||||
canvasData: { nodes: [], edges: [], viewport: { x: 0, y: 0, zoom: 1 } },
|
||||
}).returning({ id: canvasProject.id });
|
||||
redirect(canvasHref(created.id, params));
|
||||
}
|
||||
|
||||
const [rows, siteSettings] = await Promise.all([db.select({
|
||||
id: canvasProject.id,
|
||||
title: canvasProject.title,
|
||||
thumbnail: canvasProject.thumbnail,
|
||||
canvasData: canvasProject.canvasData,
|
||||
createdAt: canvasProject.createdAt,
|
||||
updatedAt: canvasProject.updatedAt,
|
||||
}).from(canvasProject).where(and(
|
||||
eq(canvasProject.userId, session.user.id),
|
||||
eq(canvasProject.workspaceType, "video"),
|
||||
)).orderBy(desc(canvasProject.updatedAt)).limit(25), getSiteSettings()]);
|
||||
|
||||
const projects: VideoCanvasHomeProject[] = rows.slice(0, 24).map(row => ({
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
thumbnail: projectPreview(row.thumbnail, row.canvasData),
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
updatedAt: row.updatedAt.toISOString(),
|
||||
}));
|
||||
|
||||
return <VideoCanvasHome initialHasMore={rows.length > 24} initialProjects={projects} siteLogo={siteSettings.siteLogo} siteLogoDark={siteSettings.siteLogoDark} siteName={siteSettings.siteName} userName={session.user.name || ""} />;
|
||||
}
|
||||
|
||||
@ -33,15 +33,30 @@ export async function POST(request: Request) {
|
||||
const input = join(dir, "input");
|
||||
const buffer = await file.arrayBuffer();
|
||||
await writeFile(input, Buffer.from(buffer));
|
||||
const { stdout } = await probe("ffprobe", ["-v", "error", "-protocol_whitelist", "file", "-format_whitelist", "mov,matroska,webm,mp3,wav,ogg,flac", "-show_streams", "-show_format", "-of", "json", input], { timeout: 15000, maxBuffer: 1024 * 1024 });
|
||||
let stdout: string;
|
||||
try {
|
||||
({ stdout } = await probe("ffprobe", ["-v", "error", "-protocol_whitelist", "file", "-format_whitelist", "mov,matroska,webm,mp3,wav,ogg,flac", "-show_streams", "-show_format", "-of", "json", input], { timeout: 15000, maxBuffer: 1024 * 1024 }));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return NextResponse.json({ success: false, error: "服务器缺少 ffprobe 视频解析组件，请联系管理员安装 FFmpeg" }, { status: 503 });
|
||||
}
|
||||
return NextResponse.json({ success: false, error: "视频文件无法解析，请确认素材完整且格式受支持" }, { status: 400 });
|
||||
}
|
||||
const info = JSON.parse(stdout);
|
||||
const kind = file.type.startsWith("video/") ? "video" : "audio";
|
||||
const stream = info.streams?.find((item: { codec_type: string }) => item.codec_type === kind);
|
||||
const duration = Number(info.format?.duration);
|
||||
if (!stream || !Number.isFinite(duration) || duration <= 0 || duration > 600) return NextResponse.json({ success: false, error: "请上传内容完整、10 分钟以内的音视频" }, { status: 400 });
|
||||
const uploaded = await uploadFile(`canvas-${randomUUID()}.${formats[file.type]}`, buffer);
|
||||
let uploaded;
|
||||
try {
|
||||
uploaded = await uploadFile(`canvas-${randomUUID()}.${formats[file.type]}`, buffer);
|
||||
} catch (error) {
|
||||
console.error("[Canvas media upload] Storage unavailable:", error);
|
||||
return NextResponse.json({ success: false, error: "媒体存储暂不可用，请稍后重试" }, { status: 503 });
|
||||
}
|
||||
return NextResponse.json({ success: true, data: { url: uploaded.url, duration, width: stream.width, height: stream.height } });
|
||||
} catch {
|
||||
} catch (error) {
|
||||
console.error("[Canvas media upload] Failed:", error);
|
||||
return NextResponse.json({ success: false, error: "素材上传失败，请确认文件完整或稍后重试" }, { status: 400 });
|
||||
} finally {
|
||||
running.delete(session.user.id);
|
||||
|
||||
@ -408,6 +408,15 @@
|
||||
}
|
||||
|
||||
/* Dense, shared system-settings language. Scoped so canvas tools keep their own scale. */
|
||||
.video-canvas-workspace {
|
||||
/* Match the compact typography used by the reference canvas. */
|
||||
--canvas-font-caption: 12px;
|
||||
--canvas-font-body: 13px;
|
||||
--canvas-font-heading: 14px;
|
||||
font-size: var(--canvas-font-body);
|
||||
line-height: 1.45;
|
||||
}
|
||||
|
||||
.settings-workspace {
|
||||
font-size: 13px;
|
||||
line-height: 1.45;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
253
components/canvas/VideoCanvasHome.tsx
Normal file
253
components/canvas/VideoCanvasHome.tsx
Normal file
@ -0,0 +1,253 @@
|
||||
"use client";
|
||||
|
||||
import { useRouter } from "next/navigation";
|
||||
import { useEffect, useState } from "react";
|
||||
import useSWR from "swr";
|
||||
import {
|
||||
ArrowLeft, Clapperboard, FileText, Film, FolderOpen, Image as ImageIcon,
|
||||
LayoutGrid, MoreHorizontal, Music2, Plus, Scissors, Sparkles, Video,
|
||||
WandSparkles, Zap,
|
||||
} from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { filterCanvasModels, type CanvasModelDescriptor } from "@/lib/video-canvas/model-matching";
|
||||
|
||||
export type VideoCanvasHomeProject = {
|
||||
id: string;
|
||||
title: string;
|
||||
thumbnail: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
};
|
||||
|
||||
type Section = "home" | "projects";
|
||||
type Starter = "image" | "video" | "audio" | "script" | "smart-edit";
|
||||
type BrandingSettings = {
|
||||
siteName?: string | null;
|
||||
siteLogo?: string | null;
|
||||
siteLogoDark?: string | null;
|
||||
};
|
||||
|
||||
const quickStarts: Array<{ label: string; icon: typeof Video; starter: Starter; mode?: string; model?: string }> = [
|
||||
{ label: "文生视频", icon: WandSparkles, starter: "video", mode: "text-to-video" },
|
||||
{ label: "首帧视频", icon: Video, starter: "video", mode: "first-frame" },
|
||||
{ label: "多参考视频", icon: Film, starter: "video", mode: "reference-images" },
|
||||
{ label: "视频生成", icon: Clapperboard, starter: "video" },
|
||||
{ label: "图片生成", icon: ImageIcon, starter: "image" },
|
||||
{ label: "音频生成", icon: Music2, starter: "audio" },
|
||||
{ label: "剧本生成", icon: FileText, starter: "script" },
|
||||
{ label: "智能剪辑", icon: Scissors, starter: "smart-edit" },
|
||||
];
|
||||
|
||||
function projectHref(id: string, starter?: Starter, mode?: string, model?: string) {
|
||||
const params = new URLSearchParams({ mode: "video", workspaceType: "video" });
|
||||
if (starter) params.set("starter", starter);
|
||||
if (mode) params.set("starterMode", mode);
|
||||
if (model) params.set("starterModel", model);
|
||||
return `/canvas/${encodeURIComponent(id)}?${params.toString()}`;
|
||||
}
|
||||
|
||||
function projectDate(value: string) {
|
||||
const parts = new Intl.DateTimeFormat("zh-CN", {
|
||||
timeZone: "Asia/Shanghai", year: "numeric", month: "2-digit", day: "2-digit",
|
||||
}).formatToParts(new Date(value));
|
||||
const get = (name: string) => parts.find(part => part.type === name)?.value || "";
|
||||
return `${get("year")}-${get("month")}-${get("day")}`;
|
||||
}
|
||||
|
||||
function projectTitle(value: string) {
|
||||
return !value || value === "未命名视频画布" ? "未命名" : value;
|
||||
}
|
||||
|
||||
export function VideoCanvasHome({
|
||||
initialHasMore, initialProjects, siteLogo, siteLogoDark, siteName, userName,
|
||||
}: {
|
||||
initialHasMore: boolean;
|
||||
initialProjects: VideoCanvasHomeProject[];
|
||||
siteLogo: string | null;
|
||||
siteLogoDark: string | null;
|
||||
siteName: string;
|
||||
userName: string;
|
||||
}) {
|
||||
const router = useRouter();
|
||||
const [section, setSection] = useState<Section>("home");
|
||||
const [projects, setProjects] = useState(initialProjects);
|
||||
const [hasMore, setHasMore] = useState(initialHasMore);
|
||||
const [loadingMore, setLoadingMore] = useState(false);
|
||||
const [creating, setCreating] = useState(false);
|
||||
const [menuId, setMenuId] = useState<string | null>(null);
|
||||
const [editingId, setEditingId] = useState<string | null>(null);
|
||||
const [draftTitle, setDraftTitle] = useState("");
|
||||
const { data: branding } = useSWR<BrandingSettings>(
|
||||
"/api/site-settings?scope=branding",
|
||||
async (url: string) => {
|
||||
const response = await fetch(url, { cache: "no-store" });
|
||||
if (!response.ok) throw new Error("品牌信息不可用");
|
||||
return response.json();
|
||||
},
|
||||
{ refreshInterval: 30_000, revalidateOnFocus: true },
|
||||
);
|
||||
const brandName = String(branding?.siteName ?? siteName).trim() || "AI Chatbot";
|
||||
const brandLogo = (branding ? branding.siteLogoDark : siteLogoDark)?.trim()
|
||||
|| (branding ? branding.siteLogo : siteLogo)?.trim();
|
||||
const { data: membership } = useSWR<{ summary?: { currentCredits?: number } }>(
|
||||
"/api/user/membership-summary",
|
||||
async (url: string) => {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) throw new Error("积分信息不可用");
|
||||
return response.json();
|
||||
},
|
||||
);
|
||||
const { data: modelData } = useSWR<{ models?: CanvasModelDescriptor[] }>("/api/models", async (url: string) => {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) throw new Error("模型目录不可用");
|
||||
return response.json();
|
||||
});
|
||||
const availableVideoModels = filterCanvasModels(modelData?.models || [], "video", "text-to-video").slice(0, 3);
|
||||
const shortcuts = quickStarts.map((shortcut, index) => {
|
||||
const model = index < 3 ? availableVideoModels[index] : undefined;
|
||||
return model ? {
|
||||
...shortcut,
|
||||
label: model.displayName || model.modelId || shortcut.label,
|
||||
model: model.modelId || model.id,
|
||||
mode: "text-to-video",
|
||||
} : shortcut;
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
async function refreshRecent() {
|
||||
try {
|
||||
const response = await fetch("/api/canvas?workspaceType=video&limit=24&page=1", { cache: "no-store" });
|
||||
const result = await response.json();
|
||||
if (!active || !response.ok || !result.success) return;
|
||||
setProjects(current => result.data.map((item: VideoCanvasHomeProject) => ({
|
||||
...item,
|
||||
thumbnail: item.thumbnail || current.find(existing => existing.id === item.id)?.thumbnail || null,
|
||||
})));
|
||||
setHasMore(Boolean(result.pagination?.hasMore));
|
||||
} catch {
|
||||
// The server-rendered project list remains available if refreshing fails.
|
||||
}
|
||||
}
|
||||
void refreshRecent();
|
||||
return () => { active = false; };
|
||||
}, []);
|
||||
|
||||
async function createProject(starter?: Starter, mode?: string, model?: string) {
|
||||
if (creating) return;
|
||||
setCreating(true);
|
||||
try {
|
||||
const response = await fetch("/api/canvas", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ workspaceType: "video", title: "未命名视频画布" }),
|
||||
});
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success || !result.data?.id) throw new Error(result.error || "创建项目失败");
|
||||
setProjects(current => [{
|
||||
id: result.data.id,
|
||||
title: result.data.title || "未命名视频画布",
|
||||
thumbnail: null,
|
||||
createdAt: result.data.createdAt || new Date().toISOString(),
|
||||
updatedAt: result.data.updatedAt || new Date().toISOString(),
|
||||
}, ...current]);
|
||||
router.push(projectHref(result.data.id, starter, mode, model));
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "创建项目失败，请重试");
|
||||
setCreating(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadMore() {
|
||||
if (loadingMore || !hasMore) return;
|
||||
setLoadingMore(true);
|
||||
try {
|
||||
const page = Math.floor(projects.length / 24) + 1;
|
||||
const response = await fetch(`/api/canvas?workspaceType=video&limit=24&page=${page}`, { cache: "no-store" });
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success) throw new Error(result.error || "加载项目失败");
|
||||
setProjects(current => [...current, ...result.data.filter((item: VideoCanvasHomeProject) => !current.some(existing => existing.id === item.id))]);
|
||||
setHasMore(Boolean(result.pagination?.hasMore));
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "加载项目失败");
|
||||
} finally {
|
||||
setLoadingMore(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function saveTitle(id: string) {
|
||||
const title = draftTitle.trim();
|
||||
if (!title) { toast.error("项目名称不能为空"); return; }
|
||||
try {
|
||||
const response = await fetch(`/api/canvas/${encodeURIComponent(id)}`, {
|
||||
method: "PUT", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ title, workspaceType: "video" }),
|
||||
});
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success) throw new Error(result.error || "重命名失败");
|
||||
setProjects(current => current.map(item => item.id === id ? { ...item, title, updatedAt: result.data.updatedAt || item.updatedAt } : item));
|
||||
setEditingId(null);
|
||||
toast.success("项目已重命名");
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "重命名失败");
|
||||
}
|
||||
}
|
||||
|
||||
const visibleProjects = section === "home" ? projects.slice(0, 4) : projects;
|
||||
|
||||
return (
|
||||
<div className="flex h-svh min-w-0 overflow-hidden bg-[#191919] text-[#f7f7f7]">
|
||||
<aside className="flex w-[210px] shrink-0 flex-col border-r border-white/[0.06] bg-[#1c1c1c] px-3 py-[18px] max-sm:w-[60px] max-sm:px-2">
|
||||
<a aria-label={`返回${brandName}首页`} className="mb-4 flex h-7 min-w-0 items-center gap-2 px-2 text-[18px] font-semibold" href="/">
|
||||
{brandLogo ? <img alt="" className="size-6 shrink-0 object-contain" src={brandLogo} /> : <span className="flex size-5 shrink-0 items-center justify-center rounded bg-white text-[#191919]"><Film className="size-3.5" /></span>}
|
||||
<span className="truncate max-sm:hidden">{brandName}</span>
|
||||
</a>
|
||||
<button className="flex h-[33px] items-center gap-2 rounded-[6px] bg-[#15bde4] px-2.5 text-left text-[13px] font-medium text-black hover:bg-[#3ed0ed] disabled:opacity-55" disabled={creating} onClick={() => void createProject()} type="button">
|
||||
<Plus className="size-4" /><span className="max-sm:hidden">{creating ? "正在创建…" : "新建项目"}</span>
|
||||
</button>
|
||||
<div className="mt-5 space-y-1 border-b border-white/[0.06] pb-4">
|
||||
<button className="flex h-8 w-full items-center gap-2 rounded-md px-2.5 text-left text-[12px] text-white/75 hover:bg-white/[0.06]" onClick={() => void createProject("video")} type="button"><Sparkles className="size-4" /><span className="max-sm:hidden">视频创作</span></button>
|
||||
<button className="flex h-8 w-full items-center gap-2 rounded-md px-2.5 text-left text-[12px] text-white/75 hover:bg-white/[0.06]" onClick={() => void createProject("smart-edit")} type="button"><Scissors className="size-4" /><span className="max-sm:hidden">智能剪辑</span></button>
|
||||
</div>
|
||||
<nav aria-label="视频画布导航" className="mt-3 space-y-1">
|
||||
<button aria-current={section === "home" ? "page" : undefined} className={`flex h-8 w-full items-center gap-2 rounded-md px-2.5 text-left text-[12px] ${section === "home" ? "bg-[#282828] text-white" : "text-white/75 hover:bg-white/[0.06]"}`} onClick={() => setSection("home")} type="button"><LayoutGrid className="size-4" /><span className="max-sm:hidden">首页</span></button>
|
||||
<button aria-current={section === "projects" ? "page" : undefined} className={`flex h-8 w-full items-center gap-2 rounded-md px-2.5 text-left text-[12px] ${section === "projects" ? "bg-[#282828] text-white" : "text-white/75 hover:bg-white/[0.06]"}`} onClick={() => setSection("projects")} type="button"><FolderOpen className="size-4" /><span className="max-sm:hidden">项目</span></button>
|
||||
<a className="flex h-8 items-center gap-2 rounded-md px-2.5 text-[12px] text-white/75 hover:bg-white/[0.06]" href="/gallery"><ImageIcon className="size-4" /><span className="max-sm:hidden">资产</span></a>
|
||||
</nav>
|
||||
<a aria-label="返回生图画布首页" className="mt-3 flex h-8 items-center gap-2 rounded-md border border-white/10 px-2.5 text-[12px] text-white/75 hover:bg-white/[0.06] hover:text-white" href="/" title="返回生图画布首页"><ArrowLeft className="size-4 shrink-0" /><span className="truncate max-sm:hidden">返回生图画布首页</span></a>
|
||||
<div className="mt-auto flex items-center gap-2 border-t border-white/[0.06] px-2 pt-4 text-[12px] text-white/70"><span className="flex size-7 items-center justify-center rounded-full bg-[#414a62] font-medium text-white">{(userName || "用").slice(0, 1)}</span><span className="truncate max-sm:hidden">{userName || "我的工作区"}</span></div>
|
||||
</aside>
|
||||
|
||||
<main className="min-w-0 flex-1 overflow-y-auto">
|
||||
<header className="flex h-[46px] items-center justify-end gap-2 border-b border-white/[0.04] px-5 md:px-9">
|
||||
<span className="flex h-8 items-center gap-1 rounded-lg border border-white/10 px-3 text-[12px] text-white/75"><Zap className="size-3.5" />{membership?.summary?.currentCredits ?? "—"}</span>
|
||||
<span aria-label={userName || "当前用户"} className="flex size-8 items-center justify-center rounded-full bg-[#414a62] text-[12px] font-medium">{(userName || "用").slice(0, 1)}</span>
|
||||
</header>
|
||||
<div className="mx-auto max-w-[1500px] px-5 pb-12 pt-[10px] md:px-9">
|
||||
{section === "home" ? <>
|
||||
<button aria-label="新建画布创作" className="flex h-[140px] w-full flex-col items-center justify-center gap-3 rounded-[12px] border border-white/[0.14] bg-[radial-gradient(circle_at_center,rgba(255,255,255,.045),transparent_65%),radial-gradient(#5c5c5c_0.8px,transparent_0.8px)] bg-[length:100%_100%,18px_18px] text-[14px] font-semibold hover:border-white/30" disabled={creating} onClick={() => void createProject()} type="button">
|
||||
<span className="flex h-[48px] w-[80px] items-center justify-center rounded-[15px] bg-[linear-gradient(#fff,#e9e9e9)] text-black shadow-[0_8px_25px_rgba(255,255,255,.1)]"><Plus className="size-8 stroke-[1.8]" /></span>
|
||||
新建画布创作
|
||||
</button>
|
||||
<div className="mt-7 grid grid-cols-4 gap-3 lg:grid-cols-8 lg:gap-5">
|
||||
{shortcuts.map(({ label, icon: Icon, starter, mode, model }, index) => <button className="group min-w-0 text-center disabled:opacity-50" disabled={creating} key={`${index}-${label}`} onClick={() => void createProject(starter, mode, model)} title={label} type="button"><span className="mx-auto flex h-[50px] w-[86px] max-w-full items-center justify-center rounded-[16px] bg-[#262626] transition-colors group-hover:bg-[#343434]"><Icon className="size-5 text-white/90" /></span><span className="mt-2 block truncate text-[12px] font-medium">{label}</span></button>)}
|
||||
</div>
|
||||
</> : null}
|
||||
|
||||
<div className={section === "home" ? "mt-10" : "mt-1"}>
|
||||
<div className="mb-4 flex items-center justify-between"><h1 className="text-[14px] font-semibold">{section === "home" ? "最近项目" : "全部项目"}</h1>{section === "home" && projects.length > 0 ? <button className="text-[12px] text-white/75 hover:text-white" onClick={() => setSection("projects")} type="button">查看全部 ›</button> : null}</div>
|
||||
{visibleProjects.length ? <div className={`grid grid-cols-1 gap-3 ${section === "home" ? "sm:grid-cols-2 lg:grid-cols-4" : "sm:grid-cols-2"}`}>{visibleProjects.map(project => <div className="group relative flex h-[74px] min-w-0 items-center gap-3 rounded-[24px] border border-white/[0.09] bg-[#202020] pr-11 transition-colors hover:border-white/[0.19] hover:bg-[#292929]" key={project.id}>
|
||||
<button aria-label={`打开项目 ${projectTitle(project.title)}`} className="absolute inset-0 rounded-[24px]" onClick={() => router.push(projectHref(project.id))} type="button" />
|
||||
<span className="pointer-events-none z-[1] ml-1 flex h-[66px] w-[88px] shrink-0 items-center justify-center overflow-hidden rounded-[20px] bg-[#292929] text-white/30">{project.thumbnail ? <img alt="" className="size-full object-cover" src={project.thumbnail} /> : <Sparkles className="size-5" />}</span>
|
||||
<span className="pointer-events-none z-[1] min-w-0 flex-1"><span className="block truncate text-[13px] font-semibold">{projectTitle(project.title)}</span><span className="mt-1 block text-[11px] text-white/50">{projectDate(project.updatedAt)}</span></span>
|
||||
<button aria-label={`项目 ${projectTitle(project.title)} 的更多操作`} aria-expanded={menuId === project.id} className="absolute right-4 top-4 z-[2] flex size-7 items-center justify-center rounded-lg text-white/60 opacity-0 hover:bg-white/10 hover:text-white focus:opacity-100 group-hover:opacity-100" onClick={() => setMenuId(current => current === project.id ? null : project.id)} type="button"><MoreHorizontal className="size-4" /></button>
|
||||
{menuId === project.id ? <div className="absolute right-3 top-11 z-[4] min-w-[110px] rounded-lg border border-white/10 bg-[#303030] p-1 shadow-xl"><button className="w-full rounded px-3 py-2 text-left text-[12px] hover:bg-white/10" onClick={() => { setEditingId(project.id); setDraftTitle(projectTitle(project.title)); setMenuId(null); }} type="button">重命名</button></div> : null}
|
||||
{editingId === project.id ? <form className="absolute inset-1 z-[5] flex items-center gap-2 rounded-[20px] bg-[#292929] px-3" onSubmit={event => { event.preventDefault(); void saveTitle(project.id); }}><input aria-label="项目名称" autoFocus className="min-w-0 flex-1 rounded-md border border-white/20 bg-[#202020] px-3 py-2 text-[13px] outline-none focus:border-white/60" maxLength={256} onChange={event => setDraftTitle(event.target.value)} value={draftTitle} /><button className="text-[12px] text-cyan-300" type="submit">保存</button><button className="text-[12px] text-white/50" onClick={() => setEditingId(null)} type="button">取消</button></form> : null}
|
||||
</div>)}</div> : <div className="rounded-2xl border border-dashed border-white/10 px-5 py-14 text-center text-[13px] text-white/55">还没有视频项目，点击上方“新建画布创作”开始。</div>}
|
||||
{section === "projects" && hasMore ? <button className="mx-auto mt-6 block rounded-lg border border-white/15 px-5 py-2 text-[12px] text-white/70 hover:bg-white/10 disabled:opacity-50" disabled={loadingMore} onClick={() => void loadMore()} type="button">{loadingMore ? "加载中…" : "加载更多"}</button> : null}
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -9,7 +9,7 @@ import {
|
||||
} from "@/lib/video-canvas/director-extensions";
|
||||
import { directorPoses, type DirectorPoseName } from "@/lib/video-canvas/director-poses";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Camera, Plus, Upload, X, Trash2 } from "lucide-react";
|
||||
import { ArrowUp, Camera, Plus, Upload, X, Trash2 } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { DirectorViewport, type DirectorViewportHandle } from "./DirectorViewport";
|
||||
import {
|
||||
@ -53,7 +53,7 @@ async function upload(file: File) {
|
||||
);
|
||||
const result = await response.json();
|
||||
const url = result.url || result.data?.url;
|
||||
if (!response.ok || !url) throw Error(result.error || "上传失败");
|
||||
if (!response.ok || !url) throw new Error(result.error || "上传失败");
|
||||
return String(url);
|
||||
}
|
||||
export function DirectorStudioDialog({
|
||||
@ -86,7 +86,12 @@ export function DirectorStudioDialog({
|
||||
onChange(next);
|
||||
};
|
||||
const [objectSearch, setObjectSearch] = useState("");
|
||||
const [selected, setSelected] = useState(scene.objects[0]?.id || "");
|
||||
const [scenePrompt, setScenePrompt] = useState("");
|
||||
const [sceneReferenceUrl, setSceneReferenceUrl] = useState("");
|
||||
const [scenePromptBusy, setScenePromptBusy] = useState(false);
|
||||
const sceneReferenceInput = useRef<HTMLInputElement>(null);
|
||||
const [selected, setSelected] = useState("");
|
||||
const [rightTab, setRightTab] = useState<"properties" | "pose">("properties");
|
||||
const [mode, setMode] = useState<"translate" | "rotate" | "scale">("translate");
|
||||
const [cameraView, setCameraView] = useState(false);
|
||||
const [busy, setBusy] = useState(false);
|
||||
@ -156,7 +161,7 @@ export function DirectorStudioDialog({
|
||||
})),
|
||||
};
|
||||
const pose = sampleDirectorMotion(motion, 0);
|
||||
if (!pose || motion.keyframes.length < 2) throw Error("至少需要两个关键帧");
|
||||
if (!pose || motion.keyframes.length < 2) throw new Error("至少需要两个关键帧");
|
||||
const image = await viewport.current!.capture(pose, 0);
|
||||
const url = await upload(
|
||||
new File([image.blob], `director-motion-${Date.now()}.png`, {
|
||||
@ -279,8 +284,8 @@ export function DirectorStudioDialog({
|
||||
className="fixed inset-0 z-[600] flex flex-col bg-[#141414] text-white"
|
||||
onPointerDown={(e) => e.stopPropagation()}
|
||||
>
|
||||
<header className="flex h-16 shrink-0 items-center justify-between border-b border-white/10 px-5">
|
||||
<strong>3D导演台</strong>
|
||||
<header className="flex h-11 shrink-0 items-center justify-between border-b border-white/10 px-4">
|
||||
<strong className="text-[13px]">3D导演台</strong>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
className={button}
|
||||
@ -310,9 +315,9 @@ export function DirectorStudioDialog({
|
||||
<X size={18} />
|
||||
</button>
|
||||
</header>
|
||||
<div className="grid min-h-0 flex-1 grid-cols-[210px_minmax(0,1fr)_260px]">
|
||||
<aside inert={busy} className="overflow-y-auto border-r border-white/10 p-4">
|
||||
<div className="mb-3 text-sm text-white/60">场景对象</div>
|
||||
<div className="grid min-h-0 flex-1 grid-cols-[248px_minmax(0,1fr)_248px]">
|
||||
<aside inert={busy} className="overflow-y-auto border-r border-white/10 bg-[#1c1c1c] p-3">
|
||||
<div className="mb-3 text-sm text-white/70">场景</div>
|
||||
<input
|
||||
aria-label="搜索场景对象"
|
||||
className={`${field} mb-3`}
|
||||
@ -320,7 +325,12 @@ export function DirectorStudioDialog({
|
||||
value={objectSearch}
|
||||
onChange={(e) => setObjectSearch(e.target.value)}
|
||||
/>
|
||||
<div className="space-y-2">
|
||||
<div className="space-y-1">
|
||||
<button className={`${button} w-full text-left ${!selected?"bg-white/10":""}`} onClick={()=>setSelected("")} type="button">▤ 场景</button>
|
||||
{scene.cameras.map((cameraItem) => (
|
||||
<button className={`${button} w-full text-left ${scene.activeCameraId===cameraItem.id?"bg-white/10":""}`} key={cameraItem.id}
|
||||
onClick={()=>{commit({...current.current,activeCameraId:cameraItem.id});setCameraView(true)}} type="button">◧ {cameraItem.name}</button>
|
||||
))}
|
||||
{scene.objects
|
||||
.filter((o) => o.name.toLowerCase().includes(objectSearch.trim().toLowerCase()))
|
||||
.map((o) => (
|
||||
@ -333,7 +343,9 @@ export function DirectorStudioDialog({
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="mt-5 flex flex-col gap-2">
|
||||
<details className="mt-5 border-t border-white/10 pt-2">
|
||||
<summary className="cursor-pointer text-xs text-white/55">添加与场景资源</summary>
|
||||
<div className="mt-3 flex flex-col gap-2">
|
||||
<button className={button} onClick={() => addObject("actor", "基础人偶")}>
|
||||
添加人偶
|
||||
</button>
|
||||
@ -386,6 +398,7 @@ export function DirectorStudioDialog({
|
||||
AI 识图重建布景
|
||||
</button>
|
||||
</div>
|
||||
</details>
|
||||
<p className="mt-4 text-xs leading-5 text-white/45">
|
||||
左键旋转视角，右键平移，滚轮缩放。选择对象后使用变换轴调整。
|
||||
</p>
|
||||
@ -410,7 +423,7 @@ export function DirectorStudioDialog({
|
||||
onCamera={(position, target) => patchCamera({ position, target })}
|
||||
/>
|
||||
</div>
|
||||
<div className="absolute bottom-4 left-1/2 flex -translate-x-1/2 gap-1 rounded-xl bg-black/75 p-2">
|
||||
<div className="absolute bottom-4 left-[39%] flex -translate-x-1/2 gap-1 rounded-xl border border-white/10 bg-[#242424] p-2">
|
||||
{(["translate", "rotate", "scale"] as const).map((m, i) => (
|
||||
<button
|
||||
key={m}
|
||||
@ -425,9 +438,64 @@ export function DirectorStudioDialog({
|
||||
{busy ? "处理中..." : "截图"}
|
||||
</button>
|
||||
</div>
|
||||
{onReconstruct ? (
|
||||
<div className="absolute bottom-4 left-[65%] flex h-[42px] w-[min(360px,42%)] -translate-x-1/2 items-center gap-2 rounded-full border border-white/10 bg-[#242424] px-2 text-white shadow-lg">
|
||||
<button aria-label="上传场景参考图" className="flex size-8 shrink-0 items-center justify-center rounded-full hover:bg-white/10" disabled={scenePromptBusy} onClick={()=>sceneReferenceInput.current?.click()} type="button"><Plus size={16}/></button>
|
||||
<input ref={sceneReferenceInput} hidden type="file" accept="image/png,image/jpeg,image/webp" onChange={async event=>{
|
||||
const file=event.target.files?.[0];event.target.value="";if(!file)return;
|
||||
if(file.size>20*1024*1024){toast.error("参考图需小于 20 MiB");return}
|
||||
setScenePromptBusy(true);try{setSceneReferenceUrl(await upload(file));toast.success("已添加场景参考图")}catch(cause){toast.error(cause instanceof Error?cause.message:"参考图上传失败")}finally{setScenePromptBusy(false)}
|
||||
}}/>
|
||||
<input aria-label="描述想搭建的场景" className="min-w-0 flex-1 bg-transparent text-[12px] text-white outline-none placeholder:text-white/35" placeholder={sceneReferenceUrl?"描述想搭建的场景 · 已添加参考图":"描述想搭建的场景"} value={scenePrompt} onChange={event=>setScenePrompt(event.target.value)}/>
|
||||
<button aria-label="创建场景" className="flex size-8 shrink-0 items-center justify-center rounded-full bg-white/12 disabled:opacity-35" disabled={!sceneReferenceUrl||scenePromptBusy||busy} onClick={async()=>{
|
||||
if(!sceneReferenceUrl)return;setScenePromptBusy(true);try{await onReconstruct({imageUrl:sceneReferenceUrl,prompt:scenePrompt});timelineDrawer.current!.open=true;toast.success("重建任务已提交")}catch(cause){toast.error(cause instanceof Error?cause.message:"重建提交失败")}finally{setScenePromptBusy(false)}
|
||||
}} type="button"><ArrowUp size={16}/></button>
|
||||
</div>
|
||||
) : null}
|
||||
</main>
|
||||
<aside className="space-y-4 overflow-y-auto border-l border-white/10 p-4">
|
||||
<fieldset disabled={busy || previewTime !== null} className="space-y-4">
|
||||
<aside className="space-y-4 overflow-y-auto border-l border-white/10 bg-[#222] p-3">
|
||||
{!object ? (
|
||||
<div className="space-y-4 text-[12px]">
|
||||
<div className="border-b border-white/10 pb-2 text-[14px] font-medium">3D场景</div>
|
||||
<label className="flex items-center justify-between gap-2">天空颜色
|
||||
<input aria-label="天空颜色" type="color" value={scene.skyColor || "#080808"} onChange={event=>commit({...current.current,skyColor:event.target.value})}/>
|
||||
</label>
|
||||
<label className="flex items-center justify-between gap-2">网格吸附
|
||||
<input aria-label="显示场景网格" type="checkbox" checked={scene.gridVisible !== false} onChange={event=>commit({...current.current,gridVisible:event.target.checked})}/>
|
||||
</label>
|
||||
<label className="flex items-center justify-between gap-2">地面
|
||||
<input aria-label="显示场景地面" type="checkbox" checked={scene.groundVisible !== false} onChange={event=>commit({...current.current,groundVisible:event.target.checked})}/>
|
||||
</label>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-3 text-[12px]">
|
||||
<div className="border-b border-white/10 pb-2 text-[14px] font-medium">{object.kind==="actor"?"角色":"场景对象"}</div>
|
||||
<div className="flex gap-2 border-b border-white/10 pb-2">
|
||||
<button className={rightTab==="properties"?"rounded bg-white/10 px-3 py-1":"px-3 py-1 text-white/45"} onClick={()=>setRightTab("properties")} type="button">属性</button>
|
||||
<button className={rightTab==="pose"?"rounded bg-white/10 px-3 py-1":"px-3 py-1 text-white/45"} onClick={()=>setRightTab("pose")} type="button">姿势</button>
|
||||
</div>
|
||||
{rightTab==="properties" ? <>
|
||||
<label className="block text-white/55">名称<input aria-label="对象名称" className={`${field} mt-1`} value={object.name} onChange={event=>patchObject(object.id,{name:event.target.value})}/></label>
|
||||
{(["position","rotation","scale"] as const).map(key=>(
|
||||
<div key={key} className="text-white/55">
|
||||
<span>{key==="position"?"位置":key==="rotation"?"旋转":"缩放"}</span>
|
||||
<div className="mt-1 grid grid-cols-3 gap-1">
|
||||
{object[key].map((value,index)=>(<label className="flex items-center rounded bg-[#353535] px-1" key={index}>
|
||||
<span className="text-[10px] text-white/40">{"XYZ"[index]}</span>
|
||||
<input aria-label={`${key}-${"XYZ"[index]}`} className="w-full min-w-0 bg-transparent px-1 py-1 text-[11px] text-white outline-none" type="number" step="0.1"
|
||||
value={Number((key==="rotation"?value*180/Math.PI:value).toFixed(2))}
|
||||
onChange={event=>{const next=Number(event.target.value);if(!Number.isFinite(next)||(key==="scale"&&next<=0))return;const values=[...object[key]] as DirectorVector;values[index]=key==="rotation"?next*Math.PI/180:next;patchObject(object.id,{[key]:values})}}/>
|
||||
</label>))}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</> : <select aria-label="角色姿势" className={field} value={object.pose||""} onChange={event=>patchObject(object.id,{pose:event.target.value as DirectorPoseName})}>
|
||||
<option value="">默认姿势</option>{Object.keys(directorPoses).map(pose=><option key={pose} value={pose}>{pose}</option>)}
|
||||
</select>}
|
||||
</div>
|
||||
)}
|
||||
<details className="border-t border-white/10 pt-2"><summary className="cursor-pointer text-xs text-white/55">相机与高级属性</summary>
|
||||
<fieldset disabled={busy || previewTime !== null} className="mt-3 space-y-4">
|
||||
<label className="block text-sm">
|
||||
机位
|
||||
<select
|
||||
@ -770,6 +838,7 @@ export function DirectorStudioDialog({
|
||||
</section>
|
||||
) : null}
|
||||
</fieldset>
|
||||
</details>
|
||||
{exportedVideo ? (
|
||||
<section className="space-y-2 border-t border-white/10 pt-3" aria-label="运镜导出结果">
|
||||
<video
|
||||
@ -834,7 +903,6 @@ export function DirectorStudioDialog({
|
||||
</div>
|
||||
<details
|
||||
ref={timelineDrawer}
|
||||
open
|
||||
className="max-h-[32vh] shrink-0 overflow-auto border-t border-white/15 bg-[#1c1c1c] px-4 pb-3"
|
||||
>
|
||||
<summary className="cursor-pointer py-2 text-sm">
|
||||
@ -915,7 +983,7 @@ export function DirectorStudioDialog({
|
||||
const bitmap = await createImageBitmap(file);
|
||||
const ratio = bitmap.width / bitmap.height;
|
||||
bitmap.close();
|
||||
if (Math.abs(ratio - 2) > 0.02) throw Error("请上传 2:1 比例的等距柱状全景图");
|
||||
if (Math.abs(ratio - 2) > 0.02) throw new Error("请上传 2:1 比例的等距柱状全景图");
|
||||
const url = await upload(file);
|
||||
commit({
|
||||
...current.current,
|
||||
@ -948,7 +1016,7 @@ export function DirectorStudioDialog({
|
||||
try {
|
||||
const header = new DataView(await file.slice(0, 12).arrayBuffer());
|
||||
if (header.byteLength < 12 || header.getUint32(0, true) !== 0x46546c67)
|
||||
throw Error("文件不是有效 GLB 模型");
|
||||
throw new Error("文件不是有效 GLB 模型");
|
||||
const url = await upload(file);
|
||||
addObject("model", file.name, url);
|
||||
} catch (e) {
|
||||
|
||||
@ -56,6 +56,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
orbit: OrbitControls;
|
||||
transform: TransformControls;
|
||||
helper: THREE.Object3D;
|
||||
floor: THREE.Mesh;
|
||||
grid: THREE.GridHelper;
|
||||
path: THREE.Group;
|
||||
objects: THREE.Group;
|
||||
@ -69,6 +70,12 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [backgroundPending, setBackgroundPending] = useState(false);
|
||||
const [backgroundError, setBackgroundError] = useState("");
|
||||
useEffect(() => {
|
||||
const r = runtime.current;
|
||||
if (!r) return;
|
||||
r.grid.visible = props.scene.gridVisible !== false;
|
||||
r.floor.visible = props.scene.groundVisible !== false;
|
||||
}, [props.scene.gridVisible, props.scene.groundVisible, ready]);
|
||||
const applyObjectTime = (time: number | null) => {
|
||||
const objects = sampleDirectorObjects(latest.current.scene, time),
|
||||
r = runtime.current;
|
||||
@ -99,7 +106,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
}
|
||||
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
||||
renderer.outputColorSpace = THREE.SRGBColorSpace;
|
||||
renderer.setClearColor("#c5cbd3");
|
||||
renderer.setClearColor("#080808");
|
||||
host.current.appendChild(renderer.domElement);
|
||||
renderer.domElement.setAttribute("aria-label", "3D 导演台场景");
|
||||
Object.assign(renderer.domElement.style, {
|
||||
@ -109,19 +116,19 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
transform: "translate(-50%, -50%)",
|
||||
});
|
||||
const world = new THREE.Scene();
|
||||
world.background = new THREE.Color("#c5cbd3");
|
||||
world.background = new THREE.Color(props.scene.skyColor || "#080808");
|
||||
world.add(new THREE.HemisphereLight(0xffffff, 0x647080, 2.4));
|
||||
const light = new THREE.DirectionalLight(0xffffff, 3);
|
||||
light.position.set(4, 8, 5);
|
||||
world.add(light);
|
||||
const floor = new THREE.Mesh(
|
||||
new THREE.PlaneGeometry(100, 100),
|
||||
new THREE.MeshStandardMaterial({ color: 0xb1b6be, roughness: 1 }),
|
||||
new THREE.MeshBasicMaterial({ color: 0x0b0f15 }),
|
||||
);
|
||||
floor.rotation.x = -Math.PI / 2;
|
||||
floor.position.y = -0.02;
|
||||
world.add(floor);
|
||||
const grid = new THREE.GridHelper(30, 30, 0x596573, 0x929daa);
|
||||
const grid = new THREE.GridHelper(30, 30, 0x21445b, 0x183346);
|
||||
world.add(grid);
|
||||
const path = new THREE.Group();
|
||||
world.add(path);
|
||||
@ -171,6 +178,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
orbit,
|
||||
transform,
|
||||
helper,
|
||||
floor,
|
||||
grid,
|
||||
path,
|
||||
objects,
|
||||
@ -300,7 +308,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
!size.toArray().every(Number.isFinite) ||
|
||||
size.length() < 0.000001
|
||||
) {
|
||||
throw Error("模型没有可显示的有效几何内容");
|
||||
throw new Error("模型没有可显示的有效几何内容");
|
||||
}
|
||||
const scale = 2 / Math.max(size.x, size.y, size.z, 0.001);
|
||||
g.scene.scale.setScalar(scale);
|
||||
@ -415,7 +423,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
let active = true;
|
||||
let texture: THREE.Texture | undefined;
|
||||
setBackgroundError("");
|
||||
r.world.background = new THREE.Color("#c5cbd3");
|
||||
r.world.background = new THREE.Color(props.scene.skyColor || "#080808");
|
||||
if (!props.scene.backgroundUrl) {
|
||||
setBackgroundPending(false);
|
||||
return;
|
||||
@ -454,10 +462,10 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
return () => {
|
||||
active = false;
|
||||
clearTimeout(timeout);
|
||||
if (r.world.background === texture) r.world.background = new THREE.Color("#c5cbd3");
|
||||
if (r.world.background === texture) r.world.background = new THREE.Color(latest.current.scene.skyColor || "#080808");
|
||||
texture?.dispose();
|
||||
};
|
||||
}, [props.scene.backgroundUrl, ready, resourceRevision]);
|
||||
}, [props.scene.backgroundUrl, props.scene.skyColor, ready, resourceRevision]);
|
||||
useEffect(() => {
|
||||
const r = runtime.current;
|
||||
if (!r) return;
|
||||
@ -604,16 +612,16 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
},
|
||||
record: async (signal, onProgress) => {
|
||||
if (backgroundPending || backgroundError)
|
||||
throw Error(backgroundError || "全景背景仍在加载，请稍后导出");
|
||||
throw new Error(backgroundError || "全景背景仍在加载，请稍后导出");
|
||||
const r = runtime.current;
|
||||
if (!r || !ready || r.pending || error) throw Error(error || "3D 场景尚未准备好");
|
||||
if (!r || !ready || r.pending || error) throw new Error(error || "3D 场景尚未准备好");
|
||||
if (
|
||||
latest.current.scene.motion.keyframes.length < 2 &&
|
||||
!Object.values(latest.current.scene.objectTracks || {}).some(
|
||||
(t) => !t.muted && t.keyframes.length >= 2,
|
||||
)
|
||||
)
|
||||
throw Error("相机或对象轨道至少需要两个关键帧");
|
||||
throw new Error("相机或对象轨道至少需要两个关键帧");
|
||||
const motion = latest.current.scene.motion;
|
||||
const size = r.renderer.getSize(new THREE.Vector2()),
|
||||
pixelRatio = r.renderer.getPixelRatio();
|
||||
@ -639,7 +647,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
onProgress,
|
||||
render: (time) => {
|
||||
if (r.renderer.getContext().isContextLost())
|
||||
throw Error("3D 显示连接中断，已取消视频导出");
|
||||
throw new Error("3D 显示连接中断，已取消视频导出");
|
||||
const pose = resolveDirectorCamera(
|
||||
applyObjectTime(time),
|
||||
sampleDirectorMotion(motion, time) || undefined,
|
||||
@ -670,9 +678,9 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
},
|
||||
capture: async (pose, time) => {
|
||||
if (backgroundPending || backgroundError)
|
||||
throw Error(backgroundError || "全景背景仍在加载，请稍后截图");
|
||||
throw new Error(backgroundError || "全景背景仍在加载，请稍后截图");
|
||||
const r = runtime.current;
|
||||
if (!r || !ready || r.pending || error) throw Error(error || "3D 场景尚未准备好");
|
||||
if (!r || !ready || r.pending || error) throw new Error(error || "3D 场景尚未准备好");
|
||||
const { width, height } = directorImageSize(latest.current.scene.ratio);
|
||||
const previousSize = r.renderer.getSize(new THREE.Vector2());
|
||||
const pixelRatio = r.renderer.getPixelRatio();
|
||||
@ -685,10 +693,10 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
r.orbit.enabled = false;
|
||||
const capturedScene = applyObjectTime(time ?? latest.current.previewTime);
|
||||
if (pose) {
|
||||
pose = resolveDirectorCamera(capturedScene, pose);
|
||||
r.camera.position.fromArray(pose.position);
|
||||
r.orbit.target.fromArray(pose.target);
|
||||
r.camera.fov = pose.fov;
|
||||
const resolvedPose = resolveDirectorCamera(capturedScene, pose);
|
||||
r.camera.position.fromArray(resolvedPose.position);
|
||||
r.orbit.target.fromArray(resolvedPose.target);
|
||||
r.camera.fov = resolvedPose.fov;
|
||||
r.camera.lookAt(r.orbit.target);
|
||||
}
|
||||
r.path.visible = false;
|
||||
@ -702,7 +710,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
try {
|
||||
const blob = await new Promise<Blob>((resolve, reject) =>
|
||||
r.renderer.domElement.toBlob(
|
||||
(b) => (b ? resolve(b) : reject(Error("截图失败"))),
|
||||
(b) => (b ? resolve(b) : reject(new Error("截图失败"))),
|
||||
"image/png",
|
||||
),
|
||||
);
|
||||
|
||||
@ -50,9 +50,9 @@ export function SmartEditDialog({
|
||||
assets: [],
|
||||
mode: "auto",
|
||||
prompt: "",
|
||||
targetDuration: 20,
|
||||
targetDuration: 30,
|
||||
ratio: "16:9",
|
||||
resolution: "1080p",
|
||||
resolution: "720p",
|
||||
semantic: true,
|
||||
removeSilence: false,
|
||||
transcribe: false,
|
||||
|
||||
@ -27,6 +27,33 @@
|
||||
border-radius: 10px 10px 0 0;
|
||||
background: #202020;
|
||||
}
|
||||
.emptyCard .preview {
|
||||
height: 100%;
|
||||
border-radius: 10px;
|
||||
}
|
||||
.emptyActions {
|
||||
position: absolute;
|
||||
bottom: 24px;
|
||||
left: 28px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
color: #a5a5a5;
|
||||
font-size: 13px;
|
||||
}
|
||||
.emptyActions button {
|
||||
padding: 0;
|
||||
border: 0;
|
||||
background: transparent;
|
||||
color: #eee;
|
||||
font-size: 14px;
|
||||
line-height: 20px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.emptyActions button:hover {
|
||||
color: white;
|
||||
}
|
||||
.preview > video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
@ -247,6 +274,15 @@
|
||||
left: 112px;
|
||||
width: 340px;
|
||||
}
|
||||
.musicDetails {
|
||||
margin-top: 10px;
|
||||
border-top: 1px solid #393939;
|
||||
padding-top: 7px;
|
||||
}
|
||||
.musicDetails summary {
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
}
|
||||
.settings label {
|
||||
display: block;
|
||||
font-size: 12px;
|
||||
@ -384,3 +420,14 @@
|
||||
.settings select { width: 100%; height: 30px; padding: 0 8px; border-radius: 6px; background: #333; color: #eee; border: 0; }
|
||||
.musicUpload { display: flex; align-items: center; gap: 4px; margin-top: 8px; font-size: 12px; }
|
||||
.settings label.beatToggle { display: flex; gap: 8px; align-items: center; }
|
||||
|
||||
.emptyCard .open {
|
||||
justify-content: flex-start;
|
||||
padding-top: 94px;
|
||||
gap: 16px;
|
||||
color: #999;
|
||||
font-size: 12px;
|
||||
}
|
||||
.emptyCard .open svg {
|
||||
color: #aaa;
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ import {
|
||||
import { resolveSmartEditIntent, assertBeatMusic } from "@/lib/video-canvas/smart-edit/intent";
|
||||
import styles from "./SmartEditNode.module.css";
|
||||
export type SmartEditSource = { id: string; name: string; url: string; duration?: number };
|
||||
export type SmartEditMode = "explain" | "ads" | "talk" | "montage";
|
||||
const modes = [
|
||||
{ id: "auto", name: "默认模式" },
|
||||
{ id: "explain", name: "讲解视频" },
|
||||
@ -34,6 +35,7 @@ export function SmartEditNodeCard({
|
||||
previewUrl,
|
||||
plan,
|
||||
onOpen,
|
||||
onMode,
|
||||
selected = false,
|
||||
}: {
|
||||
title: string;
|
||||
@ -41,6 +43,7 @@ export function SmartEditNodeCard({
|
||||
previewUrl?: string;
|
||||
plan?: EditPlan;
|
||||
onOpen: () => void;
|
||||
onMode?: (mode: SmartEditMode) => void;
|
||||
}) {
|
||||
let timeline: ReturnType<typeof planTimeline> | undefined;
|
||||
try {
|
||||
@ -50,8 +53,9 @@ export function SmartEditNodeCard({
|
||||
}
|
||||
const length = timeline?.duration || 0,
|
||||
tick = Math.max(1, Math.ceil(length / SECOND / 5));
|
||||
const empty = !timeline && !previewUrl;
|
||||
return (
|
||||
<div className={styles.card} style={{ borderColor: selected ? "#888" : "#454545" }}>
|
||||
<div className={`${styles.card} ${empty ? styles.emptyCard : ""}`} style={{ borderColor: selected ? "#888" : "#454545" }}>
|
||||
<div className={styles.title}>
|
||||
<Play size={12} fill="currentColor" />
|
||||
{title}
|
||||
@ -60,11 +64,29 @@ export function SmartEditNodeCard({
|
||||
{previewUrl && <video src={`${previewUrl}#t=0.1`} muted preload="metadata" />}
|
||||
<div className={styles.open}>
|
||||
<Scissors size={36} />
|
||||
<button onPointerDown={(e) => e.stopPropagation()} onClick={onOpen}>
|
||||
打开智能剪辑
|
||||
</button>
|
||||
{empty ? (
|
||||
<span>空空如也，请连接视频节点后操作</span>
|
||||
) : (
|
||||
<button onPointerDown={(e) => e.stopPropagation()} onClick={onOpen}>
|
||||
打开智能剪辑
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{empty ? (
|
||||
<div className={styles.emptyActions}>
|
||||
<span>尝试：</span>
|
||||
{([
|
||||
["explain", "讲解视频"],
|
||||
["ads", "批量广告"],
|
||||
["talk", "口播视频"],
|
||||
["montage", "素材混剪"],
|
||||
] as const).map(([mode, label]) => (
|
||||
<button key={mode} onPointerDown={(e) => e.stopPropagation()} onClick={() => onMode?.(mode)} type="button">{label}</button>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
{!empty ? <>
|
||||
<div className={styles.ruler} aria-label="剪辑时间刻度">
|
||||
{timeline ? (
|
||||
Array.from({ length: Math.floor(length / SECOND / tick) + 1 }, (_, i) => {
|
||||
@ -126,6 +148,7 @@ export function SmartEditNodeCard({
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</> : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -172,8 +195,8 @@ export function SmartEditNodeComposer({
|
||||
return () => document.removeEventListener("pointerdown", close);
|
||||
}, []);
|
||||
const ratio = draft.ratio || "16:9",
|
||||
resolution = draft.resolution || "1080p",
|
||||
duration = draft.targetDuration ?? 20,
|
||||
resolution = draft.resolution || "720p",
|
||||
duration = draft.targetDuration ?? 30,
|
||||
mode = modes.find((m) => m.id === draft.mode) || modes[0];
|
||||
const toggle = (value: typeof menu) => setMenu(menu === value ? null : value);
|
||||
return (
|
||||
@ -380,6 +403,8 @@ export function SmartEditNodeComposer({
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<details className={styles.musicDetails}>
|
||||
<summary>背景音乐与卡点</summary>
|
||||
<label htmlFor="smart-edit-music">背景音乐</label>
|
||||
<select
|
||||
id="smart-edit-music"
|
||||
@ -431,6 +456,7 @@ export function SmartEditNodeComposer({
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</details>
|
||||
<label>清晰度</label>
|
||||
<div className={styles.resolutions}>
|
||||
{editResolutions.map((r) => (
|
||||
|
||||
@ -80,12 +80,13 @@ export function ChatShell({
|
||||
const isAiCadWorkspace = pathname === "/ai-cad";
|
||||
const isFlowBoardWorkspace = pathname === "/flow-board";
|
||||
const isPascalEditorWorkspace = pathname === "/pascal-editor";
|
||||
const isVideoCanvasHome = pathname === "/video-canvas";
|
||||
|
||||
if (isLovartHome) {
|
||||
return <div className="h-svh w-full overflow-y-auto">{children}</div>;
|
||||
}
|
||||
|
||||
if (isAiCadWorkspace || isFlowBoardWorkspace || isPascalEditorWorkspace) {
|
||||
if (isAiCadWorkspace || isFlowBoardWorkspace || isPascalEditorWorkspace || isVideoCanvasHome) {
|
||||
return <div className="h-svh w-full overflow-hidden">{children}</div>;
|
||||
}
|
||||
|
||||
|
||||
21
docs/video-canvas-aliyun-analysis-2026-09-24.md
Normal file
21
docs/video-canvas-aliyun-analysis-2026-09-24.md
Normal file
@ -0,0 +1,21 @@
|
||||
# 阿里云视频画布模型核对与本地验收（2026-09-24）
|
||||
|
||||
## 官方能力与对应节点
|
||||
|
||||
| 节点/需求 | 官方模型与实际边界 | 本站处理 |
|
||||
| --- | --- | --- |
|
||||
| 逐帧拉片的画面、分镜和动作 | [Qwen3.8 Max](https://help.aliyun.com/zh/model-studio/vision-model) 支持直接输入最长 2 小时、2GB 的视频；[视频输入示例](https://help.aliyun.com/zh/model-studio/vision) 使用 `video_url`。该视觉模型不理解原声。 | 官方阿里云线路改为把**完整视频**送入模型；本机抽取的 4 帧仅供界面预览。默认优先选已有 `qwen3.8-max`。非官方适配模型仍走关键帧回退，抽帧失败会显式报错。 |
|
||||
| 拉片音乐、人声、音效 | [Qwen3.8 Omni Flash](https://help.aliyun.com/zh/model-studio/qwen-omni) 可同时理解视频画面与声音。 | 选择“音乐”维度时只使用已启用的 Omni；未配置则明确禁用提交。仅用视觉模型时，结果音频字段强制清空，防止虚构听觉内容。 |
|
||||
| 音频生成 | [Qwen3 TTS](https://help.aliyun.com/zh/model-studio/non-realtime-tts-user-guide) 已有两条生产模型配置；[Fun-Music](https://help.aliyun.com/zh/model-studio/fun-music) 仍处于邀测。 | TTS 继续沿用本站阿里云音频链路；音乐生成不能仅凭现有 Key 认定已开通。音频面板布局与参考站的差异不由换模型解决。 |
|
||||
| 五分钟成片 | [万相 3.0](https://help.aliyun.com/zh/model-studio/video-generate-edit-model) 单次最多 30 秒，官方建议用首尾帧串联；Wan 2.7 单次最多 15 秒。 | 无官方一次输出 300 秒的模型契约。需至少 10 段 Wan 3.0 加镜头连续性、失败续跑、成本展示和现有合成服务拼接；当前**不开放伪 5 分钟直生入口**。 |
|
||||
| 智能剪辑、3D 导演台 | 视频理解模型可给出语义和镜头建议，Wan 可生成短片；两者均不提供本站时间轴编辑、场景对象变换、镜头轨迹和 UI。 | 继续按功能与参考站单独验收；配置阿里云 Key 不能自动补齐交互或像素差异。 |
|
||||
|
||||
## 实际验证
|
||||
|
||||
- 48 主服务器的官方 `alibaba` 提供商已启用且有 Key，模型表已有 `qwen3.8-max`、`qwen3-vl-plus`。服务器只读检查未输出 Key；为本地端到端验收，Key 曾短时写入隔离测试库的临时提供商行，测试结束后该行已删除。数据库备份或 WAL 是否保留旧页需按测试库策略核对。`qwen3.8-omni-flash` 生产模型行尚不存在。
|
||||
- 使用服务器 Key 向阿里云官方示例视频分别调用 `qwen3.8-max`、`qwen3.8-omni-flash`，均返回 200。前者识别画面中人物、服装、表情及单镜头；后者识别画面与背景音乐。Omni 模型可调用不等于生产画布已配置。
|
||||
- 本地临时测试库插入两条模型及临时提供商后，通过登录态 `/api/canvas/workflow/analyze` 真实调用：Max 返回 1 行结构化镜头；Omni 返回 1 行并含真实音乐、人声信息。随后通过画布生成任务 API 提交 Omni 拉片，任务为 `succeeded`，结果含 1 行镜头及音乐、人声字段，画布打开时能看到结果节点。临时项目、提供商和模型已删除。
|
||||
- 最终复验发现并修复预览帧问题：四个并发 HTTP seek 曾使同一视频返回 0 张图；120 秒以内的视频现单路解码抽帧，最新登录态接口返回 1 行镜头、4 张预览帧，视觉模型的音乐/人声字段均为空。该次端到端请求约 79 秒，仍需对不同素材和长度做性能验收。
|
||||
- 新增 4 项阿里云适配器契约测试；与画布任务回归合计 39 项通过。TypeScript 与相关文件静态检查通过。生产模型配置脚本默认仅 dry-run，尚未执行 `--apply`。
|
||||
|
||||
这只解除“找不到适用视频理解模型”和“4 张静帧冒充完整视频拉片”的技术问题。上传素材经本站对象存储到阿里云模型的真实路径、不同长度和格式的视频、五分钟生成/合成、音频面板、智能剪辑模板、导演台及像素级对照仍需继续验收，不能据此判定可以发布。
|
||||
77
docs/video-canvas-liblib-parity-audit-2026-09-24.md
Normal file
77
docs/video-canvas-liblib-parity-audit-2026-09-24.md
Normal file
@ -0,0 +1,77 @@
|
||||
# 视频画布对照记录（2026-09-24）
|
||||
|
||||
参考：用户已登录的 [LibLib 画布](https://www.liblib.tv/canvas?guideSource=home-feature-grid&spaceId=9684219&projectId=289bb2626edd465baafc57f4e29d189f) 与本次提供的七张对照截图。通过页面实际点击确认了空文本节点、文本转视频、空智能剪辑节点、素材混剪预设、空导演台、空视频节点和五分钟视频入口。测试创建的参考站节点已用撤销清理，刷新后确认画布恢复为空。
|
||||
|
||||
| 功能 | 参考站实际行为 | 本轮本地修改 | 尚未通过的验收 |
|
||||
| --- | --- | --- | --- |
|
||||
| 新建画布 | 左上“画布 1”下拉显示画布列表和加号，新画布与原画布独立 | 新增真实创建、切换与重命名入口，离开前同步保存当前画布；本机实际创建“画布 2”，添加文本节点后切回原画布、再切回新画布，节点各自保留；重命名后名称保存且立即更新 | 现有数据模型以独立视频画布项目承载每张画布；参考站“同一工作区下多画布”的更细数据结构仍需后端重构 |
|
||||
| 字号 | 节点标题与快捷操作采用紧凑层级 | 统一画布正文 13px、标题 14px、说明 12px；缩小空节点、媒体节点、文本资源、输入区和顶部导航；编组工具箱不再常驻右上方 | 其他业务页面未逐页视觉验收，不宣称全站所有页面已统一 |
|
||||
| 文本节点 | 空节点四个快捷操作；输入区约 650px 宽 | 修正第 4 项被裁剪；扩大输入区比例 | 本地页面截图和四个入口的端到端提交 |
|
||||
| 图片节点 | 未选中约 307×307px，选中后从原位置向右展开约 548×309px，出现输入区；空卡只有“图生图、图片高清”；输入区有“参考、标记、风格” | 卡片改为随选择收起/展开，实机验证两态和左边缘锚点；移除卡片内多余的说明和参数；把三个入口放进输入区；补齐此前缺失的空图片节点输入区；无参考图走文生图配置，有参考图走重绘配置；空提示词拦截；无图片时拦截不可执行的局部标记模式 | 文生图、重绘、标记和风格的真实模型任务；不同缩放与视窗尺寸下仍需视觉回归 |
|
||||
| 视频节点 | 三个快捷操作；“5分钟超长视频”切换为 300 秒/720P/Auto 的专用模式 | 暂无 300 秒模型契约，未伪装此入口 | 确定现有服务商能否原生生成 300 秒；若不能，需要设计分段生成与合成后再开放入口 |
|
||||
| 智能剪辑 | 空节点约 350px 宽，四种模式；默认 720P/30s；素材混剪预设为 1080P/20s；参数弹层首屏为比例、时长、清晰度 | 缩小节点；补齐四种模式并接入真实草稿；对齐空节点与素材混剪默认参数；背景音乐和卡点收进可展开区 | 素材上传、规划、渲染和输出浏览器全链路；参考站预设示例的表现仍不同 |
|
||||
| 3D 导演台 | 暗色场景/网格；左场景树、中央视图、右属性面板；画布内和工作区底部都有参考图输入 | 改暗场景/网格；收起默认时间轴；新增画布内和工作区内的真实参考图上传、重建任务提交；调整栏宽、场景树与属性首屏；右侧初始显示场景属性，天空颜色、网格和地面控件接入渲染并持久化 | WebGL 视觉、上传/重建/截图/轨迹/回画布全链路；参考站的场景变换与全景球参数仍未对齐，左右工具栏仍不能宣称像素级一致 |
|
||||
| 视频模型菜单 | Seedance/Wan/Minimax 等参考站专有模型 | 保留本项目现有模型，按能力过滤与提交 | 无法按参考站模型名称复刻；必须按本服务商实际能力核对支持的尺寸、时长和计费 |
|
||||
| 其他生成输入区 | 右上角展开输入框；操作项随当前模式变化 | 接通预设输入框和音频生视频输入框的展开/收起；去掉音频生视频框内不能切换的五个假模式标签与预设输入框内无效的摄像机、数量按钮 | 仍需浏览器逐项核查节点内其他辅助操作 |
|
||||
|
||||
本机已恢复独立 PostgreSQL 测试库并完成迁移，使用本地测试账号实际打开改后的画布。与同一 Chromium 窗口中的参考站逐项点击对照：图片节点选中卡片约 548×309px、输入区约 575px 宽，几何尺寸基本一致；取消选中后恢复约 307×307px，重新选中从原位置向右展开；图生图确实创建并连上图片输入节点；无图片时“标记”给出明确提示；文本节点四个入口均显示；智能剪辑四个模式可切换，素材混剪切到 1080P/20s；3D 导演台完成 WebGL 渲染，网格开关实际改变画面。生产构建、`tsc --noEmit`、`git diff --check` 和所改七个源码/测试文件的 Ultracite 检查通过；剪辑、模型匹配、导演台相关逻辑测试 60 项通过。开发热更新曾留下依赖数组变动提示，完整刷新后消失，画布保存状态正常。
|
||||
|
||||
新建画布与字号调整后又完成一次生产构建、类型/静态检查和 60 项相关测试。浏览器中实际创建并重命名本地测试画布，添加节点、来回切换、刷新后名称和节点都从数据库恢复；右上常驻编组入口迁入底部工具箱并验证可打开。
|
||||
|
||||
仍未满足发布门槛：本地独立库的 `AiModel` 为 0，无法实际生成图片/视频、规划剪辑或验证计费；参考站的 5 分钟直生视频缺少可验证的本地模型契约；模型菜单、智能剪辑示例工作流、导演台场景变换/全景球等还存在视觉与交互差异。当前只是部分入口和选中态接近，不能称为全站像素级一致或全链路通过，因此不应推送或部署到服务器。
|
||||
|
||||
## 再次逐层核对（同日）
|
||||
|
||||
本次在同一 Chromium 窗口、同一视窗下重新打开参考站和本地站的画布；实际点击了两边的添加节点、音频、脚本快捷操作，并在参考站查看故事板和逐帧拉片。参考站与本地站新建的核对节点均已撤销，本地页面显示“已保存”。下表将“亲自看见的差异”和“尚缺真实模型/素材验收”分开，不能把入口可见等同于功能通过。
|
||||
|
||||
| 优先级 | 范围 | 参考站实测 | 本地实测或代码核对 | 结论与下一项验收 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| P0 | 工作流/故事板双视图 | 顶部有两个独立切换按钮；故事板按文本、图片、视频分栏，已有文本节点可进入列表 | 顶部无对应切换；现有“脚本生成分镜”只是节点流程 | 缺少故事板视图及与画布节点同步、排序的完整交互 |
|
||||
| P0 | 逐帧拉片 | 添加菜单有“逐帧拉片 SD 2.5”；节点提供上传视频、分镜/动态/音乐拆解维度和开始拉片 | 添加菜单对应位置为“视频合成 Beta”，是合并视频片段的另一项功能 | 整项能力缺失，不能以视频合成替代；需明确视频拆解服务、输出节点和任务状态 |
|
||||
| P0 | 生成链路与模型 | 音频节点选中后是 Seed Audio 1.0 输入区，提供参考、语言/采样率/格式、语速/声调/音量；视频节点有五分钟模式 | 本地音频打开独立居中阿里云面板，当前显示“暂无已开通模型”，生成按钮禁用；本地视频缺五分钟模式；独立测试库 `AiModel=0` | 这些实际生成与计费仍不可验收。对音频用阿里云是既定产品选择，但布局/参数与参考站明显不同 |
|
||||
| P1 | 添加节点菜单与搜索 | 菜单首项可搜索画布节点，节点名称/徽标为“智能剪辑 Beta”“逐帧拉片 SD 2.5” | 菜单无搜索按钮，智能剪辑显示 NEW；全局已有 ⌘F 节点搜索，但入口位置和搜索体验不同 | 菜单结构、文案、徽标和搜索入口未对齐 |
|
||||
| P1 | 工作区顶部与空画布 | 左上工作区名可直接编辑，画布下拉和双视图按钮并列；空画布展示“图片/视频/音频/剧本生成、智能剪辑”五个快捷创建入口；右上有 TV Director | 左上是 AI Chatbot 品牌、静态“未命名工作区”和画布下拉；没有双视图或 TV Director 的对应入口；空画布代码展示“故事脚本生成、角色三视图、全能参考生视频、音频生视频”四个预设工作流 | 外观和行为都未像素级对齐；画布创建/切换/重命名本地已验证，但“同一工作区多画布”的数据结构不同 |
|
||||
| P1 | 脚本节点 | “脚本 NEW/脚本（旧版）Beta”二级菜单；新版节点三项快捷操作；点击“剧本生成分镜脚本”会插入《我在盛唐写天下》示例文本并连线 | 二级菜单、三项快捷操作和示例插入均存在；本地示例的标点和排版不同 | **示例文本是参考站也有的模板行为，不能误报为本地伪生成。** 仍需对齐结果节点排版，并用真实模型验收提交/分镜拆分 |
|
||||
| P1 | 图片与视频节点 | 图片两态、参考/标记/风格、视频三项入口与动态参数 | 图片几何尺寸在本机单视窗已基本接近；视频仅两项快捷操作；图片生成、高清和五分钟视频未完成真实任务 | 继续做不同缩放/窗口宽度截图和素材上传、输出回显、失败重试的全链路验收 |
|
||||
| P1 | 智能剪辑与特效 | 参考站四模式、模板示例、多轨编辑、转场/特效/字体/字幕等完整操作 | 四模式和参数入口在本机打开过；本地另有剪辑编辑器、转写/人声分离及远端合成代码 | 缺对同一素材从创建、选片、编辑、渲染到回画布的浏览器实测；模板自动铺排和编辑器细部仍不能宣称一致 |
|
||||
| P1 | 3D 导演台 | 左场景树、中央暗色视口、右场景属性，含场景变换和全景设置 | 暗色网格、基础人偶、参考图/重建入口已在本机实测 | 场景变换/全景球、对象操作、镜头轨迹、导出视频需同场景逐项对照及端到端验收 |
|
||||
| P1 | 编组、连线、九宫格、素材拖入 | 参考站有整组执行、分镜组和各节点间的可消费关系 | 本地有编组及拓扑校验、依赖执行、九宫格切分和拖拽上传实现 | 单元逻辑存在但本轮未用真实多媒体和供应商任务验收跨节点数据流；不能据此认定与参考站同效 |
|
||||
| P2 | 排版与全站字体 | 节点、输入区、工具栏采用更紧凑的排版；音频用节点下方浮动输入区 | 画布内部分字体已收小，但音频仍是覆盖画布的 544px 居中面板，顶部及底部控件结构也不同 | 画布仍非像素级；“全站统一字体”尚未逐页对照，不应宣称完成 |
|
||||
|
||||
核对范围说明：本轮没有提交任何收费生成请求，也没有上传用户素材到参考站；未看到真实生成输出的项目标为未验收。参考站的示例节点已撤销，未改变参考项目最终画布。本地独立测试库无模型配置，因而这轮无法完成图片、视频、音频、智能剪辑和导演台的真实服务商输出验证。发布前应优先解决 P0，再以同一套测试素材逐项录制参考站与本地站的输入、任务状态、输出、回显和刷新恢复。
|
||||
|
||||
## 首批修复与本地复核（同日）
|
||||
|
||||
已补上工作流/故事板切换：故事板按文本、图片、视频列出同一画布节点，支持拖拽调整各类节点的展示顺序；点击卡片回到工作流并定位节点。空画布改为参考站的五个基础创建入口，点击后直接选中新节点。添加菜单增加搜索入口；智能剪辑标为 Beta。
|
||||
|
||||
新增独立“逐帧拉片”节点，支持上传或连接视频、选择分镜/动态/音乐维度，并走现有视频解析任务输出结构化故事表。连线只接受视频；任务提交前检查上游上传和生成状态，并要求真实视频地址，不能把缩略图当成视频。取消文件选择不会留下空白视频节点。此功能复用当前配置的视频理解模型，**并未接入参考站的 SD 2.5 模型**，因此本地徽标显示“视频解析”，避免虚标。独立测试库无模型时开始按钮明确显示“解析模型未配置”。
|
||||
|
||||
本地 Chromium 实际查看了空画布五入口、添加菜单、逐帧拉片的维度及禁用状态；还查看了故事板中节点卡片的定位和返回工作流。相关连接测试 28 项、TypeScript、静态检查及生产构建通过。首次复核时尚未完成真实视频上传和解析结果的浏览器全链路，后续素材链路结果见下节；参考站故事板的布局细节、顶部工作区、音频面板、5 分钟视频、剪辑示例和导演台等仍有明显差异。本记录不应作为上线验收通过证明。
|
||||
|
||||
## 本地视频素材链路复验（同日）
|
||||
|
||||
使用公开测试视频在隔离的本机 Chromium、测试库和本机对象存储中复验。文件选择取消时没有遗留节点；选择约 1.1 MB 的 MP4 后，上传成功才创建视频素材节点与指向“逐帧拉片”的连线。随后用 VP9 WebM 复验浏览器解码：画布实际显示视频帧，可播放和暂停；刷新后素材、连线与画面仍在，故事板视频卡片也显示真实画面。测试节点最后已从测试画布删除。这里验证的是**上传、播放、保存、恢复和故事板预览**，没有验证模型解析结果。
|
||||
|
||||
本机首次遇到对象存储不可用时，上传接口原本只返回笼统 400；现改为明确 503，不再预先生成空素材节点。视频文件无法由 `ffprobe` 解析时返回 400，服务器缺少 `ffprobe` 时返回 503。视频对象可直接访问但画布内无法播放时，通过 Chromium 开发者工具确认是本地 HTTP 对象地址被 `media-src` 内容安全策略拦截；开发环境现仅放行配置中的本机存储源，生产策略不变。Chromium 对该样本 MP4 的播放失败与上传链路无关，改用其可解码的 VP9 WebM 完成播放验收。代码复核还发现“逐帧拉片”有缩略图时可能把缩略图地址传给解析任务，现改为只取可播放视频地址。
|
||||
|
||||
未完成的发布门槛仍然是：隔离测试库没有视频理解模型（`AiModel=0`），因此逐帧拉片的真正解析和结果回显没有完成；参考站 SD 2.5、五分钟视频、音频节点布局、智能剪辑示例工作流、导演台与整体像素级对照也未达成。不可把本机播放通过当成整套画布功能通过。
|
||||
|
||||
收尾验证：本轮涉及的视频画布路由与逻辑测试共 148 项通过；TypeScript 检查、所改核心文件静态检查、生产构建和 `git diff --check` 通过。独立本机测试服务与对象存储已停止，测试画布显示“已保存”。尚未提交 Git 或更新服务器，因为上述发布门槛仍未满足。
|
||||
|
||||
## 项目首页与视频参数复核（同日）
|
||||
|
||||
参考站首页实测有左侧导航、新建画布、八个快捷入口、最近项目卡片，以及下方“最近上新”和 TV Show 内容区。本地 `/video-canvas` 现作为项目入口，首屏实现紧凑侧栏、新建画布、快捷入口和最近项目；新建会先持久化项目再进入画布，返回首页、最近项目重开、查看全部、加载更多与重命名均接入现有项目接口。前三个快捷位优先显示本站目录中真正可执行的文生视频模型，选择后预置到新视频节点；无模型时显示通用视频模式。参考站专属 Agent、3D-BOX、社区、最近上新和 TV Show 在本站没有同等产品或数据源，目前没有放置无效入口，因此**整页尚非像素级完整复刻**。
|
||||
|
||||
本机 Chromium 验证新建项目 `67ea8543-9206-45ce-bbb5-4ce8df204c73` 后进入画布，首页刷新可见；快捷创建项目 `730aeedf-112a-4475-ba9f-143dadae7d5a` 后自动添加文生视频节点，刷新仍保留。项目更名为“本地快捷创建验收”后可从首页重新打开；左上品牌按钮保存并返回首页，最近项目展示新名称。上述 ID 仅在隔离本地测试库。画布相关逻辑测试现为 149 项通过，新增测试会拒绝仅有 4K 规格的模型进入 1080P 视频生成链路。
|
||||
|
||||
参考站视频参数弹层有 Auto/六种具体比例、480P/720P/1080P/4K、时长、生成音频和 1/2/4 个。本站弹层现逐项呈现，并按所选模型的已确认能力启用，不支持项置灰；生成数量在提交时会创建对应的独立任务。由于本站视频生成契约只支持到 1080P，4K 虽展示但不可选，不能把 4K 请求暗降为 1080P；可用的视频高清属于生成后的独立任务。隔离库 `AiModel=0`，本机浏览器只能验证弹层完整显示和缺模型时的禁用行为，不能宣称已经验证生产模型的比例、时长或带音频真实输出。五分钟超长视频和参考站其他专属参数仍待服务商能力核对。
|
||||
|
||||
## 拉片素材卡复核（同日）
|
||||
|
||||
用户截图中左侧白色“拉片视频素材”卡与参考站不一致。参考站空拉片节点只有内嵌的深色“视频素材”选择区；点击后给出“上传视频”“从画布选择”，不会预先建立白色上传卡。本站此前的通用输入占位节点采用纯白背景、居中三行说明与白色按钮，视觉层级和暗色画布冲突。本轮将通用占位卡改为深色边框、紧凑标题/两行说明/小按钮；拉片节点内也增加同样的两项选择入口，“从画布选择”仅接受已有可播放视频并建立真实连线，退出选择模式可恢复画布。上传成功才建立视频素材节点；取消文件选择不生成空素材。浏览器已核对本地节点、两项菜单和选择模式，白色卡不再出现。
|
||||
|
||||
此处仍不能称为像素级一致：参考站为专属 SD 2.5 拉片模型和更窄的节点布局，本站显示真实的“视频解析”能力；隔离测试库没有视频理解模型，无法验证解析结果。视觉对照只覆盖当前空态与菜单，真实视频上传和任务输出仍受前述发布门槛约束。
|
||||
|
||||
## 阿里云官方模型本地补充验收
|
||||
|
||||
详见 [阿里云视频画布模型核对与本地验收](video-canvas-aliyun-analysis-2026-09-24.md)。本轮用生产服务器已有的官方 Key 在隔离测试库临时配置 Qwen3.8 Max 与 Qwen3.8 Omni，真实 `/api/canvas/workflow/analyze` 和画布任务队列均返回有效镜头表；Omni 返回视频中实际音乐、人声。最终短视频复验含 4 张预览帧，视觉模型的音频字段为空。临时项目与模型已清理。此前“逐帧拉片真实结果未验收”这一项有了本地短视频证据，但本地上传素材经存储的完整路径、长视频及多格式仍需测试。五分钟生成与视觉/交互差异仍未过发布门槛。生产库的 Omni 行尚未写入，应用源码也未部署。
|
||||
@ -51,7 +51,7 @@ export function useVideoCanvasPersistence(
|
||||
});
|
||||
const remote = await latest.json();
|
||||
if (!latest.ok || !remote.success || !remote.data?.updatedAt)
|
||||
throw Error("读取其他页面的修改失败，当前草稿已保留，请重试");
|
||||
throw new Error("读取其他页面的修改失败，当前草稿已保留，请重试");
|
||||
throw new CanvasRevisionConflict(
|
||||
"其他页面修改了相同内容，请选择冲突部分保留哪一版",
|
||||
JSON.stringify({ title: remote.data.title, canvasData: remote.data.canvasData }),
|
||||
@ -141,6 +141,14 @@ export function useVideoCanvasPersistence(
|
||||
void queueRef.current?.flush();
|
||||
}, [ready]);
|
||||
|
||||
const saveAndConfirm = useCallback(async () => {
|
||||
if (!ready || !queueRef.current) return false;
|
||||
const queue = queueRef.current;
|
||||
queue.update(latestSnapshot.current);
|
||||
await queue.flush();
|
||||
return !queue.dirty && !queue.failed;
|
||||
}, [ready]);
|
||||
|
||||
useEffect(() => {
|
||||
const onKeyDown = (event: KeyboardEvent) => {
|
||||
if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === "s") {
|
||||
@ -165,5 +173,5 @@ export function useVideoCanvasPersistence(
|
||||
anchor.click();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
}, [projectId]);
|
||||
return { ...state, save, ready, conflicts, resolveConflict, downloadDraft };
|
||||
return { ...state, save, saveAndConfirm, ready, conflicts, resolveConflict, downloadDraft };
|
||||
}
|
||||
|
||||
92
lib/video-canvas/aliyun-video-analysis.ts
Normal file
92
lib/video-canvas/aliyun-video-analysis.ts
Normal file
@ -0,0 +1,92 @@
|
||||
type AliyunVideoModel = "qwen3.8-max" | "qwen3.8-flash" | "qwen3.8-omni-flash";
|
||||
|
||||
export function isAliyunVideoModel(modelId: string, providerType: string) {
|
||||
return providerType === "alibaba" &&
|
||||
["qwen3.8-max", "qwen3.8-flash", "qwen3.8-omni-flash"].includes(modelId);
|
||||
}
|
||||
|
||||
export function supportsVideoSound(modelId: string) {
|
||||
return modelId === "qwen3.8-omni-flash";
|
||||
}
|
||||
|
||||
function officialEndpoint(baseUrl: string) {
|
||||
const url = new URL(baseUrl);
|
||||
if (url.protocol !== "https:" ||
|
||||
!(url.hostname === "dashscope.aliyuncs.com" ||
|
||||
/^[-a-z0-9]+\.[-a-z0-9]+\.maas\.aliyuncs\.com$/.test(url.hostname))) {
|
||||
throw new Error("视频解析仅允许阿里云百炼官方地址");
|
||||
}
|
||||
return `${url.href.replace(/\/$/, "")}/chat/completions`;
|
||||
}
|
||||
|
||||
function sseText(body: string) {
|
||||
let result = "";
|
||||
for (const line of body.split(/\r?\n/)) {
|
||||
if (!line.startsWith("data: ") || line === "data: [DONE]") continue;
|
||||
try {
|
||||
const event = JSON.parse(line.slice(6));
|
||||
result += event.choices?.[0]?.delta?.content || "";
|
||||
} catch {
|
||||
// Ignore keep-alive and incomplete events; a missing result is rejected below.
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function analyzeAliyunVideo({
|
||||
apiKey,
|
||||
baseUrl,
|
||||
modelId,
|
||||
prompt,
|
||||
videoUrl,
|
||||
signal,
|
||||
}: {
|
||||
apiKey: string;
|
||||
baseUrl: string;
|
||||
modelId: AliyunVideoModel;
|
||||
prompt: string;
|
||||
videoUrl: string;
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
const video = new URL(videoUrl);
|
||||
if (video.protocol !== "https:") {
|
||||
throw new Error("阿里云视频解析需要可公网访问的 HTTPS 视频地址");
|
||||
}
|
||||
const omni = supportsVideoSound(modelId);
|
||||
const response = await fetch(officialEndpoint(baseUrl), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: modelId,
|
||||
messages: [{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "video_url", video_url: { url: video.href }, fps: 1 },
|
||||
{ type: "text", text: prompt },
|
||||
],
|
||||
}],
|
||||
stream: omni,
|
||||
...(omni ? { modalities: ["text"], reasoning_effort: "none" } : {
|
||||
temperature: 0.2,
|
||||
max_tokens: 6000,
|
||||
}),
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
let code = "";
|
||||
try {
|
||||
const body = await response.json();
|
||||
code = String(body.error?.code || body.code || "").slice(0, 80);
|
||||
} catch {}
|
||||
throw new Error(`阿里云视频解析失败（HTTP ${response.status}${code ? `，${code}` : ""}）`);
|
||||
}
|
||||
const text = omni
|
||||
? sseText(await response.text())
|
||||
: String((await response.json()).choices?.[0]?.message?.content || "");
|
||||
if (!text.trim()) throw new Error("阿里云视频解析返回空结果");
|
||||
return text;
|
||||
}
|
||||
@ -96,6 +96,12 @@ export type VideoCanvasFunctionConfig = {
|
||||
};
|
||||
|
||||
const defaultFunctions: VideoCanvasFunctionDefinition[] = [
|
||||
{
|
||||
key: "image-generate", label: "图片生成", kind: "image",
|
||||
route: "canvas-generate-image", enabled: true,
|
||||
prompt: "",
|
||||
description: "根据文字提示生成图片，不要求上传参考图。",
|
||||
},
|
||||
{
|
||||
key: "video-prompt-optimize", label: "视频提示词优化", kind: "multimodal",
|
||||
route: "canvas-prompt-tools", enabled: true, requiresBackend: true,
|
||||
@ -297,7 +303,7 @@ const defaultFunctions: VideoCanvasFunctionDefinition[] = [
|
||||
enabled: true,
|
||||
prompt:
|
||||
"解析当前视频的镜头节奏、景别变化、主体动作、背景音乐、人声/音效、关键帧要点，并输出可直接衔接后续图像生成与视频生成节点的结构化故事表。",
|
||||
description: "多模态视频解析：抽取关键帧并输出视频故事表。",
|
||||
description: "阿里云千问3.8可直接理解完整视频画面；音乐与人声需千问3.8 Omni。其他视觉模型回退关键帧解析。",
|
||||
},
|
||||
{
|
||||
key: "image-to-prompt",
|
||||
|
||||
@ -32,11 +32,11 @@ export function connectedInputNodes<T extends ConnectionNode>(nodes: T[], edges:
|
||||
if (seen.has(edge.source)) return [];
|
||||
seen.add(edge.source);
|
||||
const node = nodes.find(item => item.id === edge.source);
|
||||
if (!node) throw Error("上游节点已删除，请移除失效连线后重试");
|
||||
if (!node) throw new Error("上游节点已删除，请移除失效连线后重试");
|
||||
// Also reject cycles restored from an older saved canvas.
|
||||
const remaining = edges.filter(item => item.id !== edge.id);
|
||||
const error = connectionError(nodes, remaining, edge.source, target);
|
||||
if (error && !error.includes("已经连接")) throw Error(error);
|
||||
if (error && !error.includes("已经连接")) throw new Error(error);
|
||||
return [node];
|
||||
});
|
||||
}
|
||||
@ -54,17 +54,17 @@ export function assertConnectedInputsReady(nodes: ConnectionNode[]) {
|
||||
for (const node of nodes) {
|
||||
const draft = node.draftValues || {};
|
||||
if (draft.uploadStatus === "uploading")
|
||||
throw Error(`上游「${node.title || node.id}」仍在上传，请等待上传完成`);
|
||||
throw new Error(`上游「${node.title || node.id}」仍在上传，请等待上传完成`);
|
||||
if (draft.uploadStatus === "failed")
|
||||
throw Error(`上游「${node.title || node.id}」上传失败，请重新选择素材`);
|
||||
throw new Error(`上游「${node.title || node.id}」上传失败，请重新选择素材`);
|
||||
if (["pending", "running", "queued", "failed", "cancelled", "canceled", "interrupted"].includes(draft.generationStatus))
|
||||
throw Error(`上游「${node.title || node.id}」尚未成功完成，请先完成该节点`);
|
||||
throw new Error(`上游「${node.title || node.id}」尚未成功完成，请先完成该节点`);
|
||||
if (node.inputType === "text" || node.kind === "script" || node.kind === "text") {
|
||||
if (!connectedText(node)) throw Error(`上游「${node.title || node.id}」没有文本内容`);
|
||||
if (!connectedText(node)) throw new Error(`上游「${node.title || node.id}」没有文本内容`);
|
||||
} else if (["image", "video", "audio"].includes(node.inputType || "")) {
|
||||
// sourceUrl/poster is an input preview; it is never proof of a generated output.
|
||||
const output = draft.mediaUrl || draft.resultImage || draft.url?.[0] || draft.thumbnailUrl;
|
||||
if (!output) throw Error(`上游「${node.title || node.id}」尚无可用素材，请先上传或生成`);
|
||||
if (!output) throw new Error(`上游「${node.title || node.id}」尚无可用素材，请先上传或生成`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -93,7 +93,7 @@ export function connectionTypeError(source: ConnectionNode, target: ConnectionNo
|
||||
const nodeType = draft.nodeType;
|
||||
if (source.kind === "material-effect" || source.kind === "material-role")
|
||||
return target.inputType === "video" && nodeType !== "smart-edit" ? null : "角色和视频特效素材只能连接到视频生成节点";
|
||||
const allowed = nodeType === "smart-edit" || nodeType === "video-clip" || draft.action === "VIDEO_HD_UPSCALE" ? ["video"]
|
||||
const allowed = nodeType === "smart-edit" || nodeType === "video-clip" || nodeType === "frame-analysis" || draft.action === "VIDEO_HD_UPSCALE" ? ["video"]
|
||||
: target.inputType === "image" ? ["image", "text"]
|
||||
: target.inputType === "video" ? (key === "audio-to-video" ? ["image", "text", "audio"] : ["image", "text"])
|
||||
: target.inputType === "audio" ? ["text"]
|
||||
@ -115,6 +115,6 @@ export function videoInputLimitError(mode: string, count: number) {
|
||||
export function requireResolvedReferenceImages(urls: string[], expectedCount: number) {
|
||||
const images = urls.map(url => url.trim());
|
||||
if (images.length !== expectedCount || images.some(url => !url))
|
||||
throw Error("部分宫格参考图准备失败，尚未提交生成，请重试；不会使用整张原图代替");
|
||||
throw new Error("部分宫格参考图准备失败，尚未提交生成，请重试；不会使用整张原图代替");
|
||||
return images;
|
||||
}
|
||||
|
||||
@ -49,6 +49,9 @@ export type DirectorScene = {
|
||||
objectTracks?: Record<string, DirectorObjectTrack>;
|
||||
backgroundUrl?: string;
|
||||
backgroundHistory?: string[];
|
||||
skyColor?: string;
|
||||
gridVisible?: boolean;
|
||||
groundVisible?: boolean;
|
||||
};
|
||||
export const directorRatios = ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"];
|
||||
export function directorImageSize(ratio: string) {
|
||||
@ -84,6 +87,9 @@ export function defaultDirectorScene(): DirectorScene {
|
||||
motion: defaultDirectorMotion(),
|
||||
motionTracks: {},
|
||||
objectTracks: {},
|
||||
skyColor: "#080808",
|
||||
gridVisible: true,
|
||||
groundVisible: true,
|
||||
};
|
||||
}
|
||||
export function readDirectorScene(value: unknown): DirectorScene {
|
||||
@ -157,6 +163,9 @@ export function readDirectorScene(value: unknown): DirectorScene {
|
||||
? v.activeCameraId
|
||||
: validCameras[0].id,
|
||||
ratio: directorRatios.includes(v.ratio) ? v.ratio : "16:9",
|
||||
skyColor: typeof v.skyColor === "string" && /^#[0-9a-f]{6}$/i.test(v.skyColor) ? v.skyColor : fallback.skyColor,
|
||||
gridVisible: v.gridVisible !== false,
|
||||
groundVisible: v.groundVisible !== false,
|
||||
snapshots,
|
||||
motion: readDirectorMotion(v.motion),
|
||||
...(typeof v.backgroundUrl === "string" && /^https?:\/\//i.test(v.backgroundUrl)
|
||||
|
||||
@ -113,11 +113,12 @@ export function chooseStoryVideoSettings(models: CanvasModelDescriptor[], hasIma
|
||||
const mode:VideoGenerationMode = hasImage ? "first-frame" : "text-to-video";
|
||||
const choices = filterCanvasModels(models,"video",mode);
|
||||
const model = choices.find(item=>item.id===preferredRef||item.modelId===preferredRef) || choices[0];
|
||||
if (!model) throw Error(hasImage ? "没有可用的首帧视频模型" : "镜头缺少首帧，且没有可用的文生视频模型");
|
||||
if (!model) throw new Error(hasImage ? "没有可用的首帧视频模型" : "镜头缺少首帧，且没有可用的文生视频模型");
|
||||
const options = getCanvasVideoParameters(model,mode);
|
||||
if (!options.durations.length || !options.ratios.length || !options.resolutions.length) throw Error("视频模型缺少已确认的时长、比例或清晰度规格，请先完善模型配置");
|
||||
const executableResolutions = options.resolutions.filter(value => ["480p", "720p", "1080p"].includes(value.toLowerCase()));
|
||||
if (!options.durations.length || !options.ratios.length || !executableResolutions.length) throw new Error("视频模型缺少已确认且可执行的时长、比例或清晰度规格，请先完善模型配置");
|
||||
const duration = options.durations.reduce((best,value)=>Math.abs(value-seconds)<Math.abs(best-seconds)?value:best,options.durations[0]);
|
||||
return {mode,model:model.id||model.modelId||"",duration,
|
||||
ratio:options.ratios.includes("16:9")?"16:9":options.ratios[0],
|
||||
resolution:options.resolutions.includes("720p")?"720p":options.resolutions[0],generateAudio:options.supportsAudio};
|
||||
resolution:executableResolutions.includes("720p")?"720p":executableResolutions[0],generateAudio:options.supportsAudio};
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { reconstructionPrompt, parseReconstructedScene } from "./director-reconstruction";
|
||||
import { normalizeVideoAnalysisRows } from "./video-story";
|
||||
import { analyzeAliyunVideo, isAliyunVideoModel, supportsVideoSound } from "./aliyun-video-analysis";
|
||||
import "server-only";
|
||||
import { normalizeStoryScriptRows } from "@/lib/video-canvas/story-script";
|
||||
import { isCanvasLanguageModel, isCanvasVisionModel } from "@/lib/video-canvas/model-matching";
|
||||
@ -20,7 +21,7 @@ import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider } from "@/l
|
||||
const execFileAsync = promisify(execFile);
|
||||
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
|
||||
const VIDEO_ANALYSIS_MAX_SIDE = 960;
|
||||
const VIDEO_ANALYSIS_TIMEOUT_MS = 15000;
|
||||
const VIDEO_ANALYSIS_TIMEOUT_MS = 30000;
|
||||
|
||||
const requestSchema = z.object({
|
||||
mode: z.enum([
|
||||
@ -33,6 +34,11 @@ const requestSchema = z.object({
|
||||
prompt: z.string().trim().optional(),
|
||||
imageUrls: z.array(z.string().trim()).optional(),
|
||||
videoUrls: z.array(z.string().trim()).optional(),
|
||||
analysisDimensions: z.object({
|
||||
shots: z.boolean().optional(),
|
||||
motion: z.boolean().optional(),
|
||||
music: z.boolean().optional(),
|
||||
}).optional(),
|
||||
nodeTitle: z.string().trim().optional(),
|
||||
preferredModelRef: z.string().trim().optional(),
|
||||
model: z.string().trim().optional(),
|
||||
@ -163,13 +169,20 @@ function buildVideoAnalysisPrompt(params: {
|
||||
frameCount: number;
|
||||
imageCount: number;
|
||||
videoCount: number;
|
||||
directVideo: boolean;
|
||||
soundAvailable: boolean;
|
||||
}) {
|
||||
return [
|
||||
"你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。",
|
||||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||||
`上游参考图数量：${params.imageCount}`,
|
||||
`上游视频数量：${params.videoCount}`,
|
||||
`已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
|
||||
params.directVideo
|
||||
? "已直接提供原视频文件，请根据完整时间轴识别镜头，不要只依据少量预览关键帧。"
|
||||
: `已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
|
||||
params.soundAvailable
|
||||
? "可分析原视频声音；背景音乐、人声和音效须以实际听到的内容为准。"
|
||||
: "当前模型只能看画面，不能听声音；背景音乐和人声/音效字段必须留空，禁止推测。",
|
||||
params.prompt
|
||||
? `上游脚本/说明：\n${params.prompt}`
|
||||
: "上游没有额外脚本，请重点依据参考画面分析。",
|
||||
@ -208,7 +221,7 @@ function buildVideoAnalysisPrompt(params: {
|
||||
"analysisRows 请按镜头时间顺序输出；短视频可以输出 1 行，复杂视频输出 2-6 行。",
|
||||
"每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。",
|
||||
"imagePrompt 和 motionPrompt 优先使用英文；其他字段使用中文；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。",
|
||||
"如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。",
|
||||
"没有实际看到或听到的内容必须留空，不可依据脚本虚构成视频事实。",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
@ -275,8 +288,26 @@ async function extractVideoFrameDataUrls(
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
|
||||
try {
|
||||
const durationSeconds = await getVideoDurationSeconds(videoUrl);
|
||||
if (durationSeconds && durationSeconds <= 120) {
|
||||
// Decode short clips in one pass. Four concurrent HTTP seeks are slow and
|
||||
// some CDNs reject later range requests even when the video is valid.
|
||||
await execFileAsync("ffmpeg", [
|
||||
"-hide_banner", "-loglevel", "error", "-i", videoUrl,
|
||||
"-vf", `fps=${(frameCount / durationSeconds).toFixed(6)}`,
|
||||
"-frames:v", String(frameCount), "-q:v", "4", "-y",
|
||||
path.join(tempDir, "frame-%d.jpg"),
|
||||
], { timeout: VIDEO_ANALYSIS_TIMEOUT_MS, maxBuffer: 2 * 1024 * 1024 });
|
||||
const frames = await Promise.allSettled(
|
||||
Array.from({ length: frameCount }, async (_, index) => {
|
||||
const bytes = await readFile(path.join(tempDir, `frame-${index + 1}.jpg`));
|
||||
const processed = await downscaleVideoFrameForVision(bytes);
|
||||
return `data:image/jpeg;base64,${processed.toString("base64")}`;
|
||||
}),
|
||||
);
|
||||
return frames.flatMap((result) => result.status === "fulfilled" ? [result.value] : []);
|
||||
}
|
||||
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
|
||||
const frames = await Promise.all(
|
||||
const frames = await Promise.allSettled(
|
||||
timestamps.map(async (timestamp, index) => {
|
||||
const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`);
|
||||
await execFileAsync(
|
||||
@ -306,7 +337,14 @@ async function extractVideoFrameDataUrls(
|
||||
return `data:image/jpeg;base64,${processed.toString("base64")}`;
|
||||
}),
|
||||
);
|
||||
return frames.filter(Boolean);
|
||||
const completed = frames.flatMap((result) => result.status === "fulfilled" ? [result.value] : []);
|
||||
if (completed.length < frames.length) {
|
||||
console.warn("[Canvas Workflow Analyze] some preview frames could not be extracted", {
|
||||
completed: completed.length,
|
||||
requested: frames.length,
|
||||
});
|
||||
}
|
||||
return completed;
|
||||
} catch (error) {
|
||||
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
@ -459,10 +497,12 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
const prompt = normalizeText(parsed.data.prompt);
|
||||
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
|
||||
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
|
||||
const extractedVideoFrames =
|
||||
parsed.data.mode === "video-analysis" && videoUrls.length > 0
|
||||
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
|
||||
: [];
|
||||
if (parsed.data.mode === "video-analysis" && videoUrls.length !== 1) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "视频拉片需要恰好一个可访问的视频地址" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
|
||||
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
|
||||
return NextResponse.json(
|
||||
@ -475,7 +515,9 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
getEnabledAiModelsWithProvider(false),
|
||||
getAgentAiModelWithProvider(),
|
||||
]);
|
||||
const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
|
||||
const hasVisionInputs = imageUrls.length > 0 || videoUrls.length > 0;
|
||||
const wantsSound = parsed.data.mode === "video-analysis" &&
|
||||
parsed.data.analysisDimensions?.music === true;
|
||||
const compatibleModels = allModels.filter((item) => {
|
||||
const descriptor = { ...item.model, providerType: item.provider.type };
|
||||
return (
|
||||
@ -497,9 +539,25 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
);
|
||||
}
|
||||
const selectedModel =
|
||||
preferredModel ||
|
||||
(wantsSound
|
||||
? compatibleModels.find((item) =>
|
||||
item.provider.type === "alibaba" &&
|
||||
supportsVideoSound(item.model.modelId),
|
||||
)
|
||||
: preferredModel) ||
|
||||
(parsed.data.mode === "video-analysis"
|
||||
? compatibleModels.find((item) =>
|
||||
item.provider.type === "alibaba" && item.model.modelId === "qwen3.8-max",
|
||||
)
|
||||
: undefined) ||
|
||||
compatibleModels.find((item) => item.model.id === agentModel?.model.id) ||
|
||||
compatibleModels[0];
|
||||
if (wantsSound && !selectedModel?.model.modelId?.includes("omni")) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "音乐/人声拉片需要启用阿里云 qwen3.8-omni-flash 视频音频理解模型" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
if (!selectedModel) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
@ -510,10 +568,27 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
);
|
||||
}
|
||||
|
||||
const directVideo = parsed.data.mode === "video-analysis" && videoUrls.length > 0 &&
|
||||
isAliyunVideoModel(selectedModel.model.modelId, selectedModel.provider.type);
|
||||
const soundAvailable = directVideo && supportsVideoSound(selectedModel.model.modelId);
|
||||
const extractedVideoFrames = !directVideo && parsed.data.mode === "video-analysis"
|
||||
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
|
||||
: [];
|
||||
// The official model receives the full video. Extract UI previews alongside its request.
|
||||
const framePreviewsPromise = directVideo
|
||||
? extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
|
||||
: Promise.resolve(extractedVideoFrames);
|
||||
if (parsed.data.mode === "video-analysis" && videoUrls.length > 0 &&
|
||||
!directVideo && extractedVideoFrames.length === 0) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "无法读取视频关键帧，请检查视频地址或选择阿里云视频理解模型" },
|
||||
{ status: 422 },
|
||||
);
|
||||
}
|
||||
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
|
||||
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
|
||||
const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
|
||||
if (!model) {
|
||||
const model = directVideo ? null : await getLanguageModelById(selectedModel.model.id, codingModelId);
|
||||
if (!directVideo && !model) {
|
||||
return NextResponse.json({ success: false, error: "分析模型初始化失败" }, { status: 500 });
|
||||
}
|
||||
|
||||
@ -527,6 +602,8 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
prompt,
|
||||
imageCount: imageUrls.length,
|
||||
videoCount: videoUrls.length,
|
||||
directVideo,
|
||||
soundAvailable,
|
||||
})
|
||||
: parsed.data.mode === "character-scene-extraction"
|
||||
? buildCharacterScenePrompt({
|
||||
@ -548,7 +625,7 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
videoCount: videoUrls.length,
|
||||
});
|
||||
|
||||
const inlinedImageReferences = useVision
|
||||
const inlinedImageReferences = useVision && !directVideo
|
||||
? (
|
||||
await inlineVisionImages(imageUrls, {
|
||||
maxImages: 4,
|
||||
@ -564,9 +641,20 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
: [];
|
||||
|
||||
if (parsed.data.mode === "director-reconstruct" && !referenceImages.length)
|
||||
throw Error("识图重建需要一张可读取的图片");
|
||||
const { text, finishReason } = await generateText({
|
||||
model,
|
||||
throw new Error("识图重建需要一张可读取的图片");
|
||||
const generated = directVideo
|
||||
? { text: await analyzeAliyunVideo({
|
||||
apiKey: selectedModel.provider.apiKey,
|
||||
baseUrl: selectedModel.provider.baseUrl,
|
||||
modelId: selectedModel.model.modelId as "qwen3.8-max" | "qwen3.8-flash" | "qwen3.8-omni-flash",
|
||||
prompt: promptText,
|
||||
videoUrl: videoUrls[0],
|
||||
signal: signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(300_000)])
|
||||
: AbortSignal.timeout(300_000),
|
||||
}), finishReason: "stop" }
|
||||
: await generateText({
|
||||
model: model!,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
@ -587,6 +675,7 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(150_000)])
|
||||
: AbortSignal.timeout(150_000),
|
||||
});
|
||||
const { text, finishReason } = generated;
|
||||
|
||||
if (finishReason === "length" || !text.trim()) {
|
||||
return NextResponse.json(
|
||||
@ -613,10 +702,11 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
},
|
||||
});
|
||||
if (parsed.data.mode === "video-analysis") {
|
||||
const framePreviews = await framePreviewsPromise;
|
||||
const analysisRows = normalizeVideoAnalysisRows(
|
||||
parsedJson.analysisRows,
|
||||
extractedVideoFrames,
|
||||
);
|
||||
framePreviews,
|
||||
).map((row) => soundAvailable ? row : { ...row, backgroundMusic: "", voiceAndSfx: "" });
|
||||
if (!analysisRows.length)
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "视频解析没有返回有效镜头，请重新解析" },
|
||||
@ -634,7 +724,7 @@ export async function executeWorkflowAnalysisResponse(body: unknown, signal?: Ab
|
||||
data: {
|
||||
summary,
|
||||
analysisRows,
|
||||
framePreviews: extractedVideoFrames,
|
||||
framePreviews,
|
||||
highlights: normalizeStringList(parsedJson.highlights, 6),
|
||||
shotIdeas,
|
||||
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
|
||||
|
||||
@ -28,6 +28,22 @@ function resolvePascalEditorFrameSources(isDev: boolean) {
|
||||
return sources.filter(Boolean).join(" ");
|
||||
}
|
||||
|
||||
function resolveDevMediaOrigins(isDev: boolean) {
|
||||
if (!isDev) return "";
|
||||
const origins = [process.env.S3_PUBLIC_URL, process.env.S3_ENDPOINT].flatMap((raw) => {
|
||||
if (!raw) return [];
|
||||
try {
|
||||
const url = new URL(raw);
|
||||
return url.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)
|
||||
? [url.origin]
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
return [...new Set(origins)].join(" ");
|
||||
}
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
distDir: process.env.NEXT_DIST_DIR?.trim() || ".next",
|
||||
poweredByHeader: false,
|
||||
@ -77,6 +93,7 @@ const nextConfig: NextConfig = {
|
||||
async headers() {
|
||||
const isDev = process.env.NODE_ENV === "development";
|
||||
const frameSources = resolvePascalEditorFrameSources(isDev);
|
||||
const devMediaOrigins = resolveDevMediaOrigins(isDev);
|
||||
const csp = [
|
||||
"default-src 'self'",
|
||||
// Next.js App Router/RSC emits inline bootstrap scripts in production.
|
||||
@ -87,8 +104,8 @@ const nextConfig: NextConfig = {
|
||||
"img-src 'self' data: blob: https: http:",
|
||||
"font-src 'self' data:",
|
||||
// Spark bundles its WASM decoder as a data: URL and initializes it via fetch().
|
||||
"connect-src 'self' data: https: wss:",
|
||||
"media-src 'self' blob: https:",
|
||||
`connect-src 'self' data: https: wss: ${devMediaOrigins}`,
|
||||
`media-src 'self' blob: https: ${devMediaOrigins}`,
|
||||
`frame-src ${frameSources}`,
|
||||
"worker-src 'self' blob:",
|
||||
"object-src 'none'",
|
||||
|
||||
@ -11,6 +11,7 @@ export default defineConfig({
|
||||
/routes\/video-canvas-tasks\.test\.ts/,
|
||||
/routes\/video-canvas-director\.test\.ts/,
|
||||
/routes\/aliyun-canvas-video\.test\.ts/,
|
||||
/routes\/video-canvas-aliyun-analysis\.test\.ts/,
|
||||
/routes\/video-canvas-followup\.test\.ts/,
|
||||
/routes\/video-canvas-tools\.test\.ts/,
|
||||
/routes\/video-canvas-save-queue\.test\.ts/,
|
||||
|
||||
50
scripts/configure-aliyun-canvas-analysis.cjs
Normal file
50
scripts/configure-aliyun-canvas-analysis.cjs
Normal file
@ -0,0 +1,50 @@
|
||||
"use strict";
|
||||
// On the application host: APP_ROOT=/var/www/ai-chatbot node scripts/configure-aliyun-canvas-analysis.cjs [--apply]
|
||||
// Dry-run by default. Reuses the existing official Aliyun provider and never prints its key.
|
||||
const fs = require("node:fs");
|
||||
const root = process.env.APP_ROOT || process.cwd();
|
||||
require(root + "/node_modules/dotenv").config({ path: root + "/.env.local", quiet: true });
|
||||
const sql = require(root + "/node_modules/postgres")(process.env.POSTGRES_URL, { max: 1 });
|
||||
const apply = process.argv.includes("--apply");
|
||||
const models = [
|
||||
{ id: "qwen3.8-max", name: "千问3.8 Max（视频画面理解）", order: 410 },
|
||||
{ id: "qwen3.8-omni-flash", name: "千问3.8 Omni Flash（视频画面与声音）", order: 411 },
|
||||
];
|
||||
|
||||
(async () => {
|
||||
const providers = await sql`select id,"baseUrl","isEnabled",("apiKey" is not null and length("apiKey") > 0) as "hasKey" from "AiProvider" where type='alibaba'`;
|
||||
const provider = providers.find((row) => {
|
||||
try {
|
||||
return row.isEnabled && row.hasKey &&
|
||||
new URL(row.baseUrl).hostname === "dashscope.aliyuncs.com";
|
||||
} catch { return false; }
|
||||
});
|
||||
if (!provider) throw new Error("Enabled official Aliyun provider with key not found");
|
||||
const existingModels = await sql`select * from "AiModel" where "providerId"=${provider.id} and "modelId" in ${sql(models.map((m) => m.id))}`;
|
||||
if (apply) {
|
||||
const dir = root + "/.model-config-backups";
|
||||
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(dir + "/canvas-analysis-" + Date.now() + ".json",
|
||||
JSON.stringify({ providerId: provider.id, models: existingModels }, null, 2), { mode: 0o600 });
|
||||
}
|
||||
await sql.begin(async (tx) => {
|
||||
for (const model of models) {
|
||||
const rows = existingModels.filter((row) => row.modelId === model.id);
|
||||
if (rows.length > 1) throw new Error("Duplicate model rows: " + model.id);
|
||||
const row = rows[0];
|
||||
if (apply && row) {
|
||||
await tx`update "AiModel" set "isEnabled"=true,"supportsVision"=true,
|
||||
"updatedAt"=now() where id=${row.id}`;
|
||||
} else if (apply) {
|
||||
await tx`insert into "AiModel" ("providerId","modelId","displayName","supportsVision","isEnabled","isHidden",capabilities,"sortOrder")
|
||||
values (${provider.id},${model.id},${model.name},true,true,false,
|
||||
${tx.json({ kind: "chat", modality: ["text", "image", "video"], supportsVideo: true, supportsVideoAudio: model.id.includes("omni") })},${model.order})`;
|
||||
}
|
||||
console.log(JSON.stringify({ mode: apply ? "applied" : "dry-run", modelId: model.id,
|
||||
existed: !!row, enabled: apply ? true : row?.isEnabled || false, vision: apply ? true : row?.supportsVision || false }));
|
||||
}
|
||||
});
|
||||
})().catch((error) => {
|
||||
console.error(error.name || "Configuration error");
|
||||
process.exitCode = 1;
|
||||
}).finally(() => sql.end());
|
||||
46
tests/routes/video-canvas-aliyun-analysis.test.ts
Normal file
46
tests/routes/video-canvas-aliyun-analysis.test.ts
Normal file
@ -0,0 +1,46 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { analyzeAliyunVideo, isAliyunVideoModel, supportsVideoSound } from "@/lib/video-canvas/aliyun-video-analysis";
|
||||
|
||||
test("only official Alibaba video-understanding models use direct video input", () => {
|
||||
expect(isAliyunVideoModel("qwen3.8-max", "alibaba")).toBe(true);
|
||||
expect(isAliyunVideoModel("qwen3.8-omni-flash", "alibaba")).toBe(true);
|
||||
expect(isAliyunVideoModel("qwen3-vl-plus", "alibaba")).toBe(false);
|
||||
expect(isAliyunVideoModel("qwen3.8-max", "custom")).toBe(false);
|
||||
expect(supportsVideoSound("qwen3.8-max")).toBe(false);
|
||||
expect(supportsVideoSound("qwen3.8-omni-flash")).toBe(true);
|
||||
});
|
||||
|
||||
test("direct visual analysis sends the full video to the official endpoint", async () => {
|
||||
const original = globalThis.fetch;
|
||||
let body: any;
|
||||
globalThis.fetch = async (input, init) => {
|
||||
expect(String(input)).toBe("https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions");
|
||||
body = JSON.parse(String(init?.body));
|
||||
return Response.json({ choices: [{ message: { content: '{"summary":"已解析真实画面"}' } }] });
|
||||
};
|
||||
try {
|
||||
const result = await analyzeAliyunVideo({ apiKey: "test-key", baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1", modelId: "qwen3.8-max", prompt: "按时间分析", videoUrl: "https://example.com/test.mp4" });
|
||||
expect(result).toContain("已解析真实画面");
|
||||
expect(body.messages[0].content[0]).toMatchObject({ type: "video_url", video_url: { url: "https://example.com/test.mp4" } });
|
||||
expect(body.stream).toBe(false);
|
||||
} finally { globalThis.fetch = original; }
|
||||
});
|
||||
|
||||
test("Omni parses streaming video and sound response", async () => {
|
||||
const original = globalThis.fetch;
|
||||
globalThis.fetch = async (_input, init) => {
|
||||
const body = JSON.parse(String(init?.body));
|
||||
expect(body.model).toBe("qwen3.8-omni-flash");
|
||||
expect(body.stream).toBe(true);
|
||||
expect(body.modalities).toEqual(["text"]);
|
||||
return new Response('data: {"choices":[{"delta":{"content":"有人声"}}]}\n\ndata: [DONE]\n', { status: 200 });
|
||||
};
|
||||
try {
|
||||
expect(await analyzeAliyunVideo({ apiKey: "test-key", baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1", modelId: "qwen3.8-omni-flash", prompt: "听声音", videoUrl: "https://example.com/test.mp4" })).toBe("有人声");
|
||||
} finally { globalThis.fetch = original; }
|
||||
});
|
||||
|
||||
test("rejects private video URLs and unofficial provider endpoints", async () => {
|
||||
await expect(analyzeAliyunVideo({ apiKey: "test-key", baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1", modelId: "qwen3.8-max", prompt: "分析", videoUrl: "http://127.0.0.1/test.mp4" })).rejects.toThrow("公网访问");
|
||||
await expect(analyzeAliyunVideo({ apiKey: "test-key", baseUrl: "https://example.com/v1", modelId: "qwen3.8-max", prompt: "分析", videoUrl: "https://example.com/test.mp4" })).rejects.toThrow("官方地址");
|
||||
});
|
||||
@ -67,6 +67,9 @@ test("unsupported input types cannot produce decorative connections", () => {
|
||||
expect(connectionTypeError({ id: "v", inputType: "video" }, { id: "edit", inputType: "video", draftValues: { nodeType: "smart-edit" } })).toBeNull();
|
||||
expect(connectionTypeError({ id: "t", inputType: "text" }, { id: "tts", inputType: "audio" })).toBeNull();
|
||||
expect(connectionTypeError({ id: "a", inputType: "audio" }, { ...target, draftValues: { targetPreset: "audio-to-video" } })).toBeNull();
|
||||
const frameAnalysis = { id: "frame", inputType: "text", draftValues: { nodeType: "frame-analysis" } };
|
||||
expect(connectionTypeError({ id: "v", inputType: "video" }, frameAnalysis)).toBeNull();
|
||||
expect(connectionTypeError(image("a"), frameAnalysis)).toContain("不能消费");
|
||||
});
|
||||
|
||||
|
||||
|
||||
@ -35,6 +35,18 @@ test("director preserves real model, camera and screenshot through JSON save/rel
|
||||
});
|
||||
expect(readDirectorScene(JSON.parse(JSON.stringify(scene)))).toEqual(scene);
|
||||
});
|
||||
test("director scene appearance controls persist and reject malformed colors", () => {
|
||||
const scene = defaultDirectorScene();
|
||||
scene.skyColor = "#123456";
|
||||
scene.gridVisible = false;
|
||||
scene.groundVisible = false;
|
||||
expect(readDirectorScene(JSON.parse(JSON.stringify(scene)))).toEqual(scene);
|
||||
expect(readDirectorScene({ ...scene, skyColor: "red;url(x)", gridVisible: "false", groundVisible: null })).toMatchObject({
|
||||
skyColor: "#080808",
|
||||
gridVisible: true,
|
||||
groundVisible: true,
|
||||
});
|
||||
});
|
||||
test("corrupt persisted camera/transform data cannot enter the 3D renderer", () => {
|
||||
const scene = readDirectorScene({
|
||||
objects: [
|
||||
@ -138,7 +150,7 @@ test("timeline sampling clamps endpoints, holds outside keyframes and differenti
|
||||
expect(sampleDirectorMotion(motion, -20)?.position).toEqual(motion.keyframes[0].position);
|
||||
expect(sampleDirectorMotion(motion, 30)?.position).toEqual(motion.keyframes[1].position);
|
||||
expect(sampleDirectorMotion(motion, 2.5)?.position[2]).toBe(6);
|
||||
const linear = sampleDirectorMotion(motion, 1)?.position[2]!;
|
||||
const linear = sampleDirectorMotion(motion, 1)?.position[2] ?? 0;
|
||||
motion.easing = "smooth";
|
||||
expect(sampleDirectorMotion(motion, 1)?.position[2]).toBeGreaterThan(linear);
|
||||
motion.keyframes[0].time = 1;
|
||||
@ -244,7 +256,7 @@ test("recording stops tracks and rejects partial exports on cancel or background
|
||||
duration: 2,
|
||||
signal: controller.signal,
|
||||
render: (t) => {
|
||||
if (outcome === "render-error" && t > 0) throw Error("render fault");
|
||||
if (outcome === "render-error" && t > 0) throw new Error("render fault");
|
||||
rendered.push(t);
|
||||
},
|
||||
});
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import {test, expect} from '@playwright/test';
|
||||
import {filterCanvasModels, getCanvasVideoModes, getCanvasVideoParameters, validateCanvasVideoInputs} from '@/lib/video-canvas/model-matching';
|
||||
import {chooseStoryVideoSettings, filterCanvasModels, getCanvasVideoModes, getCanvasVideoParameters, validateCanvasVideoInputs} from '@/lib/video-canvas/model-matching';
|
||||
|
||||
const language = {id: 'text', modelId: 'agnes-1.5-flash', providerType:'custom', supportsVision:true, capabilities:{kind:'text', provider:'agnes-ai'}};
|
||||
const image = {id:'image', modelId:'gpt-image-2', providerType:'vip98k', supportsImageGen:true, supportsVision:true, capabilities:{kind:'image'}};
|
||||
@ -34,6 +34,11 @@ test('mode parameter constraints override generic choices and never add 4K', ()
|
||||
test('preserves configured discrete duration choices', () => {
|
||||
expect(getCanvasVideoParameters(video).durations).toEqual([5,10]);
|
||||
});
|
||||
test('story generation never submits a 4K-only model through the 1080P pipeline', () => {
|
||||
const textVideo = {...video, capabilities:{...video.capabilities, supportedModes:['text-to-video'], supportedResolutions:['4k']}};
|
||||
expect(() => chooseStoryVideoSettings([textVideo], false, 5)).toThrow('可执行');
|
||||
expect(chooseStoryVideoSettings([{...textVideo, capabilities:{...textVideo.capabilities, supportedResolutions:['4k','1080p']}}], false, 5).resolution).toBe('1080p');
|
||||
});
|
||||
test('rejects missing tail frame and references silently attached to text-to-video', () => {
|
||||
expect(()=>validateCanvasVideoInputs('first-last-frame',['head'])).toThrow('两张');
|
||||
expect(()=>validateCanvasVideoInputs('text-to-video',['head'])).toThrow('切换');
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user