From 38be09de9ca187ee56560e8453054a114487590b Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 17 Sep 2026 00:02:03 +0800 Subject: [PATCH] feat(video-canvas): connect real 3d director and validate media chain --- app/api/canvas/3d/model-proxy/route.ts | 1 + app/api/canvas/[id]/route.ts | 149 +++--- app/api/canvas/director/upload-model/route.ts | 45 ++ app/api/canvas/prompt-tools/route.ts | 4 + .../canvas/LiblibVideoCanvasReplica.tsx | 137 +++--- .../canvas/director/DirectorStudioDialog.tsx | 438 ++++++++++++++++++ .../canvas/director/DirectorViewport.tsx | 359 ++++++++++++++ .../video-canvas-director-chain-2026-09-16.md | 49 ++ lib/canvas/three-asset-url.ts | 1 + lib/db/queries.ts | 22 +- lib/video-canvas/director-glb.ts | 42 ++ lib/video-canvas/director-scene.ts | 124 +++++ playwright.helpers.config.ts | 1 + tests/routes/video-canvas-director.test.ts | 106 +++++ 14 files changed, 1320 insertions(+), 158 deletions(-) create mode 100644 app/api/canvas/director/upload-model/route.ts create mode 100644 components/canvas/director/DirectorStudioDialog.tsx create mode 100644 components/canvas/director/DirectorViewport.tsx create mode 100644 docs/video-canvas-director-chain-2026-09-16.md create mode 100644 lib/video-canvas/director-glb.ts create mode 100644 lib/video-canvas/director-scene.ts create mode 100644 tests/routes/video-canvas-director.test.ts diff --git a/app/api/canvas/3d/model-proxy/route.ts b/app/api/canvas/3d/model-proxy/route.ts index f7be306e6..c9a0db29a 100644 --- a/app/api/canvas/3d/model-proxy/route.ts +++ b/app/api/canvas/3d/model-proxy/route.ts @@ -14,6 +14,7 @@ const ALLOWED_MODEL_HOST_DOMAINS = [ "worldlabs.ai", "s3.1080p.hk", "img.yydd.ai", + "scqny.jiansun.vip", "blob.vercel-storage.com", "aliyuncs.com", "amazonaws.com", diff --git a/app/api/canvas/[id]/route.ts b/app/api/canvas/[id]/route.ts index bb8f5a7dc..e76915da0 100644 --- a/app/api/canvas/[id]/route.ts +++ b/app/api/canvas/[id]/route.ts @@ -102,7 +102,7 @@ export async function GET( // 更新画布（支持 upsert：如果不存在则创建） export async function PUT( request: NextRequest, - { params }: { params: Promise<{ id: string }> } + { params }: { params: Promise<{ id: string }> }, ) { try { const { id } = await params; @@ -116,29 +116,33 @@ export async function PUT( const session = await auth(); if (!session?.user?.id) { - return NextResponse.json({ success: false, error: '未登录' }, { status: 401 }); + return NextResponse.json( + { success: false, error: "未登录" }, + { status: 401 }, + ); } const { title, description, thumbnail, canvasData } = body; - const baseCanvasRevision = body?.baseCanvasUpdatedAt || body?.baseCanvasRevision; + const baseCanvasRevision = + body?.baseCanvasUpdatedAt || body?.baseCanvasRevision; const workspaceType = body && Object.prototype.hasOwnProperty.call(body, "workspaceType") ? resolveWorkspaceType(body.workspaceType) : undefined; - // 先检查画布是否存在 - const [existingProject] = await db - .select() - .from(canvasProject) - .where( - and( - eq(canvasProject.id, id), - eq(canvasProject.userId, session.user.id) + return await db.transaction(async (tx) => { + const [existingProject] = await tx + .select() + .from(canvasProject) + .where( + and( + eq(canvasProject.id, id), + eq(canvasProject.userId, session.user.id), + ), ) - ); - - if (existingProject) { + .for("update"); if ( + existingProject && canvasData !== undefined && baseCanvasRevision && isStaleCanvasSaveBaseRevision({ @@ -149,101 +153,64 @@ export async function PUT( ) { return NextResponse.json( buildStaleCanvasSaveConflictPayload(existingProject.updatedAt), - { status: 409 } + { status: 409 }, ); } - + if (!existingProject) { + // The parent must exist before node/asset foreign keys are synchronized. + const [created] = await tx + .insert(canvasProject) + .values({ + id, + userId: session.user.id, + title: title || getDefaultWorkspaceTitle(workspaceType || "canvas"), + workspaceType: workspaceType || "canvas", + canvasData: { + nodes: [], + edges: [], + viewport: { x: 0, y: 0, zoom: 1 }, + }, + }) + .onConflictDoNothing({ target: canvasProject.id }) + .returning(); + if (!created) + return NextResponse.json( + { success: false, error: "画布已被创建，请重新加载后保存" }, + { status: 409 }, + ); + } const syncedCanvasData = canvasData !== undefined - ? await syncCanvasNodesForProject({ - projectId: id, - userId: session.user.id, - canvasData, - }) + ? await syncCanvasNodesForProject( + { projectId: id, userId: session.user.id, canvasData }, + tx, + ) : undefined; - - // 更新现有画布 const updateData: any = { updatedAt: new Date() }; if (title !== undefined) updateData.title = title; if (description !== undefined) updateData.description = description; if (thumbnail !== undefined) updateData.thumbnail = thumbnail; if (workspaceType !== undefined) updateData.workspaceType = workspaceType; - if (syncedCanvasData !== undefined) updateData.canvasData = syncedCanvasData; - - const [updatedProject] = await db + if (syncedCanvasData !== undefined) + updateData.canvasData = syncedCanvasData; + const [saved] = await tx .update(canvasProject) .set(updateData) .where( and( eq(canvasProject.id, id), - eq(canvasProject.userId, session.user.id) - ) + eq(canvasProject.userId, session.user.id), + ), ) .returning(); - - return NextResponse.json({ success: true, data: updatedProject }); - } else { - const syncedCanvasData = - canvasData !== undefined - ? await syncCanvasNodesForProject({ - projectId: id, - userId: session.user.id, - canvasData, - }) - : undefined; - - const nextWorkspaceType = workspaceType || "canvas"; - const nextCanvasData = syncedCanvasData || { - nodes: [], - edges: [], - viewport: { x: 0, y: 0, zoom: 1 }, - }; - const nextTitle = title || getDefaultWorkspaceTitle(nextWorkspaceType); - const insertData: any = { - id, // 使用指定的 ID - userId: session.user.id, - title: nextTitle, - workspaceType: nextWorkspaceType, - canvasData: nextCanvasData, - }; - const conflictUpdateData: any = { - title: nextTitle, - workspaceType: nextWorkspaceType, - canvasData: nextCanvasData, - updatedAt: new Date(), - }; - if (description !== undefined) { - insertData.description = description; - conflictUpdateData.description = description; - } - if (thumbnail !== undefined) { - insertData.thumbnail = thumbnail; - conflictUpdateData.thumbnail = thumbnail; - } - - // 创建新画布（使用指定 ID），并兼容同一用户并发创建同一画布的场景。 - const [newProject] = await db - .insert(canvasProject) - .values(insertData) - .onConflictDoUpdate({ - target: canvasProject.id, - set: conflictUpdateData, - where: eq(canvasProject.userId, session.user.id), - }) - .returning(); - - if (!newProject) { - return NextResponse.json( - { success: false, error: '画布ID已被占用' }, - { status: 409 } - ); - } - - return NextResponse.json({ success: true, data: newProject }); - } + return NextResponse.json({ success: true, data: saved }); + }); } catch (error) { - console.error('更新/创建画布失败:', error); - return NextResponse.json({ success: false, error: '更新画布失败' }, { status: 500 }); + console.error("更新/创建画布失败:", error); + return NextResponse.json( + { success: false, error: "更新画布失败" }, + { status: 500 }, + ); } } diff --git a/app/api/canvas/director/upload-model/route.ts b/app/api/canvas/director/upload-model/route.ts new file mode 100644 index 000000000..03600c800 --- /dev/null +++ b/app/api/canvas/director/upload-model/route.ts @@ -0,0 +1,45 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { uploadFile } from "@/lib/storage/service"; +import { + DIRECTOR_MAX_GLB_BYTES, + validateDirectorGlb, +} from "@/lib/video-canvas/director-glb"; +export const runtime = "nodejs"; +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const form = await request.formData(); + const file = form.get("file"); + if ( + !(file instanceof File) || + !/\.glb$/i.test(file.name) || + file.size > DIRECTOR_MAX_GLB_BYTES + ) + return NextResponse.json( + { error: "请上传 25 MiB 以内的 GLB 模型" }, + { status: 400 }, + ); + const buffer = await file.arrayBuffer(); + try { + validateDirectorGlb(buffer); + } catch (error) { + return NextResponse.json( + { error: error instanceof Error ? error.message : "无效 GLB" }, + { status: 400 }, + ); + } + const result = await uploadFile( + `director-${crypto.randomUUID()}.glb`, + buffer, + ); + return NextResponse.json({ success: true, data: result }); + } catch { + return NextResponse.json( + { error: "模型上传失败，请重试" }, + { status: 500 }, + ); + } +} diff --git a/app/api/canvas/prompt-tools/route.ts b/app/api/canvas/prompt-tools/route.ts index 1fb8be196..38d9476b4 100644 --- a/app/api/canvas/prompt-tools/route.ts +++ b/app/api/canvas/prompt-tools/route.ts @@ -1,4 +1,6 @@ import { generateText } from "ai"; +import { createLogger } from "@/lib/observability/logger"; +const logger = createLogger("canvas-prompt-tools"); import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; @@ -50,6 +52,8 @@ export async function POST(request: Request) { if (!text.trim()) throw new Error("empty model output"); return NextResponse.json({ success: true, data: { text: text.trim(), model: selected.id, modelName: selected.displayName || selected.modelId } }); } catch (error) { + const detail = error as { name?: string; statusCode?: number; cause?: { code?: string }; data?: { error?: { code?: string } } }; + logger.warn("Prompt tool request failed", { name: detail?.name, statusCode: detail?.statusCode, code: detail?.data?.error?.code || detail?.cause?.code }); return NextResponse.json({ success: false, error: promptToolErrorMessage(error) }, { status: 502 }); } finally { running.delete(session.user.id); diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 30931e39a..0b76b3e5e 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -1,4 +1,10 @@ "use client"; +import dynamic from "next/dynamic"; +const DirectorStudioDialog = dynamic( + () => import("@/components/canvas/director/DirectorStudioDialog").then(module => module.DirectorStudioDialog), + { ssr: false }, +); +import type { DirectorSnapshot } from "@/lib/video-canvas/director-scene"; import { filterCanvasModels, getCanvasVideoModes, getCanvasVideoParameters, type CanvasModelKind } from "@/lib/video-canvas/model-matching"; import { @@ -1084,7 +1090,7 @@ const LOCAL_EDIT_COLORS = [ "#8d6e63", ]; const VIDEO_CLIP_FRAME_COUNT = 9; -const VIDEO_CROP_INLINE_LABEL = "1024 x 576"; + const VIDEO_CROP_OUTPUT_LABEL = "1024 × 576"; const VIDEO_CROP_RECT = { x: 0.1, y: 0.1, width: 0.8, height: 0.8 }; const STORY_BEAT_GRID_FUNCTION_KEY = "story-beat-grid-4"; @@ -2257,6 +2263,13 @@ function getVideoNodeDurationSeconds(node: CloneNode | null | undefined) { return 5; } +function getVideoCropSelectionLabel(node: CloneNode) { + const rect = (node.draftValues?.cropRect || VIDEO_CROP_RECT) as GridCropRect; + const dimensions = getDraftString(node, "outputSizeLabel").match(/(\d+)\s*[×x]\s*(\d+)/); + if (!dimensions) return `裁剪区域 ${Math.round(rect.width * 100)}% × ${Math.round(rect.height * 100)}%`; + return `${Math.floor(Number(dimensions[1]) * rect.width / 2) * 2} × ${Math.floor(Number(dimensions[2]) * rect.height / 2) * 2}`; +} + function formatClipDurationLabel(seconds: number) { const normalized = Math.max(0.1, seconds); return `${normalized.toFixed(2)} s`; @@ -5004,17 +5017,17 @@ export function LiblibVideoCanvasReplica({ }, []); const sendDirectorSnapshotsToCanvas = useCallback( - (directorNode: CloneNode, snapshots: Array<{ id: string; title: string }>) => { + (directorNode: CloneNode, snapshots: DirectorSnapshot[]) => { if (snapshots.length === 0) return; const nextNodes = snapshots.map((snapshot, index) => { const created = createMediaResourceNode({ action: "DIRECTOR_CAMERA_SNAPSHOT", - height: DIRECTOR_CAMERA_SNAPSHOT_NODE_HEIGHT, + height: Math.round(350 * snapshot.height / snapshot.width), mediaType: "image", - mediaUrl: FIRST_FRAME_IMAGE_URL, + mediaUrl: snapshot.url, title: `${directorNode.title} ${snapshot.title}`, - width: DIRECTOR_CAMERA_SNAPSHOT_NODE_WIDTH, + width: 350, x: Math.round(directorNode.x + DIRECTOR_CAMERA_SNAPSHOT_NODE_OFFSET_X * (index + 1)), y: directorNode.y, }); @@ -6377,44 +6390,19 @@ export function LiblibVideoCanvasReplica({ selectedId = video.id; break; } - case "首帧生成视频": { - draftUpdates = { action: "VIDEO_GENERATE", params: { prompt: VIDEO_FIRST_FRAME_PROMPT } }; - connectToAnchor( - createMediaResourceNode({ - mediaType: "image", - mediaUrl: VIDEO_FIRST_FRAME_SOURCE_URL, - title: "首帧", - width: 420, - height: 280, - x: leftOfAnchor(420), - y: centerY(280), - }) - ); - selectedId = anchor.id; - break; - } + case "首帧生成视频": case "首尾帧生成视频": { - draftUpdates = { action: "VIDEO_GENERATE", params: { prompt: VIDEO_FIRST_LAST_PROMPT } }; - const first = createMediaResourceNode({ - mediaType: "image", - mediaUrl: VIDEO_FIRST_LAST_FIRST_URL, - title: "首帧", - width: 420, - height: 240, - x: leftOfAnchor(420), - y: Math.round(anchor.y + 18), - }); - const last = createMediaResourceNode({ - mediaType: "image", - mediaUrl: VIDEO_FIRST_LAST_LAST_URL, - title: "尾帧", - width: 420, - height: 240, - x: leftOfAnchor(420), - y: Math.round(anchor.y + 286), - }); - connectToAnchor(first); - connectToAnchor(last); + const firstOnly = actionLabel === "首帧生成视频"; + draftUpdates = { + action: "VIDEO_GENERATE", + params: { + ...(anchor.draftValues?.params as Record | undefined), + mode: firstOnly ? "first-frame" : "first-last-frame", + prompt: firstOnly ? VIDEO_FIRST_FRAME_PROMPT : VIDEO_FIRST_LAST_PROMPT, + }, + }; + setReferenceSelectionTargetId(anchor.id); + toast.info(firstOnly ? "请选择画布中的首帧图片" : "请依次选择首帧和尾帧图片"); selectedId = anchor.id; break; } @@ -8190,6 +8178,7 @@ export function LiblibVideoCanvasReplica({ mode: requestedMode, preferredModelRef, prompt: finalPrompt, + generateAudio: savedSettings.audio !== "关闭", ratio, resolution, }); @@ -10946,7 +10935,7 @@ export function LiblibVideoCanvasReplica({ <> setNodes(current => current.map(node => node.id === selectedVideoMediaNode.id ? {...node,draftValues:{...node.draftValues,cropRect:rect}} : node))} /> sendDirectorSnapshotsToCanvas(directorConsoleNode, snapshots)} - onTutorialActiveChange={setDirectorTutorialActive} + setNodes(current => current.map(node => node.id === directorConsoleNode.id ? {...node, draftValues: {...node.draftValues, directorScene: scene}} : node))} + onSend={snapshots => sendDirectorSnapshotsToCanvas(directorConsoleNode, snapshots)} onClose={() => setDirectorConsoleNodeId(null)} - tutorialActive={directorTutorialActive} /> ) : null} @@ -12986,7 +12974,7 @@ function VideoCropSelectionOverlay({anchor,node,onChange}:{anchor:PresetControlA const [drawing,setDrawing]=useState<{x:number;y:number;endX:number;endY:number}|null>(null); const rect=drawing ? {x:Math.min(drawing.x,drawing.endX),y:Math.min(drawing.y,drawing.endY),width:Math.abs(drawing.endX-drawing.x),height:Math.abs(drawing.endY-drawing.y)} : saved; const point=(event:ReactPointerEvent)=>({x:clampUnit((event.clientX-box.left)/box.width),y:clampUnit((event.clientY-box.top)/box.height)}); - return
{event.stopPropagation();event.currentTarget.setPointerCapture(event.pointerId);const p=point(event);setDrawing({...p,endX:p.x,endY:p.y});}} onPointerMove={event=>{if(!drawing)return;event.stopPropagation();const p=point(event);setDrawing({...drawing,endX:p.x,endY:p.y});}} onPointerUp={event=>{event.stopPropagation();if(drawing&&rect.width>.02&&rect.height>.02)onChange(rect);setDrawing(null);event.currentTarget.releasePointerCapture(event.pointerId);}} @@ -14799,8 +14787,8 @@ function VideoTargetModuleComposer({ const tabItems: Array<{ key: VideoComposerTabKey; label: string; enabled: boolean }> = [ { key: "text-to-video", label: "文生视频", enabled: referenceCount === 0 }, { key: "reference-images", label: "全能参考", enabled: referenceCount >= 1 }, - { key: "first-frame", label: "图生视频", enabled: referenceCount >= 1 }, - { key: "first-last-frame", label: "首尾帧", enabled: referenceCount >= 2 }, + { key: "first-frame", label: "图生视频", enabled: true }, + { key: "first-last-frame", label: "首尾帧", enabled: true }, { key: "image-reference", label: "图片参考", enabled: referenceCount >= 1 }, ]; const referenceThumbnails = imageSources @@ -14919,7 +14907,11 @@ function VideoTargetModuleComposer({ const activeModelButtonLabel = getVideoComposerModelShortLabel(getVideoComposerModelLabel(settings, modelOptions)); const activeModelButtonMinWidth = activeModelButtonLabel === "2.0 Fast" ? 115 : activeModelButtonLabel === "2.0 Mini" ? 113 : 91; - const canSubmit = Boolean( + const missingFrameMessage = activeMode === "first-frame" && referenceCount < 1 + ? "请选择首帧图片" + : activeMode === "first-last-frame" && referenceCount < 2 + ? "请选择首帧和尾帧图片" : ""; + const canSubmit = !missingFrameMessage && Boolean( prompt.trim() || referenceCount > 0 || magicMarks.length > 0 || @@ -15400,7 +15392,7 @@ function VideoTargetModuleComposer({ tab: activeTab, }) } - title={canSubmit ? visibleBillingDisplay.message : "请输入提示词、添加参考素材、标记元素或视频特效"} + title={missingFrameMessage || (canSubmit ? visibleBillingDisplay.message : "请输入提示词、添加参考素材、标记元素或视频特效")} type="button" >