import { expect, test } from "@playwright/test"; import type { ModelMessage } from "ai"; import { newAgentRunSnapshot, type AgentCall, type AgentRunSnapshot, } from "@/lib/ai/canvas-agent-runtime/protocol"; import { runCanvasAgentLoop } from "@/lib/ai/canvas-agent-runtime/loop"; import { buildVideoRuntimeTools, type VideoRuntimePorts, } from "@/lib/ai/canvas-agent-runtime/video-tools"; import { inferVideoTaskRequirements, resolveVideoMode, videoAgentSettingsSchema, serializeVideoRuntimeNode, restoreVideoRuntimeNodes, mergeVideoAgentResults, type VideoRuntimeNode, } from "@/lib/ai/canvas-agent-runtime/video-protocol"; import { verifyVideoAgentResults } from "@/lib/ai/canvas-agent-runtime/video-review"; function harness(objective = "先写分镜,再生成一条5秒视频") { const state = newAgentRunSnapshot(objective, [{ role: "user", content: objective }]); state.workspace = "video"; state.videoSettings = videoAgentSettingsSchema.parse({}); state.videoRequirements = inferVideoTaskRequirements(objective); const nodes: VideoRuntimeNode[] = [ { id: "original", kind: "media", title: "参考图", x: 20, y: 40, width: 320, height: 200, inputType: "image", accent: "#aaa", draftValues: { mediaType: "image", mediaUrl: "https://example.test/original.png" }, }, ]; const calls: string[] = []; const ports: VideoRuntimePorts = { read: async () => ({ nodes, edges: [] }), append: async (node) => { if (!nodes.some((old) => old.id === node.id)) nodes.push(node); return node.id; }, analyze: async (input) => ({ summary: `实际素材分析:${input.prompt}`, analysisRows: [] }), generateImage: async ({ callId }) => { calls.push(`image:${callId}`); return { url: "https://example.test/keyframe.png", model: "existing-image-model" }; }, generateVideo: async ({ callId }) => { calls.push(`video:${callId}`); return { url: "https://example.test/result.mp4", model: "existing-video-model" }; }, }; const tools = buildVideoRuntimeTools({ runId: "run", selectedNodeIds: ["original"], snapshot: state, ports, }); return { state, nodes, calls, ports, tools }; } const row = { sceneDescription: "展示商品", imagePrompt: "保留商品结构", motionPrompt: "缓慢推镜", duration: 5, }; const call = (id: string, name: string, input: Record = {}): AgentCall => ({ id, name, input, }); function step(calls: AgentCall[] = []) { return { text: "已完成", calls, messages: [ { role: "assistant", content: calls.length ? calls.map((item) => ({ type: "tool-call", toolCallId: item.id, toolName: item.name, input: item.input, })) : "已完成", }, ] as ModelMessage[], inputTokens: 30, outputTokens: 20, }; } test("video settings validate modes and preserve first-last reference order", () => { expect(resolveVideoMode("auto", 0)).toBe("text-to-video"); expect(resolveVideoMode("auto", 1)).toBe("first-frame"); expect(resolveVideoMode("auto", 2)).toBe("first-last-frame"); expect(resolveVideoMode("auto", 3)).toBe("reference-images"); expect(() => resolveVideoMode("first-last-frame", 1)).toThrow("不会自动丢弃"); expect(() => resolveVideoMode("text-to-video", 1)).toThrow(); expect(() => resolveVideoMode("reference-images", 5)).toThrow(); expect(videoAgentSettingsSchema.safeParse({ duration: 999 }).success).toBe(false); }); test("requirements distinguish scripts from actual video and explicit quantities", () => { expect(inferVideoTaskRequirements("只要视频分镜脚本,不要生成视频").expectsVideo).toBe(false); expect(inferVideoTaskRequirements("先写脚本,再生成三条9:16视频,每条8秒")).toMatchObject({ expectsVideo: true, expectsScript: true, duration: 8, ratio: "9:16", }); expect(inferVideoTaskRequirements("生成三条视频").expectedVideoCount).toBe(3); }); test("native video node serialization round-trips without accepting image-canvas nodes", () => { const h = harness(); expect( restoreVideoRuntimeNodes({ nodes: [ serializeVideoRuntimeNode(h.nodes[0]), { id: "x", type: "image", data: { src: "url" } }, ], }), ).toEqual(h.nodes); expect( restoreVideoRuntimeNodes({ nodes: [{ ...serializeVideoRuntimeNode(h.nodes[0]), id: "spoof" }], }), ).toEqual([]); expect( mergeVideoAgentResults( [{ id: "edited", x: 123 }], [ { id: "edited", x: 0 }, { id: "new", x: 10 }, ], ), ).toEqual([ { id: "edited", x: 123 }, { id: "new", x: 10 }, ]); }); test("scripts, keyframes and video execute in order through the real ZCode scheduling loop", async () => { const h = harness(); const sequence = [ step([ call("plan", "planCanvasTask", { steps: ["脚本", "关键帧", "视频"], expectedVideoCount: 1 }), ]), step([call("read", "readCanvas")]), step([ call("script", "writeVideoScript", { title: "商品分镜", rows: [row], sourceNodeIds: ["original"], }), ]), step([ call("keyframe", "generateVideoKeyframe", { prompt: row.imagePrompt, referenceNodeIds: ["original"], }), ]), ]; let modelStep = 0; const checkpoints: AgentRunSnapshot[] = []; const result = await runCanvasAgentLoop(h.state, { model: async () => { modelStep++; if (sequence.length) return sequence.shift()!; if (modelStep === 5) return step([ call("video", "generateVideoShot", { prompt: row.motionPrompt, referenceNodeIds: [h.nodes.find((node) => node.title === "分镜关键帧")!.id], }), ]); return step(); }, tools: h.tools.executable, checkpoint: async (state) => { checkpoints.push(structuredClone(state)); }, consumeInputs: async () => ({ cancelled: false }), isCancelled: async () => false, summarize: async () => "视频任务摘要", verify: (state) => verifyVideoAgentResults({ snapshot: state, nodes: h.nodes, inspect: async () => ({ passed: true, issues: [], duration: 5, width: 1280, height: 720 }), }), }); expect(result).toBe("completed"); expect(h.calls).toEqual(["image:keyframe", "video:video"]); expect(h.nodes[0].id).toBe("original"); expect(h.state.resultNodeIds).toHaveLength(3); expect( checkpoints.some((state) => state.calls.some((item) => item.id === "video" && item.status === "started"), ), ).toBe(true); }); test("unowned or nonexistent reference IDs never reach a paid provider", async () => { const h = harness(); await expect( h.tools.executable.generateVideoShot.execute( { prompt: "视频", referenceNodeIds: ["other-project"] }, "x", ), ).rejects.toThrow("素材节点不可用"); expect(h.calls).toEqual([]); }); test("a script-only request cannot accidentally start a billable video", async () => { const h = harness("只要分镜脚本,不要生成视频"); await expect( h.tools.executable.generateVideoShot.execute({ prompt: "视频", referenceNodeIds: [] }, "x"), ).rejects.toThrow("尚未授权"); expect(h.calls).toEqual([]); }); test("first-last mode refuses missing frames instead of falling back to another mode", async () => { const h = harness(); h.state.videoSettings!.mode = "first-last-frame"; await expect( h.tools.executable.generateVideoShot.execute( { prompt: "视频", referenceNodeIds: ["original"] }, "x", ), ).rejects.toThrow("不匹配"); expect(h.calls).toEqual([]); }); test("script results use native text resources and display every storyboard row", async () => { const h = harness("只要分镜脚本"); const result = await h.tools.executable.writeVideoScript.execute( { title: "分镜", rows: [row, { ...row, sceneDescription: "第二镜头" }], sourceNodeIds: [] }, "script", ); const node = h.nodes.find((item) => item.id === (result.createdNodeIds as string[])[0])!; expect(node.draftValues).toMatchObject({ action: "TEXT_RESOURCE", nodeType: "text", scriptRole: "resource", }); expect(node.draftValues?.content).toHaveLength(1); expect((node.draftValues?.content as string[])[0]).toContain("02. 第二镜头"); expect(node.draftValues?.rows).toHaveLength(2); }); test("scripts and keyframes cannot satisfy a request for actual videos", async () => { const h = harness("生成两条视频"); const script = await h.tools.executable.writeVideoScript.execute( { title: "分镜", rows: [row], sourceNodeIds: [] }, "script", ); h.state.resultNodeIds = script.createdNodeIds as string[]; const verdict = await verifyVideoAgentResults({ snapshot: h.state, nodes: h.nodes, inspect: async () => ({ passed: true, issues: [], duration: 5, width: 1280, height: 720 }), }); expect(verdict.passed).toBe(false); expect(verdict.issues.join()).toContain("当前仅完成 0 条"); expect(verdict.retryable).toBe(false); }); test("visual review failures retain results without automatic paid regeneration", async () => { const h = harness("生成一条视频"); const output = await h.tools.executable.generateVideoShot.execute( { prompt: "视频", referenceNodeIds: ["original"] }, "video", ); h.state.resultNodeIds = output.createdNodeIds as string[]; const result = await verifyVideoAgentResults({ snapshot: h.state, nodes: h.nodes, inspect: async () => ({ passed: false, issues: ["商品颜色不一致"] }), }); expect(result).toMatchObject({ passed: false, retryable: false }); expect(h.calls).toHaveLength(1); const unavailable = await verifyVideoAgentResults({ snapshot: h.state, nodes: h.nodes, inspect: async () => { throw new Error("cannot decode"); }, }); expect(unavailable.passed).toBe(false); expect(unavailable.issues.join()).toContain("无法读取"); }); test("video material analysis receives real URLs and excludes base64 from persisted tool history", async () => { const h = harness("分析选中图片"); let actual: unknown; h.ports.analyze = async (input) => { actual = input; return { summary: "图中有商品" }; }; const output = await h.tools.executable.analyzeVideoMaterials.execute( { nodeIds: ["original"], question: "分析商品" }, "a", ); expect(actual).toMatchObject({ imageUrls: ["https://example.test/original.png"], videoUrls: [] }); expect(output.summary).toBe("图中有商品"); }); test("verification measures actual duration and dimensions instead of trusting request metadata", async () => { const h = harness("生成一条5秒16:9视频"); const output = await h.tools.executable.generateVideoShot.execute( { prompt: "视频", referenceNodeIds: ["original"] }, "video", ); h.state.resultNodeIds = output.createdNodeIds as string[]; const result = await verifyVideoAgentResults({ snapshot: h.state, nodes: h.nodes, inspect: async () => ({ passed: true, issues: [], duration: 9, width: 720, height: 1280 }), }); expect(result.passed).toBe(false); expect(result.issues.join()).toContain("实际时长"); expect(result.issues.join()).toContain("实际尺寸"); expect(result.retryable).toBe(false); }); test("stopping during a video call retains the paid result and prevents the next shot", async () => { const h = harness("生成两条视频"); let cancelled = false; h.ports.generateVideo = async () => { cancelled = true; return { url: "https://example.test/paid.mp4", model: "configured" }; }; const sequence = [ step([call("plan", "planCanvasTask", { steps: ["两个镜头"], expectedVideoCount: 2 })]), step([ call("one", "generateVideoShot", { prompt: "第一条", referenceNodeIds: ["original"] }), call("two", "generateVideoShot", { prompt: "第二条", referenceNodeIds: ["original"] }), ]), ]; const result = await runCanvasAgentLoop(h.state, { model: async () => sequence.shift()!, tools: h.tools.executable, checkpoint: async () => {}, consumeInputs: async () => ({ cancelled }), isCancelled: async () => cancelled, summarize: async () => "摘要", verify: async () => ({ passed: true, issues: [] }), }); expect(result).toBe("cancelled"); expect(h.state.resultNodeIds).toHaveLength(1); expect(h.state.calls.find((item) => item.id === "one")?.status).toBe("succeeded"); expect(h.state.calls.find((item) => item.id === "two")?.status).toBe("failed"); }); test("an extra model tool call cannot exceed the user's authorized video count", async () => { const h = harness("生成一条视频"); h.state.calls = [ { id: "first", name: "generateVideoShot", input: {}, status: "succeeded", output: { createdNodeIds: ["first-result"] }, }, { id: "second", name: "generateVideoShot", input: {}, status: "started" }, ]; await expect( h.tools.executable.generateVideoShot.execute( { prompt: "额外视频", referenceNodeIds: ["original"] }, "second", ), ).rejects.toThrow("未再发出付费请求"); expect(h.calls).toEqual([]); });