import { expect, test } from "@playwright/test"; import { CANVAS_CAPABILITY_IDS } from "@/lib/canvas/capabilities/types"; import { CANVAS_CAPABILITY_REGISTRY } from "@/lib/canvas/capabilities/registry"; import { getCanvasAgentAccessManifest, isExplicitDestructiveIntent, summarizeAgentCapabilities, VIDEO_CANVAS_EXTRA_ACTIONS, VIDEO_CANVAS_INTERACTIVE_ACTIONS, } from "@/lib/ai/canvas-agent-runtime/capability-access"; import { buildVideoRuntimeTools, type VideoRuntimePorts, } from "@/lib/ai/canvas-agent-runtime/video-tools"; import { newAgentRunSnapshot } from "@/lib/ai/canvas-agent-runtime/protocol"; import { inferVideoTaskRequirements, videoAgentSettingsSchema, type VideoRuntimeNode, } from "@/lib/ai/canvas-agent-runtime/video-protocol"; import { defaultVideoCanvasFunctionConfig } from "@/lib/video-canvas/config"; import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing"; import { estimateContextTokens } from "@/lib/ai/canvas-agent-runtime/context"; import { finalImageResultNodeIds } from "@/lib/ai/canvas-agent-runtime/results"; test("full capability discovery leaves room for subsequent edits without losing backend limitations", () => { const manifest = getCanvasAgentAccessManifest(); const summaries = summarizeAgentCapabilities(manifest); expect(summaries.map((item) => item.id)).toEqual(manifest.map((item) => item.id)); for (const item of manifest.filter((entry) => entry.execution === "backend_required" && entry.authorized)) expect(summaries.find((entry) => entry.id === item.id)?.note).toBe(item.note); expect( estimateContextTokens([{ role: "user", content: JSON.stringify(summaries) }]), ).toBeLessThan(12_000); }); test("image conversation memory excludes text, slots and 3D results from image references", () => { const state = newAgentRunSnapshot("添加文本并生成图片", []); state.resultNodeIds = ["text", "slot", "model", "image"]; expect( finalImageResultNodeIds(state, [ { id: "text", type: "text" }, { id: "slot", type: "agent-slot" }, { id: "model", type: "model3d" }, { id: "image", type: "image" }, ]), ).toEqual(["image"]); }); function videoHarness(objective = "使用参考图生成一条视频") { const state = newAgentRunSnapshot(objective, [{ role: "user", content: objective }]); state.workspace = "video"; state.videoSettings = videoAgentSettingsSchema.parse({}); state.videoRequirements = inferVideoTaskRequirements(objective); const nodes: VideoRuntimeNode[] = [ { id: "image-a", kind: "media", title: "参考图", inputType: "image", x: 10, y: 20, width: 320, height: 220, accent: "#aaa", draftValues: { mediaType: "image", mediaUrl: "https://example.test/a.png" }, }, ]; const generated: unknown[] = [], events: Array<{ type: string; data: unknown }> = [], edits: unknown[] = []; const ports: VideoRuntimePorts = { read: async () => ({ nodes, edges: [] }), append: async (node) => { nodes.push(node); return node.id; }, generateImage: async (input) => { generated.push(input); return { url: "https://example.test/out.png", model: "configured-image" }; }, generateVideo: async (input) => { generated.push(input); return { url: "https://example.test/out.mp4", model: "configured-video" }; }, analyze: async () => ({ raw: "真实分析", analysisRows: [{ sceneDescription: "实际场景" }] }), mutate: async (_id, input) => { edits.push(input); return { status: "succeeded", updatedNodeIds: input.nodeIds }; }, processImage: async (input) => { generated.push(input); return [{ url: "https://example.test/processed.png" }]; }, emit: (type, data) => events.push({ type, data }), }; const tools = buildVideoRuntimeTools({ runId: "test-run", projectId: "11111111-1111-4111-8111-111111111111", selectedNodeIds: ["image-a"], snapshot: state, ports, }); return { state, nodes, generated, events, edits, tools, ports }; } test("every registered image capability is authorized to the Agent including local and 3D operations", async () => { expect(CANVAS_CAPABILITY_REGISTRY.every((item) => item.agent.exposed)).toBe(true); expect(CANVAS_CAPABILITY_IDS).toHaveLength(25); }); test("access manifest covers all configured video functions and node/UI actions without granting fake backends", () => { const manifest = getCanvasAgentAccessManifest(); expect(new Set(manifest.map((item) => item.id)).size).toBe(manifest.length); expect(manifest.filter(item => item.workspace === "canvas").every(item => item.authorized)).toBe(true); expect(manifest.filter(item => item.workspace === "video").every(item => !item.authorized)).toBe(true); for (const key of Object.keys(defaultVideoCanvasFunctionConfig.functions)) expect(manifest.some((item) => item.id === `video.function.${key}`)).toBe(true); for (const [key] of VIDEO_CANVAS_EXTRA_ACTIONS) expect(manifest.some((item) => item.id === `video.node.${key}`)).toBe(true); for (const [key] of VIDEO_CANVAS_INTERACTIVE_ACTIONS) expect(manifest.some((item) => item.id === `video.ui.${key}`)).toBe(true); expect(manifest.find((item) => item.id === "video.function.audio-to-video")).toMatchObject({ authorized: false, execution: "backend_required", }); expect(manifest.find((item) => item.id === "video.ui.subtitle-erase")?.note).toContain("模拟"); for (const key of ["video-hd", "subtitle-erase", "voice-split", "background-split"]) expect(manifest.find((item) => item.id === `video.ui.${key}`)?.execution).toBe( "backend_required", ); }); test("explicit destructive intent cannot be forged by negative or unrelated user requests", () => { expect(isExplicitDestructiveIntent("删除选中图片")).toBe(true); expect(isExplicitDestructiveIntent("不要删除选中图片")).toBe(false); expect(isExplicitDestructiveIntent("检查画布功能列表")).toBe(false); }); test("image routing now admits formerly excluded utilities, 3D and capability discovery", () => { for (const text of [ "下载选中的图片", "给图片重命名", "生成3D模型", "有哪些功能", "打开排版替换", "删除选中的图片", ]) expect( shouldUseCanvasAgentRuntime({ composerMode: "agent", contextPayload: { source: "canvas", projectId: "p" }, message: { parts: [{ type: "text", text }] }, }), ).toBe(true); }); test("configured video presets use actual references, prompt and configured feature billing identity", async () => { const h = videoHarness("生成角色三视图"); const result = await h.tools.executable.runVideoCanvasFunction.execute( { functionKey: "character-three-view", nodeIds: ["image-a"], prompt: "保留产品结构" }, "preset", ); expect(h.generated[0]).toMatchObject({ references: ["https://example.test/a.png"], functionKey: "character-three-view", aspectRatio: "16:9", }); expect(String((h.generated[0] as { prompt: string }).prompt)).toContain("保留产品结构"); expect(result.createdNodeIds).toHaveLength(1); expect(h.nodes[0].id).toBe("image-a"); }); test("video remove-background and grid split use actual processor results", async () => { for (const functionKey of ["remove-bg", "grid-split"]) { const h = videoHarness("给图片抠图"); const output = await h.tools.executable.runVideoCanvasFunction.execute( { functionKey, nodeIds: ["image-a"], args: { rows: 3, columns: 3 } }, functionKey, ); expect(output.createdNodeIds).toHaveLength(1); expect(h.generated).toHaveLength(1); expect(h.nodes.at(-1)?.draftValues?.mediaUrl).toBe("https://example.test/processed.png"); } }); test("a disabled backend remains disabled even though the Agent has feature authorization", async () => { const h = videoHarness(); h.ports.featureConfig = { ...defaultVideoCanvasFunctionConfig, enabled: false }; const tools = buildVideoRuntimeTools({ runId: "r", selectedNodeIds: ["image-a"], snapshot: h.state, ports: h.ports, }); await expect( tools.executable.runVideoCanvasFunction.execute( { functionKey: "remove-bg", nodeIds: ["image-a"] }, "off", ), ).rejects.toThrow("关闭"); expect(h.generated).toEqual([]); }); test("audio-driven generation only opens an honest configuration entry and never fabricates a video", async () => { const h = videoHarness("音频生成视频"); const output = await h.tools.executable.runVideoCanvasFunction.execute( { functionKey: "audio-to-video", nodeIds: ["image-a"] }, "audio", ); expect(output).toMatchObject({ mode: "interactive", backendRequired: true }); expect(h.generated).toEqual([]); expect(h.events[0].type).toBe("data-canvasAgentUiCommand"); }); test("configured video generation cannot bypass the per-task paid-video budget", async () => { const h = videoHarness(); h.state.calls = [ { id: "one", name: "runVideoCanvasFunction", input: { functionKey: "first-frame-video" }, status: "succeeded", }, { id: "two", name: "runVideoCanvasFunction", input: { functionKey: "first-frame-video" }, status: "started", }, ]; await expect( h.tools.executable.runVideoCanvasFunction.execute( { functionKey: "first-frame-video", nodeIds: ["image-a"] }, "two", ), ).rejects.toThrow("未再发出付费请求"); expect(h.generated).toEqual([]); }); test("video node editing is authorized but arbitrary or unselected deletions are rejected", async () => { const h = videoHarness("不要删除图片,只修改名称"); await h.tools.executable.editVideoCanvas.execute( { action: "rename", nodeIds: ["image-a"], text: "商品参考" }, "rename", ); await expect( h.tools.executable.editVideoCanvas.execute( { action: "delete", nodeIds: ["image-a"] }, "delete", ), ).rejects.toThrow("明确要求"); await expect( h.tools.executable.editVideoCanvas.execute({ action: "move", nodeIds: ["foreign"] }, "foreign"), ).rejects.toThrow("不属于"); expect(h.edits).toHaveLength(1); }); test("video parsing and canvas editing do not accidentally authorize paid video generation", () => { expect(inferVideoTaskRequirements("解析这个视频").expectsVideo).toBe(false); expect(inferVideoTaskRequirements("删除这个视频节点").expectsVideo).toBe(false); expect(inferVideoTaskRequirements("生成一条5秒视频").expectsVideo).toBe(true); }); test("real media tools require an owned video, retain the source, and write real returned URLs", async () => { const h = videoHarness("提取视频首帧"); h.nodes.push({ ...h.nodes[0], id: "video-a", inputType: "video", draftValues: { mediaType: "video", mediaUrl: "https://example.test/a.mp4" }, }); h.ports.processMedia = async (input) => { h.generated.push(input); return { url: "https://cdn.example.test/frame.png", type: "image", duration: 5 }; }; const result = await h.tools.executable.processVideoMedia.execute( { operation: "extract-frame", nodeId: "video-a", frame: "first" }, "frame", ); expect(result.createdNodeIds).toHaveLength(1); expect(h.generated[0]).toMatchObject({ url: "https://example.test/a.mp4", operation: "extract-frame", }); expect(h.nodes.at(-1)?.draftValues?.mediaUrl).toBe("https://cdn.example.test/frame.png"); expect(h.nodes.find((node) => node.id === "video-a")?.draftValues?.mediaUrl).toBe( "https://example.test/a.mp4", ); await expect( h.tools.executable.processVideoMedia.execute( { operation: "extract-frame", nodeId: "foreign" }, "foreign", ), ).rejects.toThrow("不属于"); }); test("custom configured video functions cannot bypass the authorized paid-video count", async () => { const h = videoHarness(); h.ports.featureConfig = { ...defaultVideoCanvasFunctionConfig, functions: { ...defaultVideoCanvasFunctionConfig.functions, customVideo: { ...defaultVideoCanvasFunctionConfig.functions["first-frame-video"], key: "customVideo", }, }, }; h.state.calls = [ { id: "one", name: "runVideoCanvasFunction", input: { functionKey: "customVideo" }, status: "succeeded", }, { id: "two", name: "runVideoCanvasFunction", input: { functionKey: "customVideo" }, status: "started", }, ]; const tools = buildVideoRuntimeTools({ runId: "custom", selectedNodeIds: ["image-a"], snapshot: h.state, ports: h.ports, }); await expect( tools.executable.runVideoCanvasFunction.execute( { functionKey: "customVideo", nodeIds: ["image-a"] }, "two", ), ).rejects.toThrow("未再发出付费请求"); expect(h.generated).toEqual([]); });