ai-chatbot/tests/routes/agent-capability-access.test.ts

import { expect, test } from "@playwright/test";
import { CANVAS_CAPABILITY_IDS } from "@/lib/canvas/capabilities/types";
import { CANVAS_CAPABILITY_REGISTRY } from "@/lib/canvas/capabilities/registry";
import {
  getCanvasAgentAccessManifest,
  isExplicitDestructiveIntent,
  summarizeAgentCapabilities,
  VIDEO_CANVAS_EXTRA_ACTIONS,
  VIDEO_CANVAS_INTERACTIVE_ACTIONS,
} from "@/lib/ai/canvas-agent-runtime/capability-access";
import {
  buildVideoRuntimeTools,
  type VideoRuntimePorts,
} from "@/lib/ai/canvas-agent-runtime/video-tools";
import { newAgentRunSnapshot } from "@/lib/ai/canvas-agent-runtime/protocol";
import {
  inferVideoTaskRequirements,
  videoAgentSettingsSchema,
  type VideoRuntimeNode,
} from "@/lib/ai/canvas-agent-runtime/video-protocol";
import { defaultVideoCanvasFunctionConfig } from "@/lib/video-canvas/config";
import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing";
import { estimateContextTokens } from "@/lib/ai/canvas-agent-runtime/context";
import { finalImageResultNodeIds } from "@/lib/ai/canvas-agent-runtime/results";

test("full capability discovery leaves room for subsequent edits without losing backend limitations", () => {
  const manifest = getCanvasAgentAccessManifest();
  const summaries = summarizeAgentCapabilities(manifest);
  expect(summaries.map((item) => item.id)).toEqual(manifest.map((item) => item.id));
  for (const item of manifest.filter((entry) => entry.execution === "backend_required" && entry.authorized))
    expect(summaries.find((entry) => entry.id === item.id)?.note).toBe(item.note);
  expect(
    estimateContextTokens([{ role: "user", content: JSON.stringify(summaries) }]),
  ).toBeLessThan(12_000);
});

test("image conversation memory excludes text, slots and 3D results from image references", () => {
  const state = newAgentRunSnapshot("添加文本并生成图片", []);
  state.resultNodeIds = ["text", "slot", "model", "image"];
  expect(
    finalImageResultNodeIds(state, [
      { id: "text", type: "text" },
      { id: "slot", type: "agent-slot" },
      { id: "model", type: "model3d" },
      { id: "image", type: "image" },
    ]),
  ).toEqual(["image"]);
});

function videoHarness(objective = "使用参考图生成一条视频") {
  const state = newAgentRunSnapshot(objective, [{ role: "user", content: objective }]);
  state.workspace = "video";
  state.videoSettings = videoAgentSettingsSchema.parse({});
  state.videoRequirements = inferVideoTaskRequirements(objective);
  const nodes: VideoRuntimeNode[] = [
    {
      id: "image-a",
      kind: "media",
      title: "参考图",
      inputType: "image",
      x: 10,
      y: 20,
      width: 320,
      height: 220,
      accent: "#aaa",
      draftValues: { mediaType: "image", mediaUrl: "https://example.test/a.png" },
    },
  ];
  const generated: unknown[] = [],
    events: Array<{ type: string; data: unknown }> = [],
    edits: unknown[] = [];
  const ports: VideoRuntimePorts = {
    read: async () => ({ nodes, edges: [] }),
    append: async (node) => {
      nodes.push(node);
      return node.id;
    },
    generateImage: async (input) => {
      generated.push(input);
      return { url: "https://example.test/out.png", model: "configured-image" };
    },
    generateVideo: async (input) => {
      generated.push(input);
      return { url: "https://example.test/out.mp4", model: "configured-video" };
    },
    analyze: async () => ({ raw: "真实分析", analysisRows: [{ sceneDescription: "实际场景" }] }),
    mutate: async (_id, input) => {
      edits.push(input);
      return { status: "succeeded", updatedNodeIds: input.nodeIds };
    },
    processImage: async (input) => {
      generated.push(input);
      return [{ url: "https://example.test/processed.png" }];
    },
    emit: (type, data) => events.push({ type, data }),
  };
  const tools = buildVideoRuntimeTools({
    runId: "test-run",
    projectId: "11111111-1111-4111-8111-111111111111",
    selectedNodeIds: ["image-a"],
    snapshot: state,
    ports,
  });
  return { state, nodes, generated, events, edits, tools, ports };
}

test("every registered image capability is authorized to the Agent including local and 3D operations", async () => {
  expect(CANVAS_CAPABILITY_REGISTRY.every((item) => item.agent.exposed)).toBe(true);
  expect(CANVAS_CAPABILITY_IDS).toHaveLength(25);
});
test("access manifest covers all configured video functions and node/UI actions without granting fake backends", () => {
  const manifest = getCanvasAgentAccessManifest();
  expect(new Set(manifest.map((item) => item.id)).size).toBe(manifest.length);
  expect(manifest.filter(item => item.workspace === "canvas").every(item => item.authorized)).toBe(true);
  expect(manifest.filter(item => item.workspace === "video").every(item => !item.authorized)).toBe(true);
  for (const key of Object.keys(defaultVideoCanvasFunctionConfig.functions))
    expect(manifest.some((item) => item.id === `video.function.${key}`)).toBe(true);
  for (const [key] of VIDEO_CANVAS_EXTRA_ACTIONS)
    expect(manifest.some((item) => item.id === `video.node.${key}`)).toBe(true);
  for (const [key] of VIDEO_CANVAS_INTERACTIVE_ACTIONS)
    expect(manifest.some((item) => item.id === `video.ui.${key}`)).toBe(true);
  expect(manifest.find((item) => item.id === "video.function.audio-to-video")).toMatchObject({
    authorized: false,
    execution: "backend_required",
  });
  expect(manifest.find((item) => item.id === "video.ui.subtitle-erase")?.note).toContain("模拟");
  for (const key of ["video-hd", "subtitle-erase", "voice-split", "background-split"])
    expect(manifest.find((item) => item.id === `video.ui.${key}`)?.execution).toBe(
      "backend_required",
    );
});
test("explicit destructive intent cannot be forged by negative or unrelated user requests", () => {
  expect(isExplicitDestructiveIntent("删除选中图片")).toBe(true);
  expect(isExplicitDestructiveIntent("不要删除选中图片")).toBe(false);
  expect(isExplicitDestructiveIntent("检查画布功能列表")).toBe(false);
});
test("image routing now admits formerly excluded utilities, 3D and capability discovery", () => {
  for (const text of [
    "下载选中的图片",
    "给图片重命名",
    "生成3D模型",
    "有哪些功能",
    "打开排版替换",
    "删除选中的图片",
  ])
    expect(
      shouldUseCanvasAgentRuntime({
        composerMode: "agent",
        contextPayload: { source: "canvas", projectId: "p" },
        message: { parts: [{ type: "text", text }] },
      }),
    ).toBe(true);
});
test("configured video presets use actual references, prompt and configured feature billing identity", async () => {
  const h = videoHarness("生成角色三视图");
  const result = await h.tools.executable.runVideoCanvasFunction.execute(
    { functionKey: "character-three-view", nodeIds: ["image-a"], prompt: "保留产品结构" },
    "preset",
  );
  expect(h.generated[0]).toMatchObject({
    references: ["https://example.test/a.png"],
    functionKey: "character-three-view",
    aspectRatio: "16:9",
  });
  expect(String((h.generated[0] as { prompt: string }).prompt)).toContain("保留产品结构");
  expect(result.createdNodeIds).toHaveLength(1);
  expect(h.nodes[0].id).toBe("image-a");
});
test("video remove-background and grid split use actual processor results", async () => {
  for (const functionKey of ["remove-bg", "grid-split"]) {
    const h = videoHarness("给图片抠图");
    const output = await h.tools.executable.runVideoCanvasFunction.execute(
      { functionKey, nodeIds: ["image-a"], args: { rows: 3, columns: 3 } },
      functionKey,
    );
    expect(output.createdNodeIds).toHaveLength(1);
    expect(h.generated).toHaveLength(1);
    expect(h.nodes.at(-1)?.draftValues?.mediaUrl).toBe("https://example.test/processed.png");
  }
});
test("a disabled backend remains disabled even though the Agent has feature authorization", async () => {
  const h = videoHarness();
  h.ports.featureConfig = { ...defaultVideoCanvasFunctionConfig, enabled: false };
  const tools = buildVideoRuntimeTools({
    runId: "r",
    selectedNodeIds: ["image-a"],
    snapshot: h.state,
    ports: h.ports,
  });
  await expect(
    tools.executable.runVideoCanvasFunction.execute(
      { functionKey: "remove-bg", nodeIds: ["image-a"] },
      "off",
    ),
  ).rejects.toThrow("关闭");
  expect(h.generated).toEqual([]);
});
test("audio-driven generation only opens an honest configuration entry and never fabricates a video", async () => {
  const h = videoHarness("音频生成视频");
  const output = await h.tools.executable.runVideoCanvasFunction.execute(
    { functionKey: "audio-to-video", nodeIds: ["image-a"] },
    "audio",
  );
  expect(output).toMatchObject({ mode: "interactive", backendRequired: true });
  expect(h.generated).toEqual([]);
  expect(h.events[0].type).toBe("data-canvasAgentUiCommand");
});
test("configured video generation cannot bypass the per-task paid-video budget", async () => {
  const h = videoHarness();
  h.state.calls = [
    {
      id: "one",
      name: "runVideoCanvasFunction",
      input: { functionKey: "first-frame-video" },
      status: "succeeded",
    },
    {
      id: "two",
      name: "runVideoCanvasFunction",
      input: { functionKey: "first-frame-video" },
      status: "started",
    },
  ];
  await expect(
    h.tools.executable.runVideoCanvasFunction.execute(
      { functionKey: "first-frame-video", nodeIds: ["image-a"] },
      "two",
    ),
  ).rejects.toThrow("未再发出付费请求");
  expect(h.generated).toEqual([]);
});
test("video node editing is authorized but arbitrary or unselected deletions are rejected", async () => {
  const h = videoHarness("不要删除图片，只修改名称");
  await h.tools.executable.editVideoCanvas.execute(
    { action: "rename", nodeIds: ["image-a"], text: "商品参考" },
    "rename",
  );
  await expect(
    h.tools.executable.editVideoCanvas.execute(
      { action: "delete", nodeIds: ["image-a"] },
      "delete",
    ),
  ).rejects.toThrow("明确要求");
  await expect(
    h.tools.executable.editVideoCanvas.execute({ action: "move", nodeIds: ["foreign"] }, "foreign"),
  ).rejects.toThrow("不属于");
  expect(h.edits).toHaveLength(1);
});
test("video parsing and canvas editing do not accidentally authorize paid video generation", () => {
  expect(inferVideoTaskRequirements("解析这个视频").expectsVideo).toBe(false);
  expect(inferVideoTaskRequirements("删除这个视频节点").expectsVideo).toBe(false);
  expect(inferVideoTaskRequirements("生成一条5秒视频").expectsVideo).toBe(true);
});

test("real media tools require an owned video, retain the source, and write real returned URLs", async () => {
  const h = videoHarness("提取视频首帧");
  h.nodes.push({
    ...h.nodes[0],
    id: "video-a",
    inputType: "video",
    draftValues: { mediaType: "video", mediaUrl: "https://example.test/a.mp4" },
  });
  h.ports.processMedia = async (input) => {
    h.generated.push(input);
    return { url: "https://cdn.example.test/frame.png", type: "image", duration: 5 };
  };
  const result = await h.tools.executable.processVideoMedia.execute(
    { operation: "extract-frame", nodeId: "video-a", frame: "first" },
    "frame",
  );
  expect(result.createdNodeIds).toHaveLength(1);
  expect(h.generated[0]).toMatchObject({
    url: "https://example.test/a.mp4",
    operation: "extract-frame",
  });
  expect(h.nodes.at(-1)?.draftValues?.mediaUrl).toBe("https://cdn.example.test/frame.png");
  expect(h.nodes.find((node) => node.id === "video-a")?.draftValues?.mediaUrl).toBe(
    "https://example.test/a.mp4",
  );
  await expect(
    h.tools.executable.processVideoMedia.execute(
      { operation: "extract-frame", nodeId: "foreign" },
      "foreign",
    ),
  ).rejects.toThrow("不属于");
});

test("custom configured video functions cannot bypass the authorized paid-video count", async () => {
  const h = videoHarness();
  h.ports.featureConfig = {
    ...defaultVideoCanvasFunctionConfig,
    functions: {
      ...defaultVideoCanvasFunctionConfig.functions,
      customVideo: {
        ...defaultVideoCanvasFunctionConfig.functions["first-frame-video"],
        key: "customVideo",
      },
    },
  };
  h.state.calls = [
    {
      id: "one",
      name: "runVideoCanvasFunction",
      input: { functionKey: "customVideo" },
      status: "succeeded",
    },
    {
      id: "two",
      name: "runVideoCanvasFunction",
      input: { functionKey: "customVideo" },
      status: "started",
    },
  ];
  const tools = buildVideoRuntimeTools({
    runId: "custom",
    selectedNodeIds: ["image-a"],
    snapshot: h.state,
    ports: h.ports,
  });
  await expect(
    tools.executable.runVideoCanvasFunction.execute(
      { functionKey: "customVideo", nodeIds: ["image-a"] },
      "two",
    ),
  ).rejects.toThrow("未再发出付费请求");
  expect(h.generated).toEqual([]);
});