353 lines
15 KiB
TypeScript
353 lines
15 KiB
TypeScript
import { expect, test } from "@playwright/test";
|
|
import { serverLoader } from "../helpers/canvas-download-server-loader";
|
|
import { CANVAS_CAPABILITY_IDS } from "@/lib/canvas/capabilities/types";
|
|
import { CANVAS_CAPABILITY_REGISTRY } from "@/lib/canvas/capabilities/registry";
|
|
import {
|
|
getCanvasAgentAccessManifest,
|
|
isExplicitDestructiveIntent,
|
|
summarizeAgentCapabilities,
|
|
VIDEO_CANVAS_EXTRA_ACTIONS,
|
|
VIDEO_CANVAS_INTERACTIVE_ACTIONS,
|
|
} from "@/lib/ai/canvas-agent-runtime/capability-access";
|
|
import {
|
|
buildVideoRuntimeTools,
|
|
type VideoRuntimePorts,
|
|
} from "@/lib/ai/canvas-agent-runtime/video-tools";
|
|
import { newAgentRunSnapshot } from "@/lib/ai/canvas-agent-runtime/protocol";
|
|
import {
|
|
inferVideoTaskRequirements,
|
|
videoAgentSettingsSchema,
|
|
type VideoRuntimeNode,
|
|
} from "@/lib/ai/canvas-agent-runtime/video-protocol";
|
|
import { defaultVideoCanvasFunctionConfig } from "@/lib/video-canvas/config";
|
|
import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing";
|
|
import { estimateContextTokens } from "@/lib/ai/canvas-agent-runtime/context";
|
|
import { finalImageResultNodeIds } from "@/lib/ai/canvas-agent-runtime/results";
|
|
|
|
test("full capability discovery leaves room for subsequent edits without losing backend limitations", () => {
|
|
const manifest = getCanvasAgentAccessManifest();
|
|
const summaries = summarizeAgentCapabilities(manifest);
|
|
expect(summaries.map((item) => item.id)).toEqual(manifest.map((item) => item.id));
|
|
for (const item of manifest.filter((entry) => entry.execution === "backend_required" && entry.authorized))
|
|
expect(summaries.find((entry) => entry.id === item.id)?.note).toBe(item.note);
|
|
expect(
|
|
estimateContextTokens([{ role: "user", content: JSON.stringify(summaries) }]),
|
|
).toBeLessThan(12_000);
|
|
});
|
|
|
|
test("image conversation memory excludes text, slots and 3D results from image references", () => {
|
|
const state = newAgentRunSnapshot("添加文本并生成图片", []);
|
|
state.resultNodeIds = ["text", "slot", "model", "image"];
|
|
expect(
|
|
finalImageResultNodeIds(state, [
|
|
{ id: "text", type: "text" },
|
|
{ id: "slot", type: "agent-slot" },
|
|
{ id: "model", type: "model3d" },
|
|
{ id: "image", type: "image" },
|
|
]),
|
|
).toEqual(["image"]);
|
|
});
|
|
|
|
function videoHarness(objective = "使用参考图生成一条视频") {
|
|
const state = newAgentRunSnapshot(objective, [{ role: "user", content: objective }]);
|
|
state.workspace = "video";
|
|
state.videoSettings = videoAgentSettingsSchema.parse({});
|
|
state.videoRequirements = inferVideoTaskRequirements(objective);
|
|
const nodes: VideoRuntimeNode[] = [
|
|
{
|
|
id: "image-a",
|
|
kind: "media",
|
|
title: "参考图",
|
|
inputType: "image",
|
|
x: 10,
|
|
y: 20,
|
|
width: 320,
|
|
height: 220,
|
|
accent: "#aaa",
|
|
draftValues: { mediaType: "image", mediaUrl: "https://example.test/a.png" },
|
|
},
|
|
];
|
|
const generated: unknown[] = [],
|
|
events: Array<{ type: string; data: unknown }> = [],
|
|
edits: unknown[] = [];
|
|
const ports: VideoRuntimePorts = {
|
|
read: async () => ({ nodes, edges: [] }),
|
|
append: async (node) => {
|
|
nodes.push(node);
|
|
return node.id;
|
|
},
|
|
generateImage: async (input) => {
|
|
generated.push(input);
|
|
return { url: "https://example.test/out.png", model: "configured-image" };
|
|
},
|
|
generateVideo: async (input) => {
|
|
generated.push(input);
|
|
return { url: "https://example.test/out.mp4", model: "configured-video" };
|
|
},
|
|
analyze: async () => ({ raw: "真实分析", analysisRows: [{ sceneDescription: "实际场景" }] }),
|
|
mutate: async (_id, input) => {
|
|
edits.push(input);
|
|
return { status: "succeeded", updatedNodeIds: input.nodeIds };
|
|
},
|
|
processImage: async (input) => {
|
|
generated.push(input);
|
|
return [{ url: "https://example.test/processed.png" }];
|
|
},
|
|
emit: (type, data) => events.push({ type, data }),
|
|
};
|
|
const tools = buildVideoRuntimeTools({
|
|
runId: "test-run",
|
|
projectId: "11111111-1111-4111-8111-111111111111",
|
|
selectedNodeIds: ["image-a"],
|
|
snapshot: state,
|
|
ports,
|
|
});
|
|
return { state, nodes, generated, events, edits, tools, ports };
|
|
}
|
|
|
|
test("supported image capabilities remain authorized while retired multi-angle tools are unavailable", async () => {
|
|
expect(CANVAS_CAPABILITY_REGISTRY.filter(item => !item.agent.exposed).map(item => item.id)).toEqual(["image.multi_angle"]);
|
|
expect(CANVAS_CAPABILITY_REGISTRY.find(item => item.id === "image.multi_angle")?.mcp.exposed).toBe(false);
|
|
expect(CANVAS_CAPABILITY_IDS).toHaveLength(25);
|
|
});
|
|
test("retired angle tools are absent even with explicit targets and toolbar visibility enabled", async () => {
|
|
const load = serverLoader({
|
|
"@/lib/db/queries": { getSystemSetting: async () => null },
|
|
"@/lib/canvas/capabilities/runtime": { executeCanvasCapabilityRuntime: async () => { throw new Error("unexpected execution"); } },
|
|
});
|
|
const { createCanvasCapabilityTools } = load("lib/canvas/capabilities/ai-sdk-tools.ts");
|
|
const tools = await createCanvasCapabilityTools({ projectId: "p", userId: "u", requestUrl: "https://example.test", requestHeaders: new Headers(), selectedNodeIds: ["reference"], allowExplicitTargets: true });
|
|
expect(tools.angleEditImage).toBeUndefined();
|
|
expect(tools.editTargetImage).toBeDefined();
|
|
const { isCanvasCapabilityEnabled } = load("lib/canvas/capabilities/availability.ts");
|
|
const { getCanvasCapability } = load("lib/canvas/capabilities/registry.ts");
|
|
expect(await isCanvasCapabilityEnabled(getCanvasCapability("image.multi_angle"), "agent")).toBe(false);
|
|
});
|
|
test("Agent cannot open the retired angle panel through a generic UI command", async () => {
|
|
const events: unknown[] = [], definitions: any = {}, executable: any = {};
|
|
const load = serverLoader({ "@/lib/canvas/agent-mutations": {}, "./canvas": {} });
|
|
load("lib/ai/canvas-agent-runtime/image-access-tools.ts").registerImageCanvasAccessTools({ userId: "u", projectId: "p", runId: "r", selectedNodeIds: ["reference"], definitions, executable, emit: (...args: unknown[]) => events.push(args) });
|
|
await expect(executable.openCanvasFeature.execute({ featureId: "canvas.ui.imageMultiAngle" }, "old-panel")).rejects.toThrow("图片画布界面入口");
|
|
expect(events).toHaveLength(0);
|
|
});
|
|
test("access manifest covers all configured video functions and node/UI actions without granting fake backends", () => {
|
|
const manifest = getCanvasAgentAccessManifest();
|
|
expect(new Set(manifest.map((item) => item.id)).size).toBe(manifest.length);
|
|
expect(manifest.filter(item => item.workspace === "canvas" && !item.authorized).map(item => item.id)).toEqual(["image.multi_angle", "canvas.ui.imageMultiAngle"]);
|
|
const retired = summarizeAgentCapabilities(manifest).find(item => item.id === "image.multi_angle");
|
|
expect(retired).not.toHaveProperty("entry");
|
|
expect(manifest.find(item => item.id === "image.multi_angle")?.note).toContain("generateCanvasImages");
|
|
expect(manifest.filter(item => item.workspace === "video").every(item => !item.authorized)).toBe(true);
|
|
for (const key of Object.keys(defaultVideoCanvasFunctionConfig.functions))
|
|
expect(manifest.some((item) => item.id === `video.function.${key}`)).toBe(true);
|
|
for (const [key] of VIDEO_CANVAS_EXTRA_ACTIONS)
|
|
expect(manifest.some((item) => item.id === `video.node.${key}`)).toBe(true);
|
|
for (const [key] of VIDEO_CANVAS_INTERACTIVE_ACTIONS)
|
|
expect(manifest.some((item) => item.id === `video.ui.${key}`)).toBe(true);
|
|
expect(manifest.find((item) => item.id === "video.function.audio-to-video")).toMatchObject({
|
|
authorized: false,
|
|
execution: "backend_required",
|
|
});
|
|
expect(manifest.find((item) => item.id === "video.ui.subtitle-erase")?.note).toContain("模拟");
|
|
for (const key of ["video-hd", "subtitle-erase", "voice-split", "background-split"])
|
|
expect(manifest.find((item) => item.id === `video.ui.${key}`)?.execution).toBe(
|
|
"backend_required",
|
|
);
|
|
});
|
|
test("explicit destructive intent cannot be forged by negative or unrelated user requests", () => {
|
|
expect(isExplicitDestructiveIntent("删除选中图片")).toBe(true);
|
|
expect(isExplicitDestructiveIntent("不要删除选中图片")).toBe(false);
|
|
expect(isExplicitDestructiveIntent("检查画布功能列表")).toBe(false);
|
|
});
|
|
test("image routing now admits formerly excluded utilities, 3D and capability discovery", () => {
|
|
for (const text of [
|
|
"下载选中的图片",
|
|
"给图片重命名",
|
|
"生成3D模型",
|
|
"有哪些功能",
|
|
"打开排版替换",
|
|
"删除选中的图片",
|
|
])
|
|
expect(
|
|
shouldUseCanvasAgentRuntime({
|
|
composerMode: "agent",
|
|
contextPayload: { source: "canvas", projectId: "p" },
|
|
message: { parts: [{ type: "text", text }] },
|
|
}),
|
|
).toBe(true);
|
|
});
|
|
test("configured video presets use actual references, prompt and configured feature billing identity", async () => {
|
|
const h = videoHarness("生成角色三视图");
|
|
const result = await h.tools.executable.runVideoCanvasFunction.execute(
|
|
{ functionKey: "character-three-view", nodeIds: ["image-a"], prompt: "保留产品结构" },
|
|
"preset",
|
|
);
|
|
expect(h.generated[0]).toMatchObject({
|
|
references: ["https://example.test/a.png"],
|
|
functionKey: "character-three-view",
|
|
aspectRatio: "16:9",
|
|
});
|
|
expect(String((h.generated[0] as { prompt: string }).prompt)).toContain("保留产品结构");
|
|
expect(result.createdNodeIds).toHaveLength(1);
|
|
expect(h.nodes[0].id).toBe("image-a");
|
|
});
|
|
test("video remove-background and grid split use actual processor results", async () => {
|
|
for (const functionKey of ["remove-bg", "grid-split"]) {
|
|
const h = videoHarness("给图片抠图");
|
|
const output = await h.tools.executable.runVideoCanvasFunction.execute(
|
|
{ functionKey, nodeIds: ["image-a"], args: { rows: 3, columns: 3 } },
|
|
functionKey,
|
|
);
|
|
expect(output.createdNodeIds).toHaveLength(1);
|
|
expect(h.generated).toHaveLength(1);
|
|
expect(h.nodes.at(-1)?.draftValues?.mediaUrl).toBe("https://example.test/processed.png");
|
|
}
|
|
});
|
|
test("a disabled backend remains disabled even though the Agent has feature authorization", async () => {
|
|
const h = videoHarness();
|
|
h.ports.featureConfig = { ...defaultVideoCanvasFunctionConfig, enabled: false };
|
|
const tools = buildVideoRuntimeTools({
|
|
runId: "r",
|
|
selectedNodeIds: ["image-a"],
|
|
snapshot: h.state,
|
|
ports: h.ports,
|
|
});
|
|
await expect(
|
|
tools.executable.runVideoCanvasFunction.execute(
|
|
{ functionKey: "remove-bg", nodeIds: ["image-a"] },
|
|
"off",
|
|
),
|
|
).rejects.toThrow("关闭");
|
|
expect(h.generated).toEqual([]);
|
|
});
|
|
test("audio-driven generation only opens an honest configuration entry and never fabricates a video", async () => {
|
|
const h = videoHarness("音频生成视频");
|
|
const output = await h.tools.executable.runVideoCanvasFunction.execute(
|
|
{ functionKey: "audio-to-video", nodeIds: ["image-a"] },
|
|
"audio",
|
|
);
|
|
expect(output).toMatchObject({ mode: "interactive", backendRequired: true });
|
|
expect(h.generated).toEqual([]);
|
|
expect(h.events[0].type).toBe("data-canvasAgentUiCommand");
|
|
});
|
|
test("configured video generation cannot bypass the per-task paid-video budget", async () => {
|
|
const h = videoHarness();
|
|
h.state.calls = [
|
|
{
|
|
id: "one",
|
|
name: "runVideoCanvasFunction",
|
|
input: { functionKey: "first-frame-video" },
|
|
status: "succeeded",
|
|
},
|
|
{
|
|
id: "two",
|
|
name: "runVideoCanvasFunction",
|
|
input: { functionKey: "first-frame-video" },
|
|
status: "started",
|
|
},
|
|
];
|
|
await expect(
|
|
h.tools.executable.runVideoCanvasFunction.execute(
|
|
{ functionKey: "first-frame-video", nodeIds: ["image-a"] },
|
|
"two",
|
|
),
|
|
).rejects.toThrow("未再发出付费请求");
|
|
expect(h.generated).toEqual([]);
|
|
});
|
|
test("video node editing is authorized but arbitrary or unselected deletions are rejected", async () => {
|
|
const h = videoHarness("不要删除图片，只修改名称");
|
|
await h.tools.executable.editVideoCanvas.execute(
|
|
{ action: "rename", nodeIds: ["image-a"], text: "商品参考" },
|
|
"rename",
|
|
);
|
|
await expect(
|
|
h.tools.executable.editVideoCanvas.execute(
|
|
{ action: "delete", nodeIds: ["image-a"] },
|
|
"delete",
|
|
),
|
|
).rejects.toThrow("明确要求");
|
|
await expect(
|
|
h.tools.executable.editVideoCanvas.execute({ action: "move", nodeIds: ["foreign"] }, "foreign"),
|
|
).rejects.toThrow("不属于");
|
|
expect(h.edits).toHaveLength(1);
|
|
});
|
|
test("video parsing and canvas editing do not accidentally authorize paid video generation", () => {
|
|
expect(inferVideoTaskRequirements("解析这个视频").expectsVideo).toBe(false);
|
|
expect(inferVideoTaskRequirements("删除这个视频节点").expectsVideo).toBe(false);
|
|
expect(inferVideoTaskRequirements("生成一条5秒视频").expectsVideo).toBe(true);
|
|
});
|
|
|
|
test("real media tools require an owned video, retain the source, and write real returned URLs", async () => {
|
|
const h = videoHarness("提取视频首帧");
|
|
h.nodes.push({
|
|
...h.nodes[0],
|
|
id: "video-a",
|
|
inputType: "video",
|
|
draftValues: { mediaType: "video", mediaUrl: "https://example.test/a.mp4" },
|
|
});
|
|
h.ports.processMedia = async (input) => {
|
|
h.generated.push(input);
|
|
return { url: "https://cdn.example.test/frame.png", type: "image", duration: 5 };
|
|
};
|
|
const result = await h.tools.executable.processVideoMedia.execute(
|
|
{ operation: "extract-frame", nodeId: "video-a", frame: "first" },
|
|
"frame",
|
|
);
|
|
expect(result.createdNodeIds).toHaveLength(1);
|
|
expect(h.generated[0]).toMatchObject({
|
|
url: "https://example.test/a.mp4",
|
|
operation: "extract-frame",
|
|
});
|
|
expect(h.nodes.at(-1)?.draftValues?.mediaUrl).toBe("https://cdn.example.test/frame.png");
|
|
expect(h.nodes.find((node) => node.id === "video-a")?.draftValues?.mediaUrl).toBe(
|
|
"https://example.test/a.mp4",
|
|
);
|
|
await expect(
|
|
h.tools.executable.processVideoMedia.execute(
|
|
{ operation: "extract-frame", nodeId: "foreign" },
|
|
"foreign",
|
|
),
|
|
).rejects.toThrow("不属于");
|
|
});
|
|
|
|
test("custom configured video functions cannot bypass the authorized paid-video count", async () => {
|
|
const h = videoHarness();
|
|
h.ports.featureConfig = {
|
|
...defaultVideoCanvasFunctionConfig,
|
|
functions: {
|
|
...defaultVideoCanvasFunctionConfig.functions,
|
|
customVideo: {
|
|
...defaultVideoCanvasFunctionConfig.functions["first-frame-video"],
|
|
key: "customVideo",
|
|
},
|
|
},
|
|
};
|
|
h.state.calls = [
|
|
{
|
|
id: "one",
|
|
name: "runVideoCanvasFunction",
|
|
input: { functionKey: "customVideo" },
|
|
status: "succeeded",
|
|
},
|
|
{
|
|
id: "two",
|
|
name: "runVideoCanvasFunction",
|
|
input: { functionKey: "customVideo" },
|
|
status: "started",
|
|
},
|
|
];
|
|
const tools = buildVideoRuntimeTools({
|
|
runId: "custom",
|
|
selectedNodeIds: ["image-a"],
|
|
snapshot: h.state,
|
|
ports: h.ports,
|
|
});
|
|
await expect(
|
|
tools.executable.runVideoCanvasFunction.execute(
|
|
{ functionKey: "customVideo", nodeIds: ["image-a"] },
|
|
"two",
|
|
),
|
|
).rejects.toThrow("未再发出付费请求");
|
|
expect(h.generated).toEqual([]);
|
|
});
|