376 lines
13 KiB
TypeScript
376 lines
13 KiB
TypeScript
import { expect, test } from "@playwright/test";
|
||
import type { ModelMessage } from "ai";
|
||
import {
|
||
newAgentRunSnapshot,
|
||
type AgentCall,
|
||
type AgentRunSnapshot,
|
||
} from "@/lib/ai/canvas-agent-runtime/protocol";
|
||
import { runCanvasAgentLoop } from "@/lib/ai/canvas-agent-runtime/loop";
|
||
import {
|
||
buildVideoRuntimeTools,
|
||
type VideoRuntimePorts,
|
||
} from "@/lib/ai/canvas-agent-runtime/video-tools";
|
||
import {
|
||
inferVideoTaskRequirements,
|
||
resolveVideoMode,
|
||
videoAgentSettingsSchema,
|
||
serializeVideoRuntimeNode,
|
||
restoreVideoRuntimeNodes,
|
||
mergeVideoAgentResults,
|
||
type VideoRuntimeNode,
|
||
} from "@/lib/ai/canvas-agent-runtime/video-protocol";
|
||
import { verifyVideoAgentResults } from "@/lib/ai/canvas-agent-runtime/video-review";
|
||
|
||
function harness(objective = "先写分镜，再生成一条5秒视频") {
|
||
const state = newAgentRunSnapshot(objective, [{ role: "user", content: objective }]);
|
||
state.workspace = "video";
|
||
state.videoSettings = videoAgentSettingsSchema.parse({});
|
||
state.videoRequirements = inferVideoTaskRequirements(objective);
|
||
const nodes: VideoRuntimeNode[] = [
|
||
{
|
||
id: "original",
|
||
kind: "media",
|
||
title: "参考图",
|
||
x: 20,
|
||
y: 40,
|
||
width: 320,
|
||
height: 200,
|
||
inputType: "image",
|
||
accent: "#aaa",
|
||
draftValues: { mediaType: "image", mediaUrl: "https://example.test/original.png" },
|
||
},
|
||
];
|
||
const calls: string[] = [];
|
||
const ports: VideoRuntimePorts = {
|
||
read: async () => ({ nodes, edges: [] }),
|
||
append: async (node) => {
|
||
if (!nodes.some((old) => old.id === node.id)) nodes.push(node);
|
||
return node.id;
|
||
},
|
||
analyze: async (input) => ({ summary: `实际素材分析：${input.prompt}`, analysisRows: [] }),
|
||
generateImage: async ({ callId }) => {
|
||
calls.push(`image:${callId}`);
|
||
return { url: "https://example.test/keyframe.png", model: "existing-image-model" };
|
||
},
|
||
generateVideo: async ({ callId }) => {
|
||
calls.push(`video:${callId}`);
|
||
return { url: "https://example.test/result.mp4", model: "existing-video-model" };
|
||
},
|
||
};
|
||
const tools = buildVideoRuntimeTools({
|
||
runId: "run",
|
||
selectedNodeIds: ["original"],
|
||
snapshot: state,
|
||
ports,
|
||
});
|
||
return { state, nodes, calls, ports, tools };
|
||
}
|
||
const row = {
|
||
sceneDescription: "展示商品",
|
||
imagePrompt: "保留商品结构",
|
||
motionPrompt: "缓慢推镜",
|
||
duration: 5,
|
||
};
|
||
const call = (id: string, name: string, input: Record<string, unknown> = {}): AgentCall => ({
|
||
id,
|
||
name,
|
||
input,
|
||
});
|
||
function step(calls: AgentCall[] = []) {
|
||
return {
|
||
text: "已完成",
|
||
calls,
|
||
messages: [
|
||
{
|
||
role: "assistant",
|
||
content: calls.length
|
||
? calls.map((item) => ({
|
||
type: "tool-call",
|
||
toolCallId: item.id,
|
||
toolName: item.name,
|
||
input: item.input,
|
||
}))
|
||
: "已完成",
|
||
},
|
||
] as ModelMessage[],
|
||
inputTokens: 30,
|
||
outputTokens: 20,
|
||
};
|
||
}
|
||
|
||
test("video settings validate modes and preserve first-last reference order", () => {
|
||
expect(resolveVideoMode("auto", 0)).toBe("text-to-video");
|
||
expect(resolveVideoMode("auto", 1)).toBe("first-frame");
|
||
expect(resolveVideoMode("auto", 2)).toBe("first-last-frame");
|
||
expect(resolveVideoMode("auto", 3)).toBe("reference-images");
|
||
expect(() => resolveVideoMode("first-last-frame", 1)).toThrow("不会自动丢弃");
|
||
expect(() => resolveVideoMode("text-to-video", 1)).toThrow();
|
||
expect(() => resolveVideoMode("reference-images", 5)).toThrow();
|
||
expect(videoAgentSettingsSchema.safeParse({ duration: 999 }).success).toBe(false);
|
||
});
|
||
test("requirements distinguish scripts from actual video and explicit quantities", () => {
|
||
expect(inferVideoTaskRequirements("只要视频分镜脚本，不要生成视频").expectsVideo).toBe(false);
|
||
expect(inferVideoTaskRequirements("先写脚本，再生成三条9:16视频，每条8秒")).toMatchObject({
|
||
expectsVideo: true,
|
||
expectsScript: true,
|
||
duration: 8,
|
||
ratio: "9:16",
|
||
});
|
||
expect(inferVideoTaskRequirements("生成三条视频").expectedVideoCount).toBe(3);
|
||
});
|
||
test("native video node serialization round-trips without accepting image-canvas nodes", () => {
|
||
const h = harness();
|
||
expect(
|
||
restoreVideoRuntimeNodes({
|
||
nodes: [
|
||
serializeVideoRuntimeNode(h.nodes[0]),
|
||
{ id: "x", type: "image", data: { src: "url" } },
|
||
],
|
||
}),
|
||
).toEqual(h.nodes);
|
||
expect(
|
||
restoreVideoRuntimeNodes({
|
||
nodes: [{ ...serializeVideoRuntimeNode(h.nodes[0]), id: "spoof" }],
|
||
}),
|
||
).toEqual([]);
|
||
expect(
|
||
mergeVideoAgentResults(
|
||
[{ id: "edited", x: 123 }],
|
||
[
|
||
{ id: "edited", x: 0 },
|
||
{ id: "new", x: 10 },
|
||
],
|
||
),
|
||
).toEqual([
|
||
{ id: "edited", x: 123 },
|
||
{ id: "new", x: 10 },
|
||
]);
|
||
});
|
||
test("scripts, keyframes and video execute in order through the real ZCode scheduling loop", async () => {
|
||
const h = harness();
|
||
const sequence = [
|
||
step([
|
||
call("plan", "planCanvasTask", { steps: ["脚本", "关键帧", "视频"], expectedVideoCount: 1 }),
|
||
]),
|
||
step([call("read", "readCanvas")]),
|
||
step([
|
||
call("script", "writeVideoScript", {
|
||
title: "商品分镜",
|
||
rows: [row],
|
||
sourceNodeIds: ["original"],
|
||
}),
|
||
]),
|
||
step([
|
||
call("keyframe", "generateVideoKeyframe", {
|
||
prompt: row.imagePrompt,
|
||
referenceNodeIds: ["original"],
|
||
}),
|
||
]),
|
||
];
|
||
let modelStep = 0;
|
||
const checkpoints: AgentRunSnapshot[] = [];
|
||
const result = await runCanvasAgentLoop(h.state, {
|
||
model: async () => {
|
||
modelStep++;
|
||
if (sequence.length) return sequence.shift()!;
|
||
if (modelStep === 5)
|
||
return step([
|
||
call("video", "generateVideoShot", {
|
||
prompt: row.motionPrompt,
|
||
referenceNodeIds: [h.nodes.find((node) => node.title === "分镜关键帧")!.id],
|
||
}),
|
||
]);
|
||
return step();
|
||
},
|
||
tools: h.tools.executable,
|
||
checkpoint: async (state) => {
|
||
checkpoints.push(structuredClone(state));
|
||
},
|
||
consumeInputs: async () => ({ cancelled: false }),
|
||
isCancelled: async () => false,
|
||
summarize: async () => "视频任务摘要",
|
||
verify: (state) =>
|
||
verifyVideoAgentResults({
|
||
snapshot: state,
|
||
nodes: h.nodes,
|
||
inspect: async () => ({ passed: true, issues: [], duration: 5, width: 1280, height: 720 }),
|
||
}),
|
||
});
|
||
expect(result).toBe("completed");
|
||
expect(h.calls).toEqual(["image:keyframe", "video:video"]);
|
||
expect(h.nodes[0].id).toBe("original");
|
||
expect(h.state.resultNodeIds).toHaveLength(3);
|
||
expect(
|
||
checkpoints.some((state) =>
|
||
state.calls.some((item) => item.id === "video" && item.status === "started"),
|
||
),
|
||
).toBe(true);
|
||
});
|
||
test("unowned or nonexistent reference IDs never reach a paid provider", async () => {
|
||
const h = harness();
|
||
await expect(
|
||
h.tools.executable.generateVideoShot.execute(
|
||
{ prompt: "视频", referenceNodeIds: ["other-project"] },
|
||
"x",
|
||
),
|
||
).rejects.toThrow("素材节点不可用");
|
||
expect(h.calls).toEqual([]);
|
||
});
|
||
test("a script-only request cannot accidentally start a billable video", async () => {
|
||
const h = harness("只要分镜脚本，不要生成视频");
|
||
await expect(
|
||
h.tools.executable.generateVideoShot.execute({ prompt: "视频", referenceNodeIds: [] }, "x"),
|
||
).rejects.toThrow("尚未授权");
|
||
expect(h.calls).toEqual([]);
|
||
});
|
||
test("first-last mode refuses missing frames instead of falling back to another mode", async () => {
|
||
const h = harness();
|
||
h.state.videoSettings!.mode = "first-last-frame";
|
||
await expect(
|
||
h.tools.executable.generateVideoShot.execute(
|
||
{ prompt: "视频", referenceNodeIds: ["original"] },
|
||
"x",
|
||
),
|
||
).rejects.toThrow("不匹配");
|
||
expect(h.calls).toEqual([]);
|
||
});
|
||
test("script results use native text resources and display every storyboard row", async () => {
|
||
const h = harness("只要分镜脚本");
|
||
const result = await h.tools.executable.writeVideoScript.execute(
|
||
{ title: "分镜", rows: [row, { ...row, sceneDescription: "第二镜头" }], sourceNodeIds: [] },
|
||
"script",
|
||
);
|
||
const node = h.nodes.find((item) => item.id === (result.createdNodeIds as string[])[0])!;
|
||
expect(node.draftValues).toMatchObject({
|
||
action: "TEXT_RESOURCE",
|
||
nodeType: "text",
|
||
scriptRole: "resource",
|
||
});
|
||
expect(node.draftValues?.content).toHaveLength(1);
|
||
expect((node.draftValues?.content as string[])[0]).toContain("02. 第二镜头");
|
||
expect(node.draftValues?.rows).toHaveLength(2);
|
||
});
|
||
|
||
test("scripts and keyframes cannot satisfy a request for actual videos", async () => {
|
||
const h = harness("生成两条视频");
|
||
const script = await h.tools.executable.writeVideoScript.execute(
|
||
{ title: "分镜", rows: [row], sourceNodeIds: [] },
|
||
"script",
|
||
);
|
||
h.state.resultNodeIds = script.createdNodeIds as string[];
|
||
const verdict = await verifyVideoAgentResults({
|
||
snapshot: h.state,
|
||
nodes: h.nodes,
|
||
inspect: async () => ({ passed: true, issues: [], duration: 5, width: 1280, height: 720 }),
|
||
});
|
||
expect(verdict.passed).toBe(false);
|
||
expect(verdict.issues.join()).toContain("当前仅完成 0 条");
|
||
expect(verdict.retryable).toBe(false);
|
||
});
|
||
test("visual review failures retain results without automatic paid regeneration", async () => {
|
||
const h = harness("生成一条视频");
|
||
const output = await h.tools.executable.generateVideoShot.execute(
|
||
{ prompt: "视频", referenceNodeIds: ["original"] },
|
||
"video",
|
||
);
|
||
h.state.resultNodeIds = output.createdNodeIds as string[];
|
||
const result = await verifyVideoAgentResults({
|
||
snapshot: h.state,
|
||
nodes: h.nodes,
|
||
inspect: async () => ({ passed: false, issues: ["商品颜色不一致"] }),
|
||
});
|
||
expect(result).toMatchObject({ passed: false, retryable: false });
|
||
expect(h.calls).toHaveLength(1);
|
||
const unavailable = await verifyVideoAgentResults({
|
||
snapshot: h.state,
|
||
nodes: h.nodes,
|
||
inspect: async () => {
|
||
throw new Error("cannot decode");
|
||
},
|
||
});
|
||
expect(unavailable.passed).toBe(false);
|
||
expect(unavailable.issues.join()).toContain("无法读取");
|
||
});
|
||
test("video material analysis receives real URLs and excludes base64 from persisted tool history", async () => {
|
||
const h = harness("分析选中图片");
|
||
let actual: unknown;
|
||
h.ports.analyze = async (input) => {
|
||
actual = input;
|
||
return { summary: "图中有商品" };
|
||
};
|
||
const output = await h.tools.executable.analyzeVideoMaterials.execute(
|
||
{ nodeIds: ["original"], question: "分析商品" },
|
||
"a",
|
||
);
|
||
expect(actual).toMatchObject({ imageUrls: ["https://example.test/original.png"], videoUrls: [] });
|
||
expect(output.summary).toBe("图中有商品");
|
||
});
|
||
|
||
test("verification measures actual duration and dimensions instead of trusting request metadata", async () => {
|
||
const h = harness("生成一条5秒16:9视频");
|
||
const output = await h.tools.executable.generateVideoShot.execute(
|
||
{ prompt: "视频", referenceNodeIds: ["original"] },
|
||
"video",
|
||
);
|
||
h.state.resultNodeIds = output.createdNodeIds as string[];
|
||
const result = await verifyVideoAgentResults({
|
||
snapshot: h.state,
|
||
nodes: h.nodes,
|
||
inspect: async () => ({ passed: true, issues: [], duration: 9, width: 720, height: 1280 }),
|
||
});
|
||
expect(result.passed).toBe(false);
|
||
expect(result.issues.join()).toContain("实际时长");
|
||
expect(result.issues.join()).toContain("实际尺寸");
|
||
expect(result.retryable).toBe(false);
|
||
});
|
||
|
||
test("stopping during a video call retains the paid result and prevents the next shot", async () => {
|
||
const h = harness("生成两条视频");
|
||
let cancelled = false;
|
||
h.ports.generateVideo = async () => {
|
||
cancelled = true;
|
||
return { url: "https://example.test/paid.mp4", model: "configured" };
|
||
};
|
||
const sequence = [
|
||
step([call("plan", "planCanvasTask", { steps: ["两个镜头"], expectedVideoCount: 2 })]),
|
||
step([
|
||
call("one", "generateVideoShot", { prompt: "第一条", referenceNodeIds: ["original"] }),
|
||
call("two", "generateVideoShot", { prompt: "第二条", referenceNodeIds: ["original"] }),
|
||
]),
|
||
];
|
||
const result = await runCanvasAgentLoop(h.state, {
|
||
model: async () => sequence.shift()!,
|
||
tools: h.tools.executable,
|
||
checkpoint: async () => {},
|
||
consumeInputs: async () => ({ cancelled }),
|
||
isCancelled: async () => cancelled,
|
||
summarize: async () => "摘要",
|
||
verify: async () => ({ passed: true, issues: [] }),
|
||
});
|
||
expect(result).toBe("cancelled");
|
||
expect(h.state.resultNodeIds).toHaveLength(1);
|
||
expect(h.state.calls.find((item) => item.id === "one")?.status).toBe("succeeded");
|
||
expect(h.state.calls.find((item) => item.id === "two")?.status).toBe("failed");
|
||
});
|
||
|
||
test("an extra model tool call cannot exceed the user's authorized video count", async () => {
|
||
const h = harness("生成一条视频");
|
||
h.state.calls = [
|
||
{
|
||
id: "first",
|
||
name: "generateVideoShot",
|
||
input: {},
|
||
status: "succeeded",
|
||
output: { createdNodeIds: ["first-result"] },
|
||
},
|
||
{ id: "second", name: "generateVideoShot", input: {}, status: "started" },
|
||
];
|
||
await expect(
|
||
h.tools.executable.generateVideoShot.execute(
|
||
{ prompt: "额外视频", referenceNodeIds: ["original"] },
|
||
"second",
|
||
),
|
||
).rejects.toThrow("未再发出付费请求");
|
||
expect(h.calls).toEqual([]);
|
||
});
|