511 lines
18 KiB
TypeScript
511 lines
18 KiB
TypeScript
import { expect, test } from "@playwright/test";
|
||
import { generateText, stepCountIs, tool, type ModelMessage } from "ai";
|
||
import type { LanguageModelV2 } from "@ai-sdk/provider";
|
||
import { z } from "zod";
|
||
import { ToolScheduler } from "@/lib/ai/third-party/zcode/scheduler";
|
||
import {
|
||
compactAgentContext,
|
||
splitCompactableHistory,
|
||
} from "@/lib/ai/canvas-agent-runtime/context";
|
||
import { runCanvasAgentLoop, type CanvasAgentLoopPorts } from "@/lib/ai/canvas-agent-runtime/loop";
|
||
import {
|
||
AgentRunControlChangedError,
|
||
newAgentRunSnapshot,
|
||
publicAgentError,
|
||
type AgentCall,
|
||
type AgentRunSnapshot,
|
||
type CanvasAgentRunStatus,
|
||
} from "@/lib/ai/canvas-agent-runtime/protocol";
|
||
import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing";
|
||
import { inferCanvasTaskRequirements } from "@/lib/ai/canvas-agent-runtime/requirements";
|
||
import { finalResultNodeIds, originalSourceNodeIds } from "@/lib/ai/canvas-agent-runtime/results";
|
||
|
||
function step(calls: AgentCall[] = [], text = "已处理") {
|
||
return {
|
||
text,
|
||
calls,
|
||
messages: [
|
||
{
|
||
role: "assistant",
|
||
content: calls.length
|
||
? calls.map((call) => ({
|
||
type: "tool-call",
|
||
toolCallId: call.id,
|
||
toolName: call.name,
|
||
input: call.input,
|
||
}))
|
||
: text,
|
||
},
|
||
] as ModelMessage[],
|
||
inputTokens: 100,
|
||
outputTokens: 30,
|
||
};
|
||
}
|
||
const call = (id: string, name = "readCanvas"): AgentCall => ({
|
||
id,
|
||
name,
|
||
input: { nodeIds: [id] },
|
||
});
|
||
function harness(
|
||
sequence: ReturnType<typeof step>[],
|
||
overrides: Partial<CanvasAgentLoopPorts> = {},
|
||
) {
|
||
const snapshots: Array<{ state: AgentRunSnapshot; status: CanvasAgentRunStatus }> = [];
|
||
const executed: string[] = [];
|
||
const ports: CanvasAgentLoopPorts = {
|
||
model: async () => {
|
||
const result = sequence.shift();
|
||
if (!result) throw new Error("Unexpected model call");
|
||
return result;
|
||
},
|
||
tools: {
|
||
planCanvasTask: {
|
||
title: "计划",
|
||
readOnly: true,
|
||
concurrentSafe: true,
|
||
execute: async () => ({ expectedImageCount: 1 }),
|
||
},
|
||
readCanvas: {
|
||
title: "读取",
|
||
readOnly: true,
|
||
concurrentSafe: true,
|
||
execute: async (_input, id) => {
|
||
executed.push(id);
|
||
return { nodes: [] };
|
||
},
|
||
},
|
||
generateCanvasImages: {
|
||
title: "生成",
|
||
readOnly: false,
|
||
concurrentSafe: false,
|
||
execute: async (_input, id) => {
|
||
executed.push(id);
|
||
return { createdNodeIds: [`image-${id}`], status: "succeeded" };
|
||
},
|
||
},
|
||
},
|
||
checkpoint: async (state, status) => {
|
||
snapshots.push({ state: structuredClone(state), status });
|
||
},
|
||
consumeInputs: async () => ({ cancelled: false }),
|
||
isCancelled: async () => false,
|
||
summarize: async () => "保留主体颜色，不改变商品结构",
|
||
verify: async () => ({ passed: true, issues: [] }),
|
||
...overrides,
|
||
};
|
||
const state = newAgentRunSnapshot("先抠图，再换背景，保持商品结构", [
|
||
{ role: "user", content: "先抠图，再换背景" },
|
||
]);
|
||
return { state, ports, snapshots, executed };
|
||
}
|
||
const plan = () => step([call("plan", "planCanvasTask")]);
|
||
|
||
test("all image and video canvas Agent requests use native runtime; Claw and direct modes remain separate", () => {
|
||
const body = { composerMode: "agent", contextPayload: { source: "canvas", projectId: "owned-project" } };
|
||
for (const override of [{}, { agentPresetId: "preset" }, { workspaceVariant: "video" },
|
||
{ message: { parts: [{ type: "text", text: "只分析不执行" }] } }, { resumeRunId: "run" }])
|
||
expect(shouldUseCanvasAgentRuntime({ ...body, ...override })).toBe(true);
|
||
for (const override of [{ composerMode: "clawd" }, { composerMode: "image" }, { composerMode: "video" },
|
||
{ contextPayload: { source: "canvas" } }, { contextPayload: { source: "chat", projectId: "p" } }])
|
||
expect(shouldUseCanvasAgentRuntime({ ...body, ...override })).toBe(false);
|
||
});
|
||
|
||
test("continues beyond eight model rounds and verifies actual tool results before completion", async () => {
|
||
const h = harness([
|
||
plan(),
|
||
...Array.from({ length: 10 }, (_, index) => step([call(String(index))])),
|
||
step(),
|
||
]);
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("completed");
|
||
expect(h.state.steps).toBe(12);
|
||
expect(h.executed).toHaveLength(10);
|
||
expect(h.snapshots.at(-1)?.state.verification?.passed).toBe(true);
|
||
});
|
||
|
||
test("checkpoint is durable before a tool with side effects starts", async () => {
|
||
const h = harness([plan(), step([call("g", "generateCanvasImages")]), step()]);
|
||
h.ports.tools.generateCanvasImages.execute = async (_input, id) => {
|
||
expect(h.snapshots.at(-1)?.state.calls.find((item) => item.id === id)?.status).toBe("started");
|
||
return { createdNodeIds: ["result"], status: "succeeded" };
|
||
};
|
||
await runCanvasAgentLoop(h.state, h.ports);
|
||
expect(h.state.resultNodeIds).toEqual(["result"]);
|
||
});
|
||
|
||
test("does not run paid tools when the model skipped the plan", async () => {
|
||
const h = harness([step([call("g", "generateCanvasImages")])]);
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("needs_review");
|
||
expect(h.executed).toEqual([]);
|
||
});
|
||
|
||
test("steering waits until every tool result in the batch is settled", async () => {
|
||
const h = harness([plan(), step([call("a"), call("b")]), step()]);
|
||
let pending = false;
|
||
h.ports.tools.readCanvas.execute = async (_input, id) => {
|
||
pending = true;
|
||
await new Promise((resolve) => setTimeout(resolve, 2));
|
||
return { nodeId: id };
|
||
};
|
||
h.ports.consumeInputs = async (state) => {
|
||
if (pending) {
|
||
const toolIds = state.messages.flatMap((message) =>
|
||
message.role === "tool" ? message.content.map((part) => part.toolCallId) : [],
|
||
);
|
||
expect(toolIds).toContain("a");
|
||
expect(toolIds).toContain("b");
|
||
state.messages.push({ role: "user", content: "不要改变产品颜色" });
|
||
state.inputs.push({ id: "extra", text: "不要改变产品颜色", applied: true });
|
||
pending = false;
|
||
}
|
||
return { cancelled: false };
|
||
};
|
||
await runCanvasAgentLoop(h.state, h.ports);
|
||
expect(h.state.inputs).toHaveLength(1);
|
||
});
|
||
|
||
test("stopping an active image request preserves its output and skips later tools", async () => {
|
||
const h = harness([
|
||
plan(),
|
||
step([call("a", "generateCanvasImages"), call("b", "generateCanvasImages")]),
|
||
]);
|
||
let cancelled = false;
|
||
h.ports.isCancelled = async () => cancelled;
|
||
h.ports.tools.generateCanvasImages.execute = async (_input, id) => {
|
||
h.executed.push(id);
|
||
cancelled = true;
|
||
return { createdNodeIds: ["finished-image"] };
|
||
};
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("cancelled");
|
||
expect(h.executed).toEqual(["a"]);
|
||
expect(h.state.resultNodeIds).toEqual(["finished-image"]);
|
||
const last = h.state.messages.at(-1);
|
||
expect(last?.role === "tool" && last.content.length).toBe(2);
|
||
});
|
||
|
||
test("a failed paid request stops subsequent work and cannot be blindly replayed", async () => {
|
||
const h = harness([
|
||
plan(),
|
||
step([call("a", "generateCanvasImages"), call("b", "generateCanvasImages")]),
|
||
]);
|
||
h.ports.tools.generateCanvasImages.execute = async () => {
|
||
throw new Error("<html>Cloudflare 524 ray id secret</html>");
|
||
};
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("needs_review");
|
||
expect(h.state.calls.find((item) => item.id === "a")?.uncertain).toBe(true);
|
||
expect(h.state.calls.find((item) => item.id === "a")?.error).not.toContain("<html>");
|
||
h.ports.model = async () => {
|
||
throw new Error("must not call model");
|
||
};
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("needs_review");
|
||
});
|
||
|
||
test("review failure never reports completion even when the assistant says it finished", async () => {
|
||
const h = harness([plan(), step([call("g", "generateCanvasImages")]), step([], "全部完成")], {
|
||
verify: async () => {
|
||
throw new Error("review provider unavailable");
|
||
},
|
||
});
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("needs_review");
|
||
expect(h.snapshots.every((item) => item.status !== "completed")).toBe(true);
|
||
});
|
||
|
||
test("an interactive panel is a waiting state, not a completed image", async () => {
|
||
const h = harness([plan(), step([call("interactive")])]);
|
||
h.ports.tools.readCanvas.execute = async () => ({ mode: "interactive", status: "succeeded" });
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("needs_review");
|
||
});
|
||
|
||
test("resumes persisted tool outputs without invoking the tool again", async () => {
|
||
const h = harness([step()]);
|
||
h.state.calls = [
|
||
{ ...call("plan", "planCanvasTask"), status: "succeeded", output: {} },
|
||
{
|
||
...call("g", "generateCanvasImages"),
|
||
status: "succeeded",
|
||
output: { createdNodeIds: ["existing"] },
|
||
},
|
||
];
|
||
h.state.messages.push(
|
||
...step([call("plan", "planCanvasTask"), call("g", "generateCanvasImages")]).messages,
|
||
);
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("completed");
|
||
expect(h.executed).toEqual([]);
|
||
expect(h.state.messages.filter((message) => message.role === "tool")).toHaveLength(2);
|
||
});
|
||
|
||
test("input accepted at the completion boundary is processed instead of silently dropped", async () => {
|
||
const h = harness([plan(), step([call("a")]), step(), step([call("b")]), step()]);
|
||
let extra = false;
|
||
const checkpoint = h.ports.checkpoint;
|
||
h.ports.checkpoint = async (state, status) => {
|
||
if (status === "completed" && !extra) {
|
||
extra = true;
|
||
throw new AgentRunControlChangedError();
|
||
}
|
||
await checkpoint(state, status);
|
||
};
|
||
h.ports.consumeInputs = async (state) => {
|
||
if (extra && !state.inputs.length) {
|
||
state.messages.push({ role: "user", content: "补一版竖图" });
|
||
state.inputs.push({ id: "tail", text: "补一版竖图", applied: true });
|
||
}
|
||
return { cancelled: false };
|
||
};
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("completed");
|
||
expect(h.executed).toEqual(["a", "b"]);
|
||
});
|
||
|
||
test("resume gets a fresh token budget while preserving the total usage ledger", async () => {
|
||
const h = harness([step([call("r")]), step()]);
|
||
h.state.inputTokens = 100_000;
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("completed");
|
||
expect(h.state.inputTokens).toBeGreaterThan(100_000);
|
||
});
|
||
|
||
test("compact preserves complete tool pairs and does not change history when summarization fails", async () => {
|
||
const messages: ModelMessage[] = [];
|
||
for (let index = 0; index < 5; index++)
|
||
messages.push(
|
||
{ role: "user", content: `要求-${index}` },
|
||
...step([call(String(index))]).messages,
|
||
{
|
||
role: "tool",
|
||
content: [
|
||
{
|
||
type: "tool-result",
|
||
toolCallId: String(index),
|
||
toolName: "readCanvas",
|
||
output: { type: "text", value: "x".repeat(3000) },
|
||
},
|
||
],
|
||
},
|
||
);
|
||
const split = splitCompactableHistory(messages, 2);
|
||
expect(split.older.at(-1)?.role).toBe("tool");
|
||
expect(split.recent[0].role).toBe("user");
|
||
const before = JSON.stringify(messages);
|
||
await expect(
|
||
compactAgentContext({
|
||
messages,
|
||
previousSummary: "",
|
||
threshold: 1,
|
||
summarize: async () => {
|
||
throw new Error("unavailable");
|
||
},
|
||
}),
|
||
).rejects.toThrow("unavailable");
|
||
expect(JSON.stringify(messages)).toBe(before);
|
||
const compacted = await compactAgentContext({
|
||
messages,
|
||
previousSummary: "",
|
||
threshold: 1,
|
||
summarize: async () => "用户要求不改颜色；已生成 image-1",
|
||
});
|
||
expect(compacted.previousSummary).toContain("不改颜色");
|
||
expect(compacted.messages[0].role).toBe("user");
|
||
});
|
||
|
||
test("vendored scheduler respects dependencies and separates unsafe mutations", () => {
|
||
const scheduler = new ToolScheduler({ maxConcurrency: 10 });
|
||
expect(
|
||
scheduler.schedule([
|
||
{ toolCallId: "r1", dependsOn: [], readOnly: true },
|
||
{ toolCallId: "r2", dependsOn: [], readOnly: true },
|
||
{ toolCallId: "w", dependsOn: ["r1", "r2"], concurrentSafe: false },
|
||
{ toolCallId: "last", dependsOn: ["w"], readOnly: true },
|
||
]).parallelGroups,
|
||
).toEqual([["r1", "r2"], ["w"], ["last"]]);
|
||
expect(() =>
|
||
scheduler.schedule([
|
||
{ toolCallId: "a", dependsOn: ["b"] },
|
||
{ toolCallId: "b", dependsOn: ["a"] },
|
||
]),
|
||
).toThrow();
|
||
});
|
||
|
||
test("public errors strip provider HTML and common key formats", () => {
|
||
expect(publicAgentError("<!doctype html><body>gateway</body>")).not.toContain("html");
|
||
expect(publicAgentError("Authorization: Bearer abcdefghijklmnop")).not.toContain(
|
||
"abcdefghijklmnop",
|
||
);
|
||
});
|
||
|
||
test("real AI SDK tool calls can be executed and fed back through the adapted loop", async () => {
|
||
const fixtures: Array<Awaited<ReturnType<LanguageModelV2["doGenerate"]>>> = [
|
||
{
|
||
content: [{ type: "tool-call", toolCallId: "plan", toolName: "planCanvasTask", input: "{}" }],
|
||
finishReason: "tool-calls",
|
||
usage: { inputTokens: 20, outputTokens: 10, totalTokens: 30 },
|
||
warnings: [],
|
||
},
|
||
{
|
||
content: [
|
||
{ type: "tool-call", toolCallId: "image", toolName: "generateCanvasImages", input: "{}" },
|
||
],
|
||
finishReason: "tool-calls",
|
||
usage: { inputTokens: 30, outputTokens: 10, totalTokens: 40 },
|
||
warnings: [],
|
||
},
|
||
{
|
||
content: [{ type: "text", text: "已完成" }],
|
||
finishReason: "stop",
|
||
usage: { inputTokens: 40, outputTokens: 10, totalTokens: 50 },
|
||
warnings: [],
|
||
},
|
||
];
|
||
const requests: Array<Parameters<LanguageModelV2["doGenerate"]>[0]> = [];
|
||
const model: LanguageModelV2 = {
|
||
specificationVersion: "v2",
|
||
provider: "test",
|
||
modelId: "canvas-test",
|
||
supportedUrls: {},
|
||
doGenerate: async (request) => {
|
||
requests.push(request);
|
||
const result = fixtures.shift();
|
||
if (!result) throw new Error("No mock response");
|
||
return result;
|
||
},
|
||
doStream: async () => {
|
||
throw new Error("Unexpected stream request");
|
||
},
|
||
};
|
||
const h = harness([]);
|
||
const definitions = {
|
||
planCanvasTask: tool({ description: "plan", inputSchema: z.object({}) }),
|
||
generateCanvasImages: tool({ description: "generate", inputSchema: z.object({}) }),
|
||
};
|
||
h.ports.model = async (messages) => {
|
||
const result = await generateText({
|
||
model,
|
||
messages,
|
||
tools: definitions,
|
||
stopWhen: stepCountIs(1),
|
||
});
|
||
return {
|
||
text: result.text,
|
||
calls: result.toolCalls.map((item) => ({
|
||
id: item.toolCallId,
|
||
name: item.toolName,
|
||
input: z.record(z.unknown()).parse(item.input),
|
||
})),
|
||
messages: result.response.messages,
|
||
inputTokens: result.usage.inputTokens ?? 0,
|
||
outputTokens: result.usage.outputTokens ?? 0,
|
||
};
|
||
};
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("completed");
|
||
expect(h.executed).toEqual(["image"]);
|
||
expect(requests[2].prompt.some((message) => message.role === "tool")).toBe(true);
|
||
});
|
||
|
||
test("resuming compacted history does not resurrect orphaned tool results", async () => {
|
||
const h = harness([step()]);
|
||
h.state.summary = "旧工具完成并已压缩，结果 existing-node";
|
||
h.state.calls = [
|
||
{ ...call("old"), status: "succeeded", output: { createdNodeIds: ["existing-node"] } },
|
||
];
|
||
h.ports.model = async (messages) => {
|
||
expect(messages.some((message) => message.role === "tool")).toBe(false);
|
||
return step();
|
||
};
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("completed");
|
||
});
|
||
|
||
test("long single-turn tasks compact at complete tool batches", async () => {
|
||
const messages: ModelMessage[] = [{ role: "user", content: "保持主体不变" }];
|
||
for (let i = 0; i < 12; i++)
|
||
messages.push(...step([call(String(i))]).messages, {
|
||
role: "tool",
|
||
content: [
|
||
{
|
||
type: "tool-result",
|
||
toolCallId: String(i),
|
||
toolName: "readCanvas",
|
||
output: { type: "text", value: "result" },
|
||
},
|
||
],
|
||
});
|
||
const parts = splitCompactableHistory(messages);
|
||
expect(parts.older.length).toBeGreaterThan(0);
|
||
expect(parts.older.at(-1)?.role).toBe("tool");
|
||
expect(parts.recent[0].role).toBe("assistant");
|
||
});
|
||
|
||
test("image requirements calculate per-image variants and avoid inventing a ratio for mixed formats", () => {
|
||
expect(inferCanvasTaskRequirements("这三张商品图统一背景，做成3:4，每张出两版", 3)).toEqual({
|
||
expectedImageCount: 6,
|
||
aspectRatio: "3:4",
|
||
});
|
||
expect(inferCanvasTaskRequirements("最终生成6张，比例1:1")).toEqual({
|
||
expectedImageCount: 6,
|
||
aspectRatio: "1:1",
|
||
});
|
||
expect(inferCanvasTaskRequirements("生成3:4和16:9各一版").aspectRatio).toBeUndefined();
|
||
});
|
||
|
||
test("renaming final images does not make them intermediate outputs", () => {
|
||
const state = newAgentRunSnapshot("换背景再增强", []);
|
||
state.resultNodeIds = ["background", "enhanced"];
|
||
state.calls = [
|
||
{
|
||
...call("a", "generateCanvasImages"),
|
||
input: { referenceNodeIds: ["original"] },
|
||
status: "succeeded",
|
||
output: { createdNodeIds: ["background"] },
|
||
},
|
||
{
|
||
...call("b", "enhanceImage"),
|
||
input: { nodeIds: ["background"] },
|
||
status: "succeeded",
|
||
output: { createdNodeIds: ["enhanced"] },
|
||
},
|
||
{
|
||
...call("c", "rename"),
|
||
input: { nodeIds: ["enhanced"] },
|
||
status: "succeeded",
|
||
output: { createdNodeIds: [] },
|
||
},
|
||
];
|
||
expect(finalResultNodeIds(state)).toEqual(["enhanced"]);
|
||
expect(originalSourceNodeIds(state, ["enhanced"])).toEqual(["original"]);
|
||
});
|
||
|
||
test("only known failed results are eligible for bounded automatic rework", async () => {
|
||
const h = harness([
|
||
plan(),
|
||
step([call("first", "generateCanvasImages")]),
|
||
step(),
|
||
step([
|
||
{ id: "repair", name: "generateCanvasImages", input: { referenceNodeIds: ["image-first"] } },
|
||
]),
|
||
step(),
|
||
]);
|
||
let reviewed = false;
|
||
h.ports.verify = async () => {
|
||
if (reviewed) return { passed: true, issues: [] };
|
||
reviewed = true;
|
||
return {
|
||
passed: false,
|
||
issues: ["背景未替换"],
|
||
failedNodeIds: ["image-first"],
|
||
retryable: true,
|
||
};
|
||
};
|
||
expect(await runCanvasAgentLoop(h.state, h.ports)).toBe("completed");
|
||
expect(h.state.repairs).toBe(1);
|
||
const outside = harness([
|
||
plan(),
|
||
step([call("first", "generateCanvasImages")]),
|
||
step(),
|
||
step([call("other", "generateCanvasImages")]),
|
||
]);
|
||
outside.ports.verify = async () => ({
|
||
passed: false,
|
||
issues: ["背景未替换"],
|
||
failedNodeIds: ["image-first"],
|
||
retryable: true,
|
||
});
|
||
expect(await runCanvasAgentLoop(outside.state, outside.ports)).toBe("needs_review");
|
||
expect(outside.executed).toEqual(["first"]);
|
||
});
|