132 lines
4.9 KiB
TypeScript
132 lines
4.9 KiB
TypeScript
import { expect, test } from "@playwright/test";
|
||
import type { LanguageModelV2 } from "@ai-sdk/provider";
|
||
import sharp from "sharp";
|
||
import { newAgentRunSnapshot } from "@/lib/ai/canvas-agent-runtime/protocol";
|
||
import {
|
||
analyzeRuntimeImages,
|
||
verifyCanvasAgentResults,
|
||
} from "@/lib/ai/canvas-agent-runtime/review";
|
||
|
||
async function fixture(answer: string, resultSize = { width: 30, height: 40 }) {
|
||
const requests: Array<Parameters<LanguageModelV2["doGenerate"]>[0]> = [];
|
||
const model: LanguageModelV2 = {
|
||
specificationVersion: "v2",
|
||
provider: "test",
|
||
modelId: "vision",
|
||
supportedUrls: {},
|
||
doGenerate: async (request) => {
|
||
requests.push(request);
|
||
return {
|
||
content: [{ type: "text", text: answer }],
|
||
finishReason: "stop",
|
||
usage: { inputTokens: 30, outputTokens: 10, totalTokens: 40 },
|
||
warnings: [],
|
||
};
|
||
},
|
||
doStream: async () => {
|
||
throw new Error("Unexpected stream");
|
||
},
|
||
};
|
||
const buffers = {
|
||
source: await sharp({ create: { width: 40, height: 40, channels: 3, background: "white" } })
|
||
.png()
|
||
.toBuffer(),
|
||
result: await sharp({ create: { ...resultSize, channels: 3, background: "white" } })
|
||
.png()
|
||
.toBuffer(),
|
||
};
|
||
const snapshot = newAgentRunSnapshot("背景改白色，主体结构不变，生成3:4", []);
|
||
snapshot.requirements = { expectedImageCount: 1, aspectRatio: "3:4" };
|
||
snapshot.resultNodeIds = ["result"];
|
||
snapshot.calls = [
|
||
{
|
||
id: "plan",
|
||
name: "planCanvasTask",
|
||
input: {},
|
||
status: "succeeded",
|
||
output: { expectedImageCount: 1, visualCriteria: ["主体结构不变"] },
|
||
},
|
||
{
|
||
id: "gen",
|
||
name: "generateCanvasImages",
|
||
input: { referenceNodeIds: ["source"] },
|
||
status: "succeeded",
|
||
output: { status: "succeeded", createdNodeIds: ["result"] },
|
||
},
|
||
];
|
||
const params = {
|
||
snapshot,
|
||
model,
|
||
userId: "test",
|
||
projectId: "test",
|
||
supportsVision: true,
|
||
reviewPorts: {
|
||
loadNodes: async (ids: string[]) =>
|
||
ids.map((id) => ({
|
||
id,
|
||
type: "image",
|
||
position: { x: 0, y: 0 },
|
||
data: { src: `https://assets.invalid/${id}` },
|
||
})),
|
||
readImage: async (url: string) => (url.endsWith("source") ? buffers.source : buffers.result),
|
||
},
|
||
};
|
||
return { params, requests };
|
||
}
|
||
test("checks real image bytes and compares results to their source images", async () => {
|
||
const f = await fixture('{"passed":true,"issues":[],"failedNodeIds":[]}');
|
||
const result = await verifyCanvasAgentResults(f.params);
|
||
expect(result.passed).toBe(true);
|
||
expect(f.requests).toHaveLength(1);
|
||
const content = f.requests[0].prompt.flatMap((message) =>
|
||
Array.isArray(message.content) ? message.content : [],
|
||
);
|
||
expect(
|
||
content.filter((part) => part.type === "file" && part.mediaType.startsWith("image/")),
|
||
).toHaveLength(2);
|
||
// A square source image is not incorrectly tested against the output's 3:4 ratio.
|
||
expect(result.issues).toEqual([]);
|
||
});
|
||
test("actual dimensions override a model's claim that the output passed", async () => {
|
||
const f = await fixture('{"passed":true,"issues":[]}', { width: 60, height: 40 });
|
||
const result = await verifyCanvasAgentResults(f.params);
|
||
expect(result.passed).toBe(false);
|
||
expect(result.failedNodeIds).toEqual(["result"]);
|
||
expect(result.issues.join(" ")).toContain("3:4");
|
||
});
|
||
test("reported problems cannot be ignored by a contradictory passed=true flag", async () => {
|
||
const f = await fixture('{"passed":true,"issues":["主体细节已改变"],"failedNodeIds":["result"]}');
|
||
const result = await verifyCanvasAgentResults(f.params);
|
||
expect(result.passed).toBe(false);
|
||
expect(result.retryable).toBe(true);
|
||
});
|
||
test("review infrastructure failure never becomes an automatic pass", async () => {
|
||
const f = await fixture("<html>unavailable</html>");
|
||
await expect(verifyCanvasAgentResults(f.params)).rejects.toThrow();
|
||
});
|
||
test("non-images are rejected before any vision model invocation", async () => {
|
||
const f = await fixture('{"passed":true,"issues":[]}');
|
||
f.params.reviewPorts.readImage = async () => Buffer.from("not an image");
|
||
await expect(verifyCanvasAgentResults(f.params)).rejects.toThrow();
|
||
expect(f.requests).toHaveLength(0);
|
||
});
|
||
test("missing output counts cannot be disguised by the model's plan", async () => {
|
||
const f = await fixture('{"passed":true,"issues":[]}');
|
||
f.params.snapshot.requirements!.expectedImageCount = 6;
|
||
const result = await verifyCanvasAgentResults(f.params);
|
||
expect(result.passed).toBe(false);
|
||
expect(result.issues.join(" ")).toContain("6 张");
|
||
expect(f.requests).toHaveLength(0);
|
||
});
|
||
test("a text-only model cannot silently approve images", async () => {
|
||
const f = await fixture('{"passed":true,"issues":[]}');
|
||
await expect(
|
||
analyzeRuntimeImages({
|
||
...f.params,
|
||
supportsVision: false,
|
||
nodeIds: ["result"],
|
||
question: "检查",
|
||
}),
|
||
).rejects.toThrow("不支持图片检查");
|
||
});
|