import { expect, test } from "@playwright/test"; import type { LanguageModelV2 } from "@ai-sdk/provider"; import sharp from "sharp"; import { newAgentRunSnapshot } from "@/lib/ai/canvas-agent-runtime/protocol"; import { analyzeRuntimeImages, verifyCanvasAgentResults, } from "@/lib/ai/canvas-agent-runtime/review"; async function fixture(answer: string, resultSize = { width: 30, height: 40 }) { const requests: Array[0]> = []; const model: LanguageModelV2 = { specificationVersion: "v2", provider: "test", modelId: "vision", supportedUrls: {}, doGenerate: async (request) => { requests.push(request); return { content: [{ type: "text", text: answer }], finishReason: "stop", usage: { inputTokens: 30, outputTokens: 10, totalTokens: 40 }, warnings: [], }; }, doStream: async () => { throw new Error("Unexpected stream"); }, }; const buffers = { source: await sharp({ create: { width: 40, height: 40, channels: 3, background: "white" } }) .png() .toBuffer(), result: await sharp({ create: { ...resultSize, channels: 3, background: "white" } }) .png() .toBuffer(), }; const snapshot = newAgentRunSnapshot("背景改白色,主体结构不变,生成3:4", []); snapshot.requirements = { expectedImageCount: 1, aspectRatio: "3:4" }; snapshot.resultNodeIds = ["result"]; snapshot.calls = [ { id: "plan", name: "planCanvasTask", input: {}, status: "succeeded", output: { expectedImageCount: 1, visualCriteria: ["主体结构不变"] }, }, { id: "gen", name: "generateCanvasImages", input: { referenceNodeIds: ["source"] }, status: "succeeded", output: { status: "succeeded", createdNodeIds: ["result"] }, }, ]; const params = { snapshot, model, userId: "test", projectId: "test", supportsVision: true, reviewPorts: { loadNodes: async (ids: string[]) => ids.map((id) => ({ id, type: "image", position: { x: 0, y: 0 }, data: { src: `https://assets.invalid/${id}` }, })), readImage: async (url: string) => (url.endsWith("source") ? buffers.source : buffers.result), }, }; return { params, requests }; } test("checks real image bytes and compares results to their source images", async () => { const f = await fixture('{"passed":true,"issues":[],"failedNodeIds":[]}'); const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(true); expect(f.requests).toHaveLength(1); const content = f.requests[0].prompt.flatMap((message) => Array.isArray(message.content) ? message.content : [], ); expect( content.filter((part) => part.type === "file" && part.mediaType.startsWith("image/")), ).toHaveLength(2); // A square source image is not incorrectly tested against the output's 3:4 ratio. expect(result.issues).toEqual([]); }); test("actual dimensions override a model's claim that the output passed", async () => { const f = await fixture('{"passed":true,"issues":[]}', { width: 60, height: 40 }); const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(false); expect(result.failedNodeIds).toEqual(["result"]); expect(result.issues.join(" ")).toContain("3:4"); }); test("reported problems cannot be ignored by a contradictory passed=true flag", async () => { const f = await fixture('{"passed":true,"issues":["主体细节已改变"],"failedNodeIds":["result"]}'); const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(false); expect(result.retryable).toBe(true); }); test("review infrastructure failure never becomes an automatic pass", async () => { const f = await fixture("unavailable"); await expect(verifyCanvasAgentResults(f.params)).rejects.toThrow(); }); test("non-images are rejected before any vision model invocation", async () => { const f = await fixture('{"passed":true,"issues":[]}'); f.params.reviewPorts.readImage = async () => Buffer.from("not an image"); await expect(verifyCanvasAgentResults(f.params)).rejects.toThrow(); expect(f.requests).toHaveLength(0); }); test("missing output counts cannot be disguised by the model's plan", async () => { const f = await fixture('{"passed":true,"issues":[]}'); f.params.snapshot.requirements!.expectedImageCount = 6; const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(false); expect(result.issues.join(" ")).toContain("6 张"); expect(f.requests).toHaveLength(0); }); test("a text-only model cannot silently approve images", async () => { const f = await fixture('{"passed":true,"issues":[]}'); await expect( analyzeRuntimeImages({ ...f.params, supportsVision: false, nodeIds: ["result"], question: "检查", }), ).rejects.toThrow("不支持图片检查"); });