import { expect, test } from "@playwright/test"; import type { LanguageModelV2 } from "@ai-sdk/provider"; import sharp from "sharp"; import { newAgentRunSnapshot } from "@/lib/ai/canvas-agent-runtime/protocol"; import { analyzeRuntimeImages, inspectImageAlpha, verifyCanvasAgentResults, } from "@/lib/ai/canvas-agent-runtime/review"; import { isCompletedSimpleCutout } from "@/lib/ai/canvas-agent-runtime/cutout-completion"; import { createAgentVisionExecutor } from "@/lib/ai/runtime/agent-vision-failover"; import type { AgentModelEntry } from "@/lib/ai/runtime/agent-model-failover"; async function fixture(answer: string, resultSize = { width: 30, height: 40 }) { const requests: Array[0]> = []; const model: LanguageModelV2 = { specificationVersion: "v2", provider: "test", modelId: "vision", supportedUrls: {}, doGenerate: async (request) => { requests.push(request); return { content: [{ type: "text", text: answer }], finishReason: "stop", usage: { inputTokens: 30, outputTokens: 10, totalTokens: 40 }, warnings: [], }; }, doStream: async () => { throw new Error("Unexpected stream"); }, }; const buffers = { source: await sharp({ create: { width: 40, height: 40, channels: 3, background: "white" } }) .png() .toBuffer(), result: await sharp({ create: { ...resultSize, channels: 3, background: "white" } }) .png() .toBuffer(), }; const snapshot = newAgentRunSnapshot("背景改白色,主体结构不变,生成3:4", []); snapshot.requirements = { expectedImageCount: 1, aspectRatio: "3:4" }; snapshot.resultNodeIds = ["result"]; snapshot.calls = [ { id: "plan", name: "planCanvasTask", input: {}, status: "succeeded", output: { expectedImageCount: 1, visualCriteria: ["主体结构不变"] }, }, { id: "gen", name: "generateCanvasImages", input: { referenceNodeIds: ["source"] }, status: "succeeded", output: { status: "succeeded", createdNodeIds: ["result"] }, }, ]; const params = { snapshot, model, userId: "test", projectId: "test", supportsVision: true, reviewPorts: { loadNodes: async (ids: string[]) => ids.map((id) => ({ id, type: "image", position: { x: 0, y: 0 }, data: { src: `https://assets.invalid/${id}` }, })), readImage: async (url: string) => (url.endsWith("source") ? buffers.source : buffers.result), }, }; return { params, requests }; } test("checks real image bytes and compares results to their source images", async () => { const f = await fixture('{"passed":true,"issues":[],"failedNodeIds":[]}'); const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(true); expect(f.requests).toHaveLength(1); expect(f.requests[0].responseFormat?.type).toBe("json"); expect(f.requests[0].providerOptions?.openai).toEqual({ structuredOutputs: false }); const content = f.requests[0].prompt.flatMap((message) => Array.isArray(message.content) ? message.content : [], ); expect( content.filter((part) => part.type === "file" && part.mediaType.startsWith("image/")), ).toHaveLength(2); // A square source image is not incorrectly tested against the output's 3:4 ratio. expect(result.issues).toEqual([]); }); test("malformed vision JSON switches models, while a valid failed review never triggers failover", async () => { const bad = await fixture("not JSON"); const validNegative = await fixture('{"passed":false,"issues":["主体不一致"],"failedNodeIds":["result"]}'); const failures: string[] = []; const entries = ["bad", "valid-negative"].map(id => ({ model: { id }, provider: { type: "custom", baseUrl: "https://fixture.invalid/v1", } }) as AgentModelEntry); const withModel = createAgentVisionExecutor(entries, { isPaused: async () => false, fail: async entry => { failures.push(entry.model.id); }, }, async id => id === "bad" ? bad.params.model : validNegative.params.model); const result = await verifyCanvasAgentResults({ ...validNegative.params, withModel }); expect(result.passed).toBe(false); expect(result.issues).toEqual(["主体不一致"]); expect(failures).toEqual(["bad"]); expect(bad.requests).toHaveLength(1); expect(validNegative.requests).toHaveLength(1); }); test("actual dimensions override a model's claim that the output passed", async () => { const f = await fixture('{"passed":true,"issues":[]}', { width: 60, height: 40 }); const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(false); expect(result.failedNodeIds).toEqual(["result"]); expect(result.issues.join(" ")).toContain("3:4"); }); test("a filtered structured review is a policy rejection, never a reason to switch providers", async () => { const f = await fixture(""); const model = f.params.model; const generate = model.doGenerate; model.doGenerate = async options => ({ ...await generate(options), finishReason: "content-filter" }); const failures: string[] = []; const entries = ["primary", "backup"].map(id => ({ model: { id }, provider: { type: "custom", baseUrl: "https://fixture.invalid/v1", } }) as AgentModelEntry); const withModel = createAgentVisionExecutor(entries, { isPaused: async () => false, fail: async entry => { failures.push(entry.model.id); }, }, async id => { expect(id).toBe("primary"); return model; }); await expect(analyzeRuntimeImages({ ...f.params, withModel, nodeIds: ["result"], question: "检查" })) .rejects.toThrow("content policy rejection"); expect(failures).toEqual([]); expect(f.requests).toHaveLength(1); }); test("an unsolicited download cannot block verified image results; requested downloads and real editor panels still wait", async () => { const f = await fixture('{"passed":true,"issues":[]}'); f.params.snapshot.calls.push({ id: "download", name: "image_download", input: {}, status: "succeeded", output: { mode: "interactive" } }); expect((await verifyCanvasAgentResults(f.params)).passed).toBe(true); f.params.snapshot.inputs.push({ id: "no-download", text: "只核验现有结果,不要下载,不要重新生成", applied: true }); expect((await verifyCanvasAgentResults(f.params)).passed).toBe(true); f.params.snapshot.objective += "并下载到电脑"; expect((await verifyCanvasAgentResults(f.params)).passed).toBe(false); f.params.snapshot.objective = "修改图片"; f.params.snapshot.calls.at(-1)!.name = "image_crop"; expect((await verifyCanvasAgentResults(f.params)).passed).toBe(false); }); test("reported problems cannot be ignored by a contradictory passed=true flag", async () => { const f = await fixture('{"passed":true,"issues":["主体细节已改变"],"failedNodeIds":["result"]}'); const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(false); expect(result.retryable).toBe(true); }); test("review infrastructure failure never becomes an automatic pass", async () => { const f = await fixture("unavailable"); await expect(verifyCanvasAgentResults(f.params)).rejects.toThrow(); }); test("non-images are rejected before any vision model invocation", async () => { const f = await fixture('{"passed":true,"issues":[]}'); f.params.reviewPorts.readImage = async () => Buffer.from("not an image"); await expect(verifyCanvasAgentResults(f.params)).rejects.toThrow(); expect(f.requests).toHaveLength(0); }); test("missing output counts cannot be disguised by the model's plan", async () => { const f = await fixture('{"passed":true,"issues":[]}'); f.params.snapshot.requirements!.expectedImageCount = 6; const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(false); expect(result.issues.join(" ")).toContain("6 张"); expect(f.requests).toHaveLength(0); }); test("a text-only model cannot silently approve images", async () => { const f = await fixture('{"passed":true,"issues":[]}'); await expect( analyzeRuntimeImages({ ...f.params, supportsVision: false, nodeIds: ["result"], question: "检查", }), ).rejects.toThrow("不支持图片检查"); }); async function cutoutFixture() { const f = await fixture('{"passed":false,"issues":["背景是黑色"],"failedNodeIds":["result"]}'); f.params.snapshot.objective = "帮我将它扣出来保持透明背景"; f.params.snapshot.requirements = { expectedImageCount: 1 }; f.params.snapshot.calls[1].name = "removeBackground"; f.params.snapshot.calls[1].input = { nodeIds: ["source"] }; const subject = await sharp({ create: { width: 16, height: 16, channels: 4, background: "blue" } }).png().toBuffer(); const bytes = await sharp({ create: { width: 32, height: 32, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } } }) .composite([{ input: subject, left: 8, top: 8 }]).png().toBuffer(); f.params.reviewPorts.readImage = async () => bytes; return { ...f, bytes }; } test("a completed add-to-conversation command does not keep a generated image waiting for an editor", async () => { const f = await fixture('{"passed":true,"issues":[],"failedNodeIds":[]}'); f.params.snapshot.calls.push({ id: "reference", name: "image_add_to_chat", input: {}, status: "succeeded", output: { mode: "interactive", status: "succeeded" } }); const result = await verifyCanvasAgentResults(f.params); expect(result.passed).toBe(true); }); test("a real transparent cutout finishes from original file evidence without a vision request", async () => { const f = await cutoutFixture(); expect(isCompletedSimpleCutout(f.params.snapshot)).toBe(true); const result = await verifyCanvasAgentResults(f.params); expect(result).toMatchObject({ passed: true, method: "alpha-channel", retryable: false }); expect(f.requests).toHaveLength(0); }); test("opaque images and fully empty transparent images cannot complete a cutout", async () => { const f = await cutoutFixture(); for (const background of ["white", { r: 0, g: 0, b: 0, alpha: 0 }]) { const bytes = await sharp({ create: { width: 32, height: 32, channels: 4, background } }).png().toBuffer(); f.params.reviewPorts.readImage = async () => bytes; expect(await verifyCanvasAgentResults(f.params)).toMatchObject({ passed: false, failedNodeIds: ["result"], retryable: false }); } expect(f.requests).toHaveLength(0); }); test("cutout completion does not skip later user actions, edge inspection or an incomplete batch", async () => { const f = await cutoutFixture(); for (const request of ["抠图然后下载", "抠图后换背景生成三张", "抠图并检查边缘白边", "抠图后把毛改成绿色", "抠图后移动到左侧", "先重命名为小鸡再抠图", "抠图再做点别的", "remove background and recolor blue"]) { f.params.snapshot.objective = request; expect(isCompletedSimpleCutout(f.params.snapshot)).toBe(false); } f.params.snapshot.objective = "抠图"; f.params.snapshot.requirements!.expectedImageCount = 2; expect(isCompletedSimpleCutout(f.params.snapshot)).toBe(false); }); test("vision inspection receives alpha facts and an opaque checkerboard preview", async () => { const f = await cutoutFixture(); await analyzeRuntimeImages({ ...f.params, nodeIds: ["result"], question: "检查主体边缘" }); const content = f.requests[0].prompt.flatMap(message => Array.isArray(message.content) ? message.content : []); expect(JSON.stringify(content.filter(part => part.type === "text"))).toContain('hasTransparentPixels'); const image = content.find(part => part.type === "file") as { data: Uint8Array }; const facts = await inspectImageAlpha(Buffer.from(image.data)); expect(facts.hasTransparentPixels).toBe(false); expect(facts.hasVisiblePixels).toBe(true); });