ai-chatbot/tests/routes/canvas-agent-runtime-review.test.ts

import { expect, test } from "@playwright/test";
import type { LanguageModelV2 } from "@ai-sdk/provider";
import sharp from "sharp";
import { newAgentRunSnapshot } from "@/lib/ai/canvas-agent-runtime/protocol";
import {
  analyzeRuntimeImages,
  inspectImageAlpha,
  verifyCanvasAgentResults,
} from "@/lib/ai/canvas-agent-runtime/review";
import { isCompletedSimpleCutout } from "@/lib/ai/canvas-agent-runtime/cutout-completion";
import { createAgentVisionExecutor } from "@/lib/ai/runtime/agent-vision-failover";
import type { AgentModelEntry } from "@/lib/ai/runtime/agent-model-failover";

async function fixture(answer: string, resultSize = { width: 30, height: 40 }) {
  const requests: Array<Parameters<LanguageModelV2["doGenerate"]>[0]> = [];
  const model: LanguageModelV2 = {
    specificationVersion: "v2",
    provider: "test",
    modelId: "vision",
    supportedUrls: {},
    doGenerate: async (request) => {
      requests.push(request);
      return {
        content: [{ type: "text", text: answer }],
        finishReason: "stop",
        usage: { inputTokens: 30, outputTokens: 10, totalTokens: 40 },
        warnings: [],
      };
    },
    doStream: async () => {
      throw new Error("Unexpected stream");
    },
  };
  const buffers = {
    source: await sharp({ create: { width: 40, height: 40, channels: 3, background: "white" } })
      .png()
      .toBuffer(),
    result: await sharp({ create: { ...resultSize, channels: 3, background: "white" } })
      .png()
      .toBuffer(),
  };
  const snapshot = newAgentRunSnapshot("背景改白色，主体结构不变，生成3:4", []);
  snapshot.requirements = { expectedImageCount: 1, aspectRatio: "3:4" };
  snapshot.resultNodeIds = ["result"];
  snapshot.calls = [
    {
      id: "plan",
      name: "planCanvasTask",
      input: {},
      status: "succeeded",
      output: { expectedImageCount: 1, visualCriteria: ["主体结构不变"] },
    },
    {
      id: "gen",
      name: "generateCanvasImages",
      input: { referenceNodeIds: ["source"] },
      status: "succeeded",
      output: { status: "succeeded", createdNodeIds: ["result"] },
    },
  ];
  const params = {
    snapshot,
    model,
    userId: "test",
    projectId: "test",
    supportsVision: true,
    reviewPorts: {
      loadNodes: async (ids: string[]) =>
        ids.map((id) => ({
          id,
          type: "image",
          position: { x: 0, y: 0 },
          data: { src: `https://assets.invalid/${id}` },
        })),
      readImage: async (url: string) => (url.endsWith("source") ? buffers.source : buffers.result),
    },
  };
  return { params, requests };
}
test("checks real image bytes and compares results to their source images", async () => {
  const f = await fixture('{"passed":true,"issues":[],"failedNodeIds":[]}');
  const result = await verifyCanvasAgentResults(f.params);
  expect(result.passed).toBe(true);
  expect(f.requests).toHaveLength(1);
  expect(f.requests[0].responseFormat?.type).toBe("json");
  expect(f.requests[0].providerOptions?.openai).toEqual({ structuredOutputs: false });
  const content = f.requests[0].prompt.flatMap((message) =>
    Array.isArray(message.content) ? message.content : [],
  );
  expect(
    content.filter((part) => part.type === "file" && part.mediaType.startsWith("image/")),
  ).toHaveLength(2);
  // A square source image is not incorrectly tested against the output's 3:4 ratio.
  expect(result.issues).toEqual([]);
});

test("malformed vision JSON switches models, while a valid failed review never triggers failover", async () => {
  const bad = await fixture("not JSON");
  const validNegative = await fixture('{"passed":false,"issues":["主体不一致"],"failedNodeIds":["result"]}');
  const failures: string[] = [];
  const entries = ["bad", "valid-negative"].map(id => ({ model: { id }, provider: {
    type: "custom", baseUrl: "https://fixture.invalid/v1",
  } }) as AgentModelEntry);
  const withModel = createAgentVisionExecutor(entries, {
    isPaused: async () => false,
    fail: async entry => { failures.push(entry.model.id); },
  }, async id => id === "bad" ? bad.params.model : validNegative.params.model);
  const result = await verifyCanvasAgentResults({ ...validNegative.params, withModel });
  expect(result.passed).toBe(false);
  expect(result.issues).toEqual(["主体不一致"]);
  expect(failures).toEqual(["bad"]);
  expect(bad.requests).toHaveLength(1);
  expect(validNegative.requests).toHaveLength(1);
});
test("actual dimensions override a model's claim that the output passed", async () => {
  const f = await fixture('{"passed":true,"issues":[]}', { width: 60, height: 40 });
  const result = await verifyCanvasAgentResults(f.params);
  expect(result.passed).toBe(false);
  expect(result.failedNodeIds).toEqual(["result"]);
  expect(result.issues.join(" ")).toContain("3:4");
});
test("a filtered structured review is a policy rejection, never a reason to switch providers", async () => {
  const f = await fixture("");
  const model = f.params.model;
  const generate = model.doGenerate;
  model.doGenerate = async options => ({ ...await generate(options), finishReason: "content-filter" });
  const failures: string[] = [];
  const entries = ["primary", "backup"].map(id => ({ model: { id }, provider: {
    type: "custom", baseUrl: "https://fixture.invalid/v1",
  } }) as AgentModelEntry);
  const withModel = createAgentVisionExecutor(entries, {
    isPaused: async () => false,
    fail: async entry => { failures.push(entry.model.id); },
  }, async id => { expect(id).toBe("primary"); return model; });
  await expect(analyzeRuntimeImages({ ...f.params, withModel, nodeIds: ["result"], question: "检查" }))
    .rejects.toThrow("content policy rejection");
  expect(failures).toEqual([]);
  expect(f.requests).toHaveLength(1);
});
test("an unsolicited download cannot block verified image results; requested downloads and real editor panels still wait", async () => {
  const f = await fixture('{"passed":true,"issues":[]}');
  f.params.snapshot.calls.push({ id: "download", name: "image_download", input: {}, status: "succeeded", output: { mode: "interactive" } });
  expect((await verifyCanvasAgentResults(f.params)).passed).toBe(true);
  f.params.snapshot.inputs.push({ id: "no-download", text: "只核验现有结果，不要下载，不要重新生成", applied: true });
  expect((await verifyCanvasAgentResults(f.params)).passed).toBe(true);
  f.params.snapshot.objective += "并下载到电脑";
  expect((await verifyCanvasAgentResults(f.params)).passed).toBe(false);
  f.params.snapshot.objective = "修改图片";
  f.params.snapshot.calls.at(-1)!.name = "image_crop";
  expect((await verifyCanvasAgentResults(f.params)).passed).toBe(false);
});
test("reported problems cannot be ignored by a contradictory passed=true flag", async () => {
  const f = await fixture('{"passed":true,"issues":["主体细节已改变"],"failedNodeIds":["result"]}');
  const result = await verifyCanvasAgentResults(f.params);
  expect(result.passed).toBe(false);
  expect(result.retryable).toBe(true);
});
test("review infrastructure failure never becomes an automatic pass", async () => {
  const f = await fixture("<html>unavailable</html>");
  await expect(verifyCanvasAgentResults(f.params)).rejects.toThrow();
});
test("non-images are rejected before any vision model invocation", async () => {
  const f = await fixture('{"passed":true,"issues":[]}');
  f.params.reviewPorts.readImage = async () => Buffer.from("not an image");
  await expect(verifyCanvasAgentResults(f.params)).rejects.toThrow();
  expect(f.requests).toHaveLength(0);
});
test("missing output counts cannot be disguised by the model's plan", async () => {
  const f = await fixture('{"passed":true,"issues":[]}');
  f.params.snapshot.requirements!.expectedImageCount = 6;
  const result = await verifyCanvasAgentResults(f.params);
  expect(result.passed).toBe(false);
  expect(result.issues.join(" ")).toContain("6 张");
  expect(f.requests).toHaveLength(0);
});
test("a text-only model cannot silently approve images", async () => {
  const f = await fixture('{"passed":true,"issues":[]}');
  await expect(
    analyzeRuntimeImages({
      ...f.params,
      supportsVision: false,
      nodeIds: ["result"],
      question: "检查",
    }),
  ).rejects.toThrow("不支持图片检查");
});

async function cutoutFixture() {
  const f = await fixture('{"passed":false,"issues":["背景是黑色"],"failedNodeIds":["result"]}');
  f.params.snapshot.objective = "帮我将它扣出来保持透明背景";
  f.params.snapshot.requirements = { expectedImageCount: 1 };
  f.params.snapshot.calls[1].name = "removeBackground";
  f.params.snapshot.calls[1].input = { nodeIds: ["source"] };
  const subject = await sharp({ create: { width: 16, height: 16, channels: 4, background: "blue" } }).png().toBuffer();
  const bytes = await sharp({ create: { width: 32, height: 32, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } } })
    .composite([{ input: subject, left: 8, top: 8 }]).png().toBuffer();
  f.params.reviewPorts.readImage = async () => bytes;
  return { ...f, bytes };
}

test("a completed add-to-conversation command does not keep a generated image waiting for an editor", async () => {
  const f = await fixture('{"passed":true,"issues":[],"failedNodeIds":[]}');
  f.params.snapshot.calls.push({ id: "reference", name: "image_add_to_chat", input: {}, status: "succeeded", output: { mode: "interactive", status: "succeeded" } });
  const result = await verifyCanvasAgentResults(f.params);
  expect(result.passed).toBe(true);
});

test("a real transparent cutout finishes from original file evidence without a vision request", async () => {
  const f = await cutoutFixture();
  expect(isCompletedSimpleCutout(f.params.snapshot)).toBe(true);
  const result = await verifyCanvasAgentResults(f.params);
  expect(result).toMatchObject({ passed: true, method: "alpha-channel", retryable: false });
  expect(f.requests).toHaveLength(0);
});

test("opaque images and fully empty transparent images cannot complete a cutout", async () => {
  const f = await cutoutFixture();
  for (const background of ["white", { r: 0, g: 0, b: 0, alpha: 0 }]) {
    const bytes = await sharp({ create: { width: 32, height: 32, channels: 4, background } }).png().toBuffer();
    f.params.reviewPorts.readImage = async () => bytes;
    expect(await verifyCanvasAgentResults(f.params)).toMatchObject({ passed: false, failedNodeIds: ["result"], retryable: false });
  }
  expect(f.requests).toHaveLength(0);
});

test("cutout completion does not skip later user actions, edge inspection or an incomplete batch", async () => {
  const f = await cutoutFixture();
  for (const request of ["抠图然后下载", "抠图后换背景生成三张", "抠图并检查边缘白边", "抠图后把毛改成绿色", "抠图后移动到左侧", "先重命名为小鸡再抠图", "抠图再做点别的", "remove background and recolor blue"]) {
    f.params.snapshot.objective = request;
    expect(isCompletedSimpleCutout(f.params.snapshot)).toBe(false);
  }
  f.params.snapshot.objective = "抠图";
  f.params.snapshot.requirements!.expectedImageCount = 2;
  expect(isCompletedSimpleCutout(f.params.snapshot)).toBe(false);
});

test("vision inspection receives alpha facts and an opaque checkerboard preview", async () => {
  const f = await cutoutFixture();
  await analyzeRuntimeImages({ ...f.params, nodeIds: ["result"], question: "检查主体边缘" });
  const content = f.requests[0].prompt.flatMap(message => Array.isArray(message.content) ? message.content : []);
  expect(JSON.stringify(content.filter(part => part.type === "text"))).toContain('hasTransparentPixels');
  const image = content.find(part => part.type === "file") as { data: Uint8Array };
  const facts = await inspectImageAlpha(Buffer.from(image.data));
  expect(facts.hasTransparentPixels).toBe(false);
  expect(facts.hasVisiblePixels).toBe(true);
});