ai-chatbot/scripts/test-vip98k-capability.ts

import { createHash } from "node:crypto";
import { appendFile, mkdir, readFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import dotenv from "dotenv";
import sharp from "sharp";

dotenv.config({ path: ".env.local" });
dotenv.config();

const VIP98K_BASE_URL = "https://api.vip98k.com";

type Vip98kCatalogModel = {
  id?: string;
  modelId?: string;
  model_id?: string;
  slug?: string;
  name?: string;
};

const CAPABILITIES = [
  "chat",
  "multimodal",
  "image",
  "multi-image-edit",
  "rmbg",
  "enhance",
  "extend",
  "video",
  "world",
  "text-to-3d",
  "image-to-3d",
] as const;

type Capability = (typeof CAPABILITIES)[number];
type LedgerEntry = {
  runId: string;
  capability: Capability;
  modelId: string;
  status: "started" | "success" | "failure" | "skipped";
  startedAt: string;
  finishedAt?: string;
  durationMs?: number;
  outputCount?: number;
  outputDigest?: string;
  error?: string;
};

const DEFAULT_MODEL_IDS: Record<Capability, string[]> = {
  chat: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"],
  multimodal: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"],
  image: ["nano-banana-2", "nano-banana-pro", "gpt-image-2-vip"],
  "multi-image-edit": ["nano-banana-2", "nano-banana-pro", "gpt-image-2-vip"],
  rmbg: ["remove-background"],
  enhance: ["enhance", "image-upscaler"],
  extend: ["image-expansion"],
  video: [
    "bytedance/seedance-v1-pro-fast/text-to-video",
    "veo-3.1-fast",
    "google/veo3.1-fast/text-to-video",
  ],
  world: ["world-generation"],
  "text-to-3d": [
    "tencent/hunyuan3d-pro/text-to-3d",
    "tencent/hunyuan3d-rapid/text-to-3d",
  ],
  "image-to-3d": [
    "tencent/hunyuan3d-pro/image-to-3d",
    "tencent/hunyuan3d-rapid/image-to-3d",
  ],
};

function getArgValue(name: string) {
  const prefix = `${name}=`;
  return process.argv.find((item) => item.startsWith(prefix))?.slice(prefix.length) || "";
}

function hasFlag(name: string) {
  return process.argv.includes(name);
}

function getModelId(model: Vip98kCatalogModel) {
  return String(model.id || model.modelId || model.model_id || model.slug || model.name || "").trim();
}

function normalizeVip98kBaseUrl(baseUrl?: string | null) {
  const normalized = String(baseUrl || VIP98K_BASE_URL).trim().replace(/\/+$/, "");
  if (!normalized) return VIP98K_BASE_URL;
  return normalized.replace(/\/v1$/i, "").replace(/\/api\/external\/task$/i, "");
}

async function fetchVip98kCatalog(params: { baseURL: string; apiKey: string }) {
  const payload = await parseResponse(
    await fetch(`${normalizeVip98kBaseUrl(params.baseURL)}/api/external/task/models`, {
      headers: {
        Authorization: `Bearer ${params.apiKey}`,
        Accept: "application/json",
      },
      signal: AbortSignal.timeout(60_000),
    }),
  );
  const record = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
  const data = record.data;
  const models =
    (Array.isArray(data?.models) && data.models) ||
    (Array.isArray(data?.items) && data.items) ||
    (Array.isArray(data) && data) ||
    (Array.isArray(record.models) && record.models) ||
    [];
  if (models.length === 0) throw new Error("VIP98K model catalog is empty");
  return models as Vip98kCatalogModel[];
}

function sanitizeError(error: unknown) {
  const message = error instanceof Error ? error.message : String(error);
  return message.replace(/Bearer\s+\S+/gi, "Bearer [REDACTED]").slice(0, 2000);
}

function digestOutputs(outputs: string[]) {
  return createHash("sha256").update(outputs.join("\n")).digest("hex");
}

async function readLedger(path: string): Promise<LedgerEntry[]> {
  const content = await readFile(path, "utf8").catch(() => "");
  return content
    .split(/\r?\n/)
    .filter(Boolean)
    .flatMap((line) => {
      try {
        return [JSON.parse(line) as LedgerEntry];
      } catch {
        return [];
      }
    });
}

async function writeLedger(path: string, entry: LedgerEntry) {
  await mkdir(dirname(path), { recursive: true });
  await appendFile(path, `${JSON.stringify(entry)}\n`, "utf8");
}

async function parseResponse(response: Response) {
  const text = await response.text();
  let payload: unknown = text;
  try {
    payload = text ? JSON.parse(text) : null;
  } catch {
    // Keep the response body for a useful provider error.
  }
  if (!response.ok) {
    const record = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
    const message =
      record.error?.message || record.error || record.message || text || `HTTP ${response.status}`;
    throw new Error(`${String(message)} (HTTP ${response.status})`);
  }
  return payload;
}

function collectUrls(value: unknown, urls = new Set<string>()) {
  if (!value) return urls;
  if (typeof value === "string") {
    for (const match of value.match(/https?:\/\/[^\s"'<>()[\]{}]+/g) || []) {
      urls.add(match.replace(/[),.;]+$/g, ""));
    }
    if (/^data:[^;]+;base64,/i.test(value)) urls.add(value);
    return urls;
  }
  if (Array.isArray(value)) {
    value.forEach((item) => collectUrls(item, urls));
    return urls;
  }
  if (typeof value === "object") {
    Object.values(value as Record<string, unknown>).forEach((item) => collectUrls(item, urls));
  }
  return urls;
}

function extractTaskId(payload: unknown) {
  const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
  const data = root.data && typeof root.data === "object" ? root.data : root;
  return String(data.taskId || data.task_id || data.id || data.requestId || "").trim();
}

function extractResultPath(payload: unknown) {
  const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
  const data = root.data && typeof root.data === "object" ? root.data : root;
  return String(data.resultPath || data.result_path || "").trim();
}

function extractStatus(payload: unknown) {
  const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
  const data = root.data && typeof root.data === "object" ? root.data : root;
  return String(data.status || data.state || data.taskStatus || "").trim().toLowerCase();
}

async function makeReferencePng(color: string, label: string) {
  return sharp({
    create: { width: 96, height: 96, channels: 4, background: color },
  })
    .composite([
      {
        input: Buffer.from(
          `<svg width="96" height="96"><text x="48" y="58" text-anchor="middle" font-size="30" fill="white">${label}</text></svg>`,
        ),
      },
    ])
    .png()
    .toBuffer();
}

async function runOpenAIImage(params: {
  baseUrl: string;
  apiKey: string;
  modelId: string;
  multiImage: boolean;
}) {
  const endpoint = `${params.baseUrl}/v1/images/${params.multiImage ? "edits" : "generations"}`;
  if (!params.multiImage) {
    const payload = await parseResponse(
      await fetch(endpoint, {
        method: "POST",
        headers: {
          Authorization: `Bearer ${params.apiKey}`,
          "Content-Type": "application/json",
        },
        body: JSON.stringify({
          model: params.modelId,
          prompt: "A small blue ceramic cup on a white studio background",
          size: "1024x1024",
          response_format: "url",
        }),
        signal: AbortSignal.timeout(900_000),
      }),
    );
    const outputs = Array.from(collectUrls(payload));
    if (outputs.length === 0) throw new Error("Image generation returned no image output");
    return outputs;
  }

  const [left, right] = await Promise.all([
    makeReferencePng("#2563eb", "A"),
    makeReferencePng("#dc2626", "B"),
  ]);
  const form = new FormData();
  form.append("model", params.modelId);
  form.append("prompt", "Combine both references into one clean side-by-side product comparison image");
  form.append("size", "1024x1024");
  form.append("response_format", "url");
  form.append("image[]", new Blob([left], { type: "image/png" }), "reference-a.png");
  form.append("image[]", new Blob([right], { type: "image/png" }), "reference-b.png");
  const payload = await parseResponse(
    await fetch(endpoint, {
      method: "POST",
      headers: { Authorization: `Bearer ${params.apiKey}` },
      body: form,
      signal: AbortSignal.timeout(900_000),
    }),
  );
  const outputs = Array.from(collectUrls(payload));
  if (outputs.length === 0) throw new Error("Multi-image edit returned no image output");
  return outputs;
}

async function runChat(params: {
  baseUrl: string;
  apiKey: string;
  modelId: string;
  multimodal: boolean;
}) {
  const content = params.multimodal
    ? [
        { type: "text", text: "Reply with the dominant color in one English word." },
        {
          type: "image_url",
          image_url: { url: "https://dummyimage.com/96x96/2563eb/ffffff.png&text=A" },
        },
      ]
    : "Reply with exactly: vip98k-ok";
  const payload = await parseResponse(
    await fetch(`${params.baseUrl}/v1/chat/completions`, {
      method: "POST",
      headers: {
        Authorization: `Bearer ${params.apiKey}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify({
        model: params.modelId,
        messages: [{ role: "user", content }],
        max_tokens: 64,
      }),
      signal: AbortSignal.timeout(180_000),
    }),
  );
  const text = JSON.stringify(payload);
  if (!text || text === "{}") throw new Error("Chat request returned no content");
  return [text.slice(0, 500)];
}

async function runExternalTask(params: {
  baseUrl: string;
  apiKey: string;
  modelId: string;
  capability: Capability;
}) {
  const imageUrl = "https://dummyimage.com/512x512/2563eb/ffffff.png&text=VIP98K";
  const body: Record<string, unknown> = {
    model: params.modelId,
    displayName: `provider-smoke-${params.capability}`,
    prompt: "Create a minimal test result with one simple object",
  };
  if (["rmbg", "enhance", "extend", "image-to-3d"].includes(params.capability)) {
    Object.assign(body, {
      image: imageUrl,
      imageUrl,
      image_url: imageUrl,
      images: [imageUrl],
      imageUrls: [imageUrl],
    });
  }
  if (params.capability === "extend") {
    Object.assign(body, { aspectRatio: "16:9", aspect_ratio: "16:9", ratio: "16:9" });
  }
  if (params.capability === "video") {
    Object.assign(body, { mode: "text-to-video", duration: 4, resolution: "480p", ratio: "16:9" });
  }
  if (params.capability === "world") body.mode = "world-generation";

  const submitted = await parseResponse(
    await fetch(`${params.baseUrl}/api/external/task/generate`, {
      method: "POST",
      headers: {
        Authorization: `Bearer ${params.apiKey}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify(body),
      signal: AbortSignal.timeout(180_000),
    }),
  );
  const immediate = Array.from(collectUrls(submitted));
  if (immediate.length > 0) return immediate;
  const taskId = extractTaskId(submitted);
  if (!taskId) throw new Error("External task returned neither output nor task ID");
  const resultPath = extractResultPath(submitted);
  const startedAt = Date.now();
  while (Date.now() - startedAt < 1_200_000) {
    await new Promise((resolvePromise) => setTimeout(resolvePromise, 5000));
    const resultUrl = resultPath
      ? resultPath.startsWith("http")
        ? resultPath
        : `${params.baseUrl}${resultPath.startsWith("/") ? "" : "/"}${resultPath}`
      : `${params.baseUrl}/api/external/task/result?taskId=${encodeURIComponent(taskId)}`;
    const payload = await parseResponse(
      await fetch(resultUrl, {
        headers: { Authorization: `Bearer ${params.apiKey}` },
        signal: AbortSignal.timeout(60_000),
      }),
    );
    const status = extractStatus(payload);
    const outputs = Array.from(collectUrls(payload));
    if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) {
      throw new Error(`External task failed with status ${status}`);
    }
    if (
      ["succeeded", "success", "completed", "done", "finished"].includes(status) ||
      outputs.length > 0
    ) {
      if (outputs.length === 0) throw new Error("External task completed without output URL");
      return outputs;
    }
  }
  throw new Error("External task did not complete within 20 minutes");
}

async function main() {
  const capabilityArg = getArgValue("--capability");
  if (!CAPABILITIES.includes(capabilityArg as Capability)) {
    throw new Error(`Pass exactly one --capability=${CAPABILITIES.join("|")}`);
  }
  const capability = capabilityArg as Capability;
  const apiKey = String(process.env.VIP98K_API_KEY || "").trim();
  if (!apiKey) throw new Error("VIP98K_API_KEY is required and must remain server-side");
  const baseUrl = normalizeVip98kBaseUrl(process.env.VIP98K_BASE_URL || VIP98K_BASE_URL);
  const catalog = await fetchVip98kCatalog({ baseURL: baseUrl, apiKey });
  const availableModelIds = new Set(catalog.map(getModelId));
  const explicitModelId = getArgValue("--model");
  const modelId =
    explicitModelId || DEFAULT_MODEL_IDS[capability].find((candidate) => availableModelIds.has(candidate));
  if (!modelId || !availableModelIds.has(modelId)) {
    throw new Error(`No catalog model is available for ${capability}; pass --model=<catalog-id>`);
  }

  const ledgerPath = resolve(
    getArgValue("--ledger") ||
      process.env.VIP98K_TEST_LEDGER ||
      ".local/vip98k-capability-tests.jsonl",
  );
  const ledger = await readLedger(ledgerPath);
  if (
    !hasFlag("--force") &&
    ledger.some(
      (entry) =>
        entry.capability === capability && entry.modelId === modelId && entry.status === "success",
    )
  ) {
    console.log(JSON.stringify({ skipped: true, capability, modelId, ledgerPath }, null, 2));
    return;
  }

  const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
  const startedAt = new Date();
  await writeLedger(ledgerPath, {
    runId,
    capability,
    modelId,
    status: "started",
    startedAt: startedAt.toISOString(),
  });

  try {
    let outputs: string[];
    if (capability === "chat" || capability === "multimodal") {
      outputs = await runChat({
        baseUrl,
        apiKey,
        modelId,
        multimodal: capability === "multimodal",
      });
    } else if (capability === "image" || capability === "multi-image-edit") {
      outputs = await runOpenAIImage({
        baseUrl,
        apiKey,
        modelId,
        multiImage: capability === "multi-image-edit",
      });
    } else {
      outputs = await runExternalTask({ baseUrl, apiKey, modelId, capability });
    }
    const finishedAt = new Date();
    const entry: LedgerEntry = {
      runId,
      capability,
      modelId,
      status: "success",
      startedAt: startedAt.toISOString(),
      finishedAt: finishedAt.toISOString(),
      durationMs: finishedAt.getTime() - startedAt.getTime(),
      outputCount: outputs.length,
      outputDigest: digestOutputs(outputs),
    };
    await writeLedger(ledgerPath, entry);
    console.log(JSON.stringify({ ...entry, ledgerPath }, null, 2));
  } catch (error) {
    const finishedAt = new Date();
    const entry: LedgerEntry = {
      runId,
      capability,
      modelId,
      status: "failure",
      startedAt: startedAt.toISOString(),
      finishedAt: finishedAt.toISOString(),
      durationMs: finishedAt.getTime() - startedAt.getTime(),
      error: sanitizeError(error),
    };
    await writeLedger(ledgerPath, entry);
    console.error(JSON.stringify({ ...entry, ledgerPath }, null, 2));
    process.exitCode = 1;
  }
}

main().catch((error) => {
  console.error(sanitizeError(error));
  process.exit(1);
});