486 lines
16 KiB
TypeScript
486 lines
16 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { appendFile, mkdir, readFile } from "node:fs/promises";
|
|
import { dirname, resolve } from "node:path";
|
|
import dotenv from "dotenv";
|
|
import sharp from "sharp";
|
|
|
|
dotenv.config({ path: ".env.local" });
|
|
dotenv.config();
|
|
|
|
const VIP98K_BASE_URL = "https://api.vip98k.com";
|
|
|
|
type Vip98kCatalogModel = {
|
|
id?: string;
|
|
modelId?: string;
|
|
model_id?: string;
|
|
slug?: string;
|
|
name?: string;
|
|
};
|
|
|
|
const CAPABILITIES = [
|
|
"chat",
|
|
"multimodal",
|
|
"image",
|
|
"multi-image-edit",
|
|
"rmbg",
|
|
"enhance",
|
|
"extend",
|
|
"video",
|
|
"world",
|
|
"text-to-3d",
|
|
"image-to-3d",
|
|
] as const;
|
|
|
|
type Capability = (typeof CAPABILITIES)[number];
|
|
type LedgerEntry = {
|
|
runId: string;
|
|
capability: Capability;
|
|
modelId: string;
|
|
status: "started" | "success" | "failure" | "skipped";
|
|
startedAt: string;
|
|
finishedAt?: string;
|
|
durationMs?: number;
|
|
outputCount?: number;
|
|
outputDigest?: string;
|
|
error?: string;
|
|
};
|
|
|
|
const DEFAULT_MODEL_IDS: Record<Capability, string[]> = {
|
|
chat: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"],
|
|
multimodal: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"],
|
|
image: ["nano-banana-2", "nano-banana-pro", "gpt-image-2-vip"],
|
|
"multi-image-edit": [
|
|
"google/nano-banana-2/reference-to-image",
|
|
"google/nano-banana-pro/edit",
|
|
"openai/gpt-image-2/edit",
|
|
],
|
|
rmbg: ["remove-background"],
|
|
enhance: ["enhance", "image-upscaler"],
|
|
extend: ["image-expansion"],
|
|
video: [
|
|
"bytedance/seedance-v1-pro-fast/text-to-video",
|
|
"veo-3.1-fast",
|
|
"google/veo3.1-fast/text-to-video",
|
|
],
|
|
world: ["world-generation"],
|
|
"text-to-3d": [
|
|
"tencent/hunyuan3d-pro/text-to-3d",
|
|
"tencent/hunyuan3d-rapid/text-to-3d",
|
|
],
|
|
"image-to-3d": [
|
|
"tencent/hunyuan3d-pro/image-to-3d",
|
|
"tencent/hunyuan3d-rapid/image-to-3d",
|
|
],
|
|
};
|
|
|
|
function getArgValue(name: string) {
|
|
const prefix = `${name}=`;
|
|
return process.argv.find((item) => item.startsWith(prefix))?.slice(prefix.length) || "";
|
|
}
|
|
|
|
function hasFlag(name: string) {
|
|
return process.argv.includes(name);
|
|
}
|
|
|
|
function getModelId(model: Vip98kCatalogModel) {
|
|
return String(model.id || model.modelId || model.model_id || model.slug || model.name || "").trim();
|
|
}
|
|
|
|
function normalizeVip98kBaseUrl(baseUrl?: string | null) {
|
|
const normalized = String(baseUrl || VIP98K_BASE_URL).trim().replace(/\/+$/, "");
|
|
if (!normalized) return VIP98K_BASE_URL;
|
|
return normalized.replace(/\/v1$/i, "").replace(/\/api\/external\/task$/i, "");
|
|
}
|
|
|
|
async function fetchVip98kCatalog(params: { baseURL: string; apiKey: string }) {
|
|
const payload = await parseResponse(
|
|
await fetch(`${normalizeVip98kBaseUrl(params.baseURL)}/api/external/task/models`, {
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
Accept: "application/json",
|
|
},
|
|
signal: AbortSignal.timeout(60_000),
|
|
}),
|
|
);
|
|
const record = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const data = record.data;
|
|
const models =
|
|
(Array.isArray(data?.models) && data.models) ||
|
|
(Array.isArray(data?.items) && data.items) ||
|
|
(Array.isArray(data) && data) ||
|
|
(Array.isArray(record.models) && record.models) ||
|
|
[];
|
|
if (models.length === 0) throw new Error("VIP98K model catalog is empty");
|
|
return models as Vip98kCatalogModel[];
|
|
}
|
|
|
|
function sanitizeError(error: unknown) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
return message.replace(/Bearer\s+\S+/gi, "Bearer [REDACTED]").slice(0, 2000);
|
|
}
|
|
|
|
function digestOutputs(outputs: string[]) {
|
|
return createHash("sha256").update(outputs.join("\n")).digest("hex");
|
|
}
|
|
|
|
async function readLedger(path: string): Promise<LedgerEntry[]> {
|
|
const content = await readFile(path, "utf8").catch(() => "");
|
|
return content
|
|
.split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.flatMap((line) => {
|
|
try {
|
|
return [JSON.parse(line) as LedgerEntry];
|
|
} catch {
|
|
return [];
|
|
}
|
|
});
|
|
}
|
|
|
|
async function writeLedger(path: string, entry: LedgerEntry) {
|
|
await mkdir(dirname(path), { recursive: true });
|
|
await appendFile(path, `${JSON.stringify(entry)}\n`, "utf8");
|
|
}
|
|
|
|
async function parseResponse(response: Response) {
|
|
const text = await response.text();
|
|
let payload: unknown = text;
|
|
try {
|
|
payload = text ? JSON.parse(text) : null;
|
|
} catch {
|
|
// Keep the response body for a useful provider error.
|
|
}
|
|
if (!response.ok) {
|
|
const record = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const message =
|
|
record.error?.message || record.error || record.message || text || `HTTP ${response.status}`;
|
|
throw new Error(`${String(message)} (HTTP ${response.status})`);
|
|
}
|
|
return payload;
|
|
}
|
|
|
|
function collectUrls(value: unknown, urls = new Set<string>()) {
|
|
if (!value) return urls;
|
|
if (typeof value === "string") {
|
|
for (const match of value.match(/https?:\/\/[^\s"'<>()[\]{}]+/g) || []) {
|
|
urls.add(match.replace(/[),.;]+$/g, ""));
|
|
}
|
|
if (/^data:[^;]+;base64,/i.test(value)) urls.add(value);
|
|
return urls;
|
|
}
|
|
if (Array.isArray(value)) {
|
|
value.forEach((item) => collectUrls(item, urls));
|
|
return urls;
|
|
}
|
|
if (typeof value === "object") {
|
|
Object.values(value as Record<string, unknown>).forEach((item) => collectUrls(item, urls));
|
|
}
|
|
return urls;
|
|
}
|
|
|
|
function extractTaskId(payload: unknown) {
|
|
const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const data = root.data && typeof root.data === "object" ? root.data : root;
|
|
return String(data.taskId || data.task_id || data.id || data.requestId || "").trim();
|
|
}
|
|
|
|
function extractResultPath(payload: unknown) {
|
|
const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const data = root.data && typeof root.data === "object" ? root.data : root;
|
|
return String(data.resultPath || data.result_path || "").trim();
|
|
}
|
|
|
|
function extractStatus(payload: unknown) {
|
|
const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const data = root.data && typeof root.data === "object" ? root.data : root;
|
|
return String(data.status || data.state || data.taskStatus || "").trim().toLowerCase();
|
|
}
|
|
|
|
async function makeReferencePng(color: string, label: string) {
|
|
return sharp({
|
|
create: { width: 96, height: 96, channels: 4, background: color },
|
|
})
|
|
.composite([
|
|
{
|
|
input: Buffer.from(
|
|
`<svg width="96" height="96"><text x="48" y="58" text-anchor="middle" font-size="30" fill="white">${label}</text></svg>`,
|
|
),
|
|
},
|
|
])
|
|
.png()
|
|
.toBuffer();
|
|
}
|
|
|
|
async function runOpenAIImage(params: {
|
|
baseUrl: string;
|
|
apiKey: string;
|
|
modelId: string;
|
|
multiImage: boolean;
|
|
}) {
|
|
const endpoint = `${params.baseUrl}/v1/images/${params.multiImage ? "edits" : "generations"}`;
|
|
if (!params.multiImage) {
|
|
const payload = await parseResponse(
|
|
await fetch(endpoint, {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({
|
|
model: params.modelId,
|
|
prompt: "A small blue ceramic cup on a white studio background",
|
|
size: "1024x1024",
|
|
response_format: "url",
|
|
}),
|
|
signal: AbortSignal.timeout(900_000),
|
|
}),
|
|
);
|
|
const outputs = Array.from(collectUrls(payload));
|
|
if (outputs.length === 0) throw new Error("Image generation returned no image output");
|
|
return outputs;
|
|
}
|
|
|
|
const [left, right] = await Promise.all([
|
|
makeReferencePng("#2563eb", "A"),
|
|
makeReferencePng("#dc2626", "B"),
|
|
]);
|
|
const form = new FormData();
|
|
form.append("model", params.modelId);
|
|
form.append("prompt", "Combine both references into one clean side-by-side product comparison image");
|
|
form.append("size", "1024x1024");
|
|
form.append("response_format", "url");
|
|
form.append("image[]", new Blob([left], { type: "image/png" }), "reference-a.png");
|
|
form.append("image[]", new Blob([right], { type: "image/png" }), "reference-b.png");
|
|
const payload = await parseResponse(
|
|
await fetch(endpoint, {
|
|
method: "POST",
|
|
headers: { Authorization: `Bearer ${params.apiKey}` },
|
|
body: form,
|
|
signal: AbortSignal.timeout(900_000),
|
|
}),
|
|
);
|
|
const outputs = Array.from(collectUrls(payload));
|
|
if (outputs.length === 0) throw new Error("Multi-image edit returned no image output");
|
|
return outputs;
|
|
}
|
|
|
|
async function runChat(params: {
|
|
baseUrl: string;
|
|
apiKey: string;
|
|
modelId: string;
|
|
multimodal: boolean;
|
|
}) {
|
|
const content = params.multimodal
|
|
? [
|
|
{ type: "text", text: "Reply with the dominant color in one English word." },
|
|
{
|
|
type: "image_url",
|
|
image_url: { url: "https://dummyimage.com/96x96/2563eb/ffffff.png&text=A" },
|
|
},
|
|
]
|
|
: "Reply with exactly: vip98k-ok";
|
|
const payload = await parseResponse(
|
|
await fetch(`${params.baseUrl}/v1/chat/completions`, {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({
|
|
model: params.modelId,
|
|
messages: [{ role: "user", content }],
|
|
max_tokens: 64,
|
|
}),
|
|
signal: AbortSignal.timeout(180_000),
|
|
}),
|
|
);
|
|
const text = JSON.stringify(payload);
|
|
if (!text || text === "{}") throw new Error("Chat request returned no content");
|
|
return [text.slice(0, 500)];
|
|
}
|
|
|
|
async function runExternalTask(params: {
|
|
baseUrl: string;
|
|
apiKey: string;
|
|
modelId: string;
|
|
capability: Capability;
|
|
}) {
|
|
const imageUrl = "https://dummyimage.com/512x512/2563eb/ffffff.png&text=VIP98K";
|
|
const secondImageUrl = "https://dummyimage.com/512x512/dc2626/ffffff.png&text=REFERENCE";
|
|
const body: Record<string, unknown> = {
|
|
model: params.modelId,
|
|
displayName: `provider-smoke-${params.capability}`,
|
|
prompt: "Create a minimal test result with one simple object",
|
|
};
|
|
if (
|
|
["multi-image-edit", "rmbg", "enhance", "extend", "image-to-3d"].includes(
|
|
params.capability,
|
|
)
|
|
) {
|
|
const images =
|
|
params.capability === "multi-image-edit" ? [imageUrl, secondImageUrl] : [imageUrl];
|
|
Object.assign(body, {
|
|
image: imageUrl,
|
|
imageUrl,
|
|
image_url: imageUrl,
|
|
images,
|
|
imageUrls: images,
|
|
referenceImages: images,
|
|
reference_images: images,
|
|
});
|
|
}
|
|
if (params.capability === "extend") {
|
|
Object.assign(body, { aspectRatio: "16:9", aspect_ratio: "16:9", ratio: "16:9" });
|
|
}
|
|
if (params.capability === "video") {
|
|
Object.assign(body, { mode: "text-to-video", duration: 4, resolution: "480p", ratio: "16:9" });
|
|
}
|
|
if (params.capability === "world") body.mode = "world-generation";
|
|
|
|
const submitted = await parseResponse(
|
|
await fetch(`${params.baseUrl}/api/external/task/generate`, {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify(body),
|
|
signal: AbortSignal.timeout(180_000),
|
|
}),
|
|
);
|
|
const immediate = Array.from(collectUrls(submitted));
|
|
if (immediate.length > 0) return immediate;
|
|
const taskId = extractTaskId(submitted);
|
|
if (!taskId) throw new Error("External task returned neither output nor task ID");
|
|
const resultPath = extractResultPath(submitted);
|
|
const startedAt = Date.now();
|
|
while (Date.now() - startedAt < 1_200_000) {
|
|
await new Promise((resolvePromise) => setTimeout(resolvePromise, 5000));
|
|
const resultUrl = resultPath
|
|
? resultPath.startsWith("http")
|
|
? resultPath
|
|
: `${params.baseUrl}${resultPath.startsWith("/") ? "" : "/"}${resultPath}`
|
|
: `${params.baseUrl}/api/external/task/result?taskId=${encodeURIComponent(taskId)}`;
|
|
const payload = await parseResponse(
|
|
await fetch(resultUrl, {
|
|
headers: { Authorization: `Bearer ${params.apiKey}` },
|
|
signal: AbortSignal.timeout(60_000),
|
|
}),
|
|
);
|
|
const status = extractStatus(payload);
|
|
const outputs = Array.from(collectUrls(payload));
|
|
if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) {
|
|
throw new Error(`External task failed with status ${status}`);
|
|
}
|
|
if (
|
|
["succeeded", "success", "completed", "done", "finished"].includes(status) ||
|
|
outputs.length > 0
|
|
) {
|
|
if (outputs.length === 0) throw new Error("External task completed without output URL");
|
|
return outputs;
|
|
}
|
|
}
|
|
throw new Error("External task did not complete within 20 minutes");
|
|
}
|
|
|
|
async function main() {
|
|
const capabilityArg = getArgValue("--capability");
|
|
if (!CAPABILITIES.includes(capabilityArg as Capability)) {
|
|
throw new Error(`Pass exactly one --capability=${CAPABILITIES.join("|")}`);
|
|
}
|
|
const capability = capabilityArg as Capability;
|
|
const apiKey = String(process.env.VIP98K_API_KEY || "").trim();
|
|
if (!apiKey) throw new Error("VIP98K_API_KEY is required and must remain server-side");
|
|
const baseUrl = normalizeVip98kBaseUrl(process.env.VIP98K_BASE_URL || VIP98K_BASE_URL);
|
|
const catalog = await fetchVip98kCatalog({ baseURL: baseUrl, apiKey });
|
|
const availableModelIds = new Set(catalog.map(getModelId));
|
|
const explicitModelId = getArgValue("--model");
|
|
const modelId =
|
|
explicitModelId || DEFAULT_MODEL_IDS[capability].find((candidate) => availableModelIds.has(candidate));
|
|
if (!modelId || !availableModelIds.has(modelId)) {
|
|
throw new Error(`No catalog model is available for ${capability}; pass --model=<catalog-id>`);
|
|
}
|
|
|
|
const ledgerPath = resolve(
|
|
getArgValue("--ledger") ||
|
|
process.env.VIP98K_TEST_LEDGER ||
|
|
".local/vip98k-capability-tests.jsonl",
|
|
);
|
|
const ledger = await readLedger(ledgerPath);
|
|
if (
|
|
!hasFlag("--force") &&
|
|
ledger.some(
|
|
(entry) =>
|
|
entry.capability === capability && entry.modelId === modelId && entry.status === "success",
|
|
)
|
|
) {
|
|
console.log(JSON.stringify({ skipped: true, capability, modelId, ledgerPath }, null, 2));
|
|
return;
|
|
}
|
|
|
|
const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
const startedAt = new Date();
|
|
await writeLedger(ledgerPath, {
|
|
runId,
|
|
capability,
|
|
modelId,
|
|
status: "started",
|
|
startedAt: startedAt.toISOString(),
|
|
});
|
|
|
|
try {
|
|
let outputs: string[];
|
|
if (capability === "chat" || capability === "multimodal") {
|
|
outputs = await runChat({
|
|
baseUrl,
|
|
apiKey,
|
|
modelId,
|
|
multimodal: capability === "multimodal",
|
|
});
|
|
} else if (capability === "image") {
|
|
outputs = await runOpenAIImage({
|
|
baseUrl,
|
|
apiKey,
|
|
modelId,
|
|
multiImage: false,
|
|
});
|
|
} else {
|
|
outputs = await runExternalTask({ baseUrl, apiKey, modelId, capability });
|
|
}
|
|
const finishedAt = new Date();
|
|
const entry: LedgerEntry = {
|
|
runId,
|
|
capability,
|
|
modelId,
|
|
status: "success",
|
|
startedAt: startedAt.toISOString(),
|
|
finishedAt: finishedAt.toISOString(),
|
|
durationMs: finishedAt.getTime() - startedAt.getTime(),
|
|
outputCount: outputs.length,
|
|
outputDigest: digestOutputs(outputs),
|
|
};
|
|
await writeLedger(ledgerPath, entry);
|
|
console.log(JSON.stringify({ ...entry, ledgerPath }, null, 2));
|
|
} catch (error) {
|
|
const finishedAt = new Date();
|
|
const entry: LedgerEntry = {
|
|
runId,
|
|
capability,
|
|
modelId,
|
|
status: "failure",
|
|
startedAt: startedAt.toISOString(),
|
|
finishedAt: finishedAt.toISOString(),
|
|
durationMs: finishedAt.getTime() - startedAt.getTime(),
|
|
error: sanitizeError(error),
|
|
};
|
|
await writeLedger(ledgerPath, entry);
|
|
console.error(JSON.stringify({ ...entry, ledgerPath }, null, 2));
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(sanitizeError(error));
|
|
process.exit(1);
|
|
});
|