lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ai): preserve realtime canvas and normalize model types

Browse Source
This commit is contained in:
134 2026-08-09 18:01:33 +08:00 committed by root
parent 27736f0a5d
commit 5decd202a1
3 changed files with 277 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
lib/ai/vip98k-models.ts
View File

@ -60,6 +60,16 @@ export interface Vip98kCatalogModel {
invocation?: Vip98kInvocationContract;
taskTypes?: unknown;
task_types?: unknown;
releasedAt?: unknown;
released_at?: unknown;
releaseDate?: unknown;
release_date?: unknown;
releaseYear?: unknown;
release_year?: unknown;
publishedAt?: unknown;
published_at?: unknown;
launchDate?: unknown;
launch_date?: unknown;
[key: string]: unknown;
}
@ -105,9 +115,79 @@ const TYPE_SORT_ORDER: Record<string, number> = {
};
const IMAGE_TOOL_MODEL_IDS = new Set([
"background-generator",
"clothing-segmentation",
"colorize",
"compress",
"enhance",
"face-swap-image",
"head-cutout",
"id-photo",
"image-expansion",
"image-upscaler",
"photo-cleanup",
"remove-background",
"remove-object",
"remove-watermark",
"smart-crop",
"youchuan/v8.1/remove-background",
]);
const VIDEO_CAPABILITY_KEYS = new Set([
"audio-to-video",
"first-frame-to-video",
"image-to-video",
"reference-to-video",
"start-end-frame-to-video",
"text-to-video",
"video-extend",
"video-to-video",
]);
const IMAGE_CAPABILITY_KEYS = new Set([
"image-to-image",
"reference-to-image",
"text-to-image",
]);
const IMAGE_TOOL_OPERATION_BY_MODEL_ID: Record<string, string> = {
"background-generator": "background-generation",
"clothing-segmentation": "segmentation",
colorize: "colorize",
compress: "compress",
enhance: "enhance",
"face-swap-image": "face-swap",
"head-cutout": "remove-background",
"id-photo": "id-photo",
"image-expansion": "extend",
"image-upscaler": "enhance",
"photo-cleanup": "cleanup",
"remove-background": "remove-background",
"remove-object": "remove-object",
"remove-watermark": "remove-watermark",
"smart-crop": "smart-crop",
"youchuan/v8.1/remove-background": "remove-background",
};
// The VIP98K catalog currently has no release-date field. Unknown image models stay
// available to backend routing but remain hidden until their 2026+ release is verified.
const VERIFIED_2026_IMAGE_MODEL_PATTERNS = [
/^nano-banana-2(?:$|[-/])/,
/^gpt-image-2(?:$|[-/])/,
/^alibaba\/wan-2\.(?:6|7)(?:$|[-/])/,
/^black-forest-labs\/flux-2(?:$|[-/])/,
/^bytedance\/seedream-v5(?:$|[.\-/])/,
/^google\/nano-banana-2(?:$|[-/])/,
/^hidream-o1-1\.5(?:$|[-/])/,
/^ideogram\/v4(?:$|[-/])/,
/^krea-2(?:$|[-/])/,
/^microsoft\/mai-image-2\.5(?:$|[-/])/,
/^nvidia\/cosmos-3(?:$|[-/])/,
/^openai\/gpt-image-2(?:$|[-/])/,
/^qwen\/qwen-image-2\.0(?:$|[-/])/,
/^youchuan\/v8\.1(?:$|[-/])/,
];
function toRecord(value: unknown): Record<string, unknown> | null {
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
return value as Record<string, unknown>;
@ -148,6 +228,35 @@ function normalizeCatalogType(model: Vip98kCatalogModel): Vip98kCatalogModelType
.toLowerCase();
}
function getReleaseYear(model: Vip98kCatalogModel): number | null {
const candidates = [
model.releaseYear,
model.release_year,
model.releasedAt,
model.released_at,
model.releaseDate,
model.release_date,
model.publishedAt,
model.published_at,
model.launchDate,
model.launch_date,
];
for (const candidate of candidates) {
const match = String(candidate || "").match(/(?:19|20)\d{2}/);
if (match) return Number(match[0]);
}
const datedModelId = getModelId(model).match(/(?:^|[^\d])((?:19|20)\d{2})\d{4}(?:[^\d]|$)/);
return datedModelId?.[1] ? Number(datedModelId[1]) : null;
}
export function isVip98kImageModelVisible(model: Vip98kCatalogModel) {
const releaseYear = getReleaseYear(model);
if (releaseYear !== null) return releaseYear >= 2026;
const modelId = getModelId(model).toLowerCase();
return VERIFIED_2026_IMAGE_MODEL_PATTERNS.some((pattern) => pattern.test(modelId));
}
function resolvePresetType(model: Vip98kCatalogModel): Vip98kCatalogModelType {
const catalogType = normalizeCatalogType(model);
const modelId = getModelId(model).toLowerCase();
@ -160,10 +269,34 @@ function resolvePresetType(model: Vip98kCatalogModel): Vip98kCatalogModelType {
return "model_3d";
}
if (catalogType === "image" && IMAGE_TOOL_MODEL_IDS.has(modelId)) {
if (catalogType === "world") return "world";
if (catalogType === "image_tool" || IMAGE_TOOL_MODEL_IDS.has(modelId)) {
return "image_tool";
}
if (
catalogType === "video" ||
capabilities.some(
(capability) =>
VIDEO_CAPABILITY_KEYS.has(capability) ||
capability.endsWith("-to-video") ||
capability.startsWith("video-"),
)
) {
return "video";
}
if (
catalogType === "image" ||
capabilities.some(
(capability) =>
IMAGE_CAPABILITY_KEYS.has(capability) || capability.endsWith("-to-image"),
)
) {
return "image";
}
return catalogType;
}
@ -327,7 +460,7 @@ function createImagePreset(
type: "image",
supportsVision: supportsReferenceImages,
supportsImageGen: true,
isHidden: false,
isHidden: !isVip98kImageModelVisible(model),
capabilities: {
provider: "vip98k",
kind: "image",
@ -596,6 +729,7 @@ function createToolPreset(
const inputModes = getInputModes(model);
const outputModes = getOutputModes(model);
const isImageTool = type === "image_tool";
const toolOperation = IMAGE_TOOL_OPERATION_BY_MODEL_ID[modelId.toLowerCase()] || type;
return {
modelId,
@ -614,12 +748,14 @@ function createToolPreset(
inputModes,
outputModes,
supportedModes: getCapabilities(model),
toolOperation,
vip98k: getVip98kMetadata(model, type),
},
defaultParams: {
provider: "vip98k",
kind: type.replace(/_/g, "-"),
endpoint: modelId,
toolOperation,
taskApi: type === "audio" ? "openai-compatible" : "external-task",
},
imageGenConfig: null,

42
scripts/consolidate-ai-providers.ts
View File

@ -28,6 +28,9 @@ const REQUIRED_CAPABILITY_MODELS = [
"tencent/hunyuan3d-pro/text-to-3d",
"tencent/hunyuan3d-pro/image-to-3d",
] as const;
const PROTECTED_SYSTEM_SETTING_KEYS = new Set([
"realtime_canvas_config",
]);
type ProviderRow = {
id: string;
@ -45,6 +48,7 @@ type ModelRow = {
displayName: string;
supportsVision: boolean;
supportsImageGen: boolean;
isHidden: boolean;
capabilities: unknown;
imageGenConfig: unknown;
};
@ -224,6 +228,12 @@ async function main() {
providers: sortById(falProviders),
models: sortById(models.filter((model) => falProviderIds.has(model.providerId))),
});
const protectedSettingsBefore = await sql<{ key: string; value: unknown }[]>`
select key, value from "SystemSettings"
where key = any(${Array.from(PROTECTED_SYSTEM_SETTING_KEYS)}::text[])
order by key
`;
const protectedSettingsFingerprintBefore = stableHash(protectedSettingsBefore);
const removableProviders = providers.filter(
(provider) => !KEEP_PROVIDER_TYPES.has(provider.type),
);
@ -367,6 +377,7 @@ async function main() {
select id, key, value from "SystemSettings"
`;
for (const setting of settings) {
if (PROTECTED_SYSTEM_SETTING_KEYS.has(setting.key)) continue;
let nextValue = rewriteSettingValue({
value: setting.value,
oldProviders: oldProviderMap,
@ -415,6 +426,36 @@ async function main() {
if (falFingerprintAfter !== falFingerprintBefore) {
throw new Error("FAL provider/model fingerprint changed; restore the verified database backup");
}
const protectedSettingsAfter = await sql<{ key: string; value: unknown }[]>`
select key, value from "SystemSettings"
where key = any(${Array.from(PROTECTED_SYSTEM_SETTING_KEYS)}::text[])
order by key
`;
if (stableHash(protectedSettingsAfter) !== protectedSettingsFingerprintBefore) {
throw new Error("Realtime canvas configuration changed; restore the verified database backup");
}
const vipProviderIds = new Set(
remainingProviders
.filter((provider) => provider.type === "vip98k")
.map((provider) => provider.id),
);
const invalidVipModels = remainingModels.filter((model) => {
if (!vipProviderIds.has(model.providerId)) return false;
const capabilities = model.capabilities as Record<string, unknown> | null;
const kind = String(capabilities?.kind || "").trim().toLowerCase();
if (["video", "model-3d", "world", "chat"].includes(kind)) {
return model.supportsImageGen;
}
return kind === "image-tool" && !model.isHidden;
});
if (invalidVipModels.length > 0) {
throw new Error(
`VIP98K model classification invalid: ${invalidVipModels
.slice(0, 10)
.map((model) => model.modelId)
.join(", ")}`,
);
}
const invalidTypes = remainingProviders.filter(
(provider) => !KEEP_PROVIDER_TYPES.has(provider.type),
);
@ -433,6 +474,7 @@ async function main() {
),
).length,
falFingerprintUnchanged: true,
realtimeCanvasConfigUnchanged: true,
},
null,
2,

97
tests/routes/model-classification.test.ts
View File

@ -14,6 +14,7 @@ import {
isImageGenerationModelDescriptor,
isVideoModelDescriptor,
} from "@/lib/ai/model-classification";
import { buildVip98kModelPresets } from "@/lib/ai/vip98k-models";
const atlasHunyuan3D = {
model: "tencent/hunyuan3d-pro/text-to-3d",
@ -305,4 +306,100 @@ test.describe("AI model media classification", () => {
watermark: true,
});
});
test("normalizes VIP98K media and tool types before they reach the frontend", () => {
const presets = buildVip98kModelPresets([
{
modelId: "mislabelled-video",
displayName: "Mislabelled video",
type: "image",
capabilities: ["text_to_video"],
},
{
modelId: "enhance",
displayName: "Enhance",
type: "image",
capabilities: ["image_processing"],
},
{
modelId: "image-expansion",
displayName: "Extend",
type: "image_tool",
capabilities: ["outpainting"],
},
{
modelId: "remove-background",
displayName: "Remove background",
type: "image_tool",
capabilities: ["image_processing"],
},
{
modelId: "multimodal-model",
displayName: "Multimodal model",
type: "model",
capabilities: ["multimodal"],
inputModes: ["text", "image"],
},
]);
expect(presets.find((item) => item.modelId === "mislabelled-video")).toMatchObject({
type: "video",
supportsImageGen: false,
});
expect(presets.find((item) => item.modelId === "enhance")).toMatchObject({
type: "image_tool",
isHidden: true,
capabilities: { toolOperation: "enhance" },
});
expect(presets.find((item) => item.modelId === "image-expansion")).toMatchObject({
type: "image_tool",
capabilities: { toolOperation: "extend" },
});
expect(presets.find((item) => item.modelId === "remove-background")).toMatchObject({
type: "image_tool",
capabilities: { toolOperation: "remove-background" },
});
expect(presets.find((item) => item.modelId === "multimodal-model")).toMatchObject({
type: "model",
supportsVision: true,
supportsImageGen: false,
});
});
test("only shows VIP98K image models verified as released in 2026 or later", () => {
const presets = buildVip98kModelPresets([
{
modelId: "black-forest-labs/flux-dev",
displayName: "Flux Dev",
type: "image",
capabilities: ["text_to_image"],
},
{
modelId: "google/nano-banana-2/text-to-image",
displayName: "Nano Banana 2",
type: "image",
capabilities: ["text_to_image"],
},
{
modelId: "future-image-with-date",
displayName: "Future image",
type: "image",
capabilities: ["text_to_image"],
releasedAt: "2027-02-01",
},
{
modelId: "old-image-with-date",
displayName: "Old image",
type: "image",
capabilities: ["text_to_image"],
releasedAt: "2025-12-15",
},
]);
expect(presets.find((item) => item.modelId === "black-forest-labs/flux-dev")?.isHidden).toBe(true);
expect(
presets.find((item) => item.modelId === "google/nano-banana-2/text-to-image")?.isHidden,
).toBe(false);
expect(presets.find((item) => item.modelId === "future-image-with-date")?.isHidden).toBe(false);
expect(presets.find((item) => item.modelId === "old-image-with-date")?.isHidden).toBe(true);
});
});