From e8d1aa25aa5aa67e93e9114573b48eda2b55d491 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 21 Jul 2026 11:01:43 +0800 Subject: [PATCH 01/83] feat: add Atlas Cloud image provider --- app/(admin)/settings/_lib/types.ts | 1 + .../settings/admin-settings-client.tsx | 73 +++ app/(chat)/api/chat/_lib/image-failover.ts | 2 + app/(chat)/api/chat/_lib/image-media.ts | 1 + app/(chat)/api/chat/_lib/stream.ts | 1 + app/(chat)/api/chat/route.ts | 13 + app/api/admin/ai-providers/route.ts | 4 + app/api/admin/atlascloud-models/route.ts | 155 +++++ app/api/canvas/text-edit/route.ts | 12 +- app/api/scene-fusion/route.ts | 58 +- components/image-generation-loading.tsx | 1 + components/image-params-bar.tsx | 1 + components/image-preview.tsx | 3 + lib/ai/atlascloud-models.ts | 523 +++++++++++++++ lib/ai/dynamic-provider.ts | 13 + lib/ai/model-classification.ts | 1 + lib/ai/model-router.ts | 13 +- lib/ai/provider-defaults.ts | 1 + lib/ai/providers/atlascloud.ts | 596 ++++++++++++++++++ lib/canvas/generate-image.ts | 25 +- lib/db/migrate.ts | 3 +- .../0092_add_atlascloud_provider_type.sql | 40 ++ lib/db/migrations/meta/_journal.json | 7 + lib/db/queries.ts | 4 +- lib/db/schema.ts | 2 +- lib/external-image-api/service.ts | 1 + lib/scene-fusion/output-capabilities.ts | 2 + 27 files changed, 1541 insertions(+), 15 deletions(-) create mode 100644 app/api/admin/atlascloud-models/route.ts create mode 100644 lib/ai/atlascloud-models.ts create mode 100644 lib/ai/providers/atlascloud.ts create mode 100644 lib/db/migrations/0092_add_atlascloud_provider_type.sql diff --git a/app/(admin)/settings/_lib/types.ts b/app/(admin)/settings/_lib/types.ts index 7a89fcbe5..38740da8f 100644 --- a/app/(admin)/settings/_lib/types.ts +++ b/app/(admin)/settings/_lib/types.ts @@ -90,6 +90,7 @@ export type AdminAiProvider = { | "openai-image" | "bananaimage" | "visionary" + | "atlascloud" | "stepfun-image" | "midjourney" | "fal-video" diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index d26512e1f..37bb911de 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -625,6 +625,7 @@ type AiProvider = { | "openai-image" | "bananaimage" | "visionary" + | "atlascloud" | "stepfun-image" | "midjourney" | "fal-video" @@ -1585,6 +1586,7 @@ const PROVIDER_TYPES = [ { value: "openai-image", label: "OpenAI 图片兼容" }, { value: "bananaimage", label: "BananaImage 生图" }, { value: "visionary", label: "Visionary 生图" }, + { value: "atlascloud", label: "Atlas Cloud 图片模型" }, { value: "stepfun-image", label: "阶跃星辰 Step Image Edit" }, { value: "bltcy-video", label: "柏拉图统一视频" }, { value: "grsai-gpt-image", label: "Grsai GPT Image" }, @@ -1621,6 +1623,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record = { "openai-image": "https://api.openai.com/v1", bananaimage: "https://bananaimage.aigenmedia.art", visionary: "https://api.visionary.beer", + atlascloud: "https://api.atlascloud.ai", "stepfun-image": "https://api.stepfun.com", "bltcy-video": "https://api.bltcy.ai", "grsai-gpt-image": "https://grsai.dakka.com.cn", @@ -4039,6 +4042,7 @@ export function AdminSettingsClient() { | "openai-image" | "bananaimage" | "visionary" + | "atlascloud" | "stepfun-image" | "midjourney" | "fal-video" @@ -6277,6 +6281,7 @@ export function AdminSettingsClient() { const isSnapgenImageProvider = providerForm.type === "snapgen-image"; const isBananaImageProvider = providerForm.type === "bananaimage"; const isVisionaryProvider = providerForm.type === "visionary"; + const isAtlasCloudProvider = providerForm.type === "atlascloud"; const isBltcyVideoProvider = providerForm.type === "bltcy-video"; const isLemonProvider = providerForm.type === "openai" && @@ -6441,6 +6446,17 @@ export function AdminSettingsClient() { } } + if (isNewProvider && isAtlasCloudProvider && providerId) { + try { + await handleQuickAddAtlasCloudModels(providerId, { skipConfirm: true }); + } catch (modelError) { + console.error("[Auto Add Atlas Cloud Models] Error:", modelError); + toast.warning( + 'Atlas Cloud 模型同步失败，请手动点击"同步 Atlas Cloud 全部图片模型"按钮' + ); + } + } + if (isNewProvider && isBltcyVideoProvider && providerId) { try { await handleQuickAddBltcyVideoModels(providerId, { @@ -7490,6 +7506,45 @@ export function AdminSettingsClient() { } }; + const handleQuickAddAtlasCloudModels = async ( + providerId: string, + options?: { skipConfirm?: boolean } + ) => { + if ( + !options?.skipConfirm && + !confirm( + "将从 Atlas Cloud 官方目录同步全部开放图片模型，并读取每个模型的参数 Schema。同步数量较多，可能需要几十秒。\n\n继续吗？" + ) + ) { + return; + } + + try { + toast.info("正在读取 Atlas Cloud 官方目录和模型参数..."); + const response = await fetch("/api/admin/atlascloud-models", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ providerId }), + }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || "同步失败"); + + const totalSynced = + Number(data.addedCount || 0) + Number(data.updatedCount || 0); + toast.success( + `已同步 ${totalSynced} 个 Atlas Cloud 图片模型（官方目录共 ${Number(data.counts?.total || totalSynced)} 个）` + ); + mutateModels(); + if (data.errors?.length) { + toast.warning(`${data.errors.length} 个 Atlas Cloud 模型读取或同步失败`); + } + } catch (error: any) { + console.error("[Quick Add Atlas Cloud Models] Error:", error); + toast.error(error.message || "同步 Atlas Cloud 图片模型失败"); + throw error; + } + }; + const handleUseFalImageProviderPreset = () => { setProviderForm({ id: "", @@ -15706,6 +15761,13 @@ export function AdminSettingsClient() { 质量档位和 1K/2K/4K 输出。

)} + {providerForm.type === "atlascloud" && ( +

+ Atlas Cloud 使用 Bearer API Key 认证，默认地址 + https://api.atlascloud.ai。保存后会从官方目录同步全部开放图片模型， + 并按每个模型自己的 Schema 配置多图数量、比例及 1K/2K/4K 档位。 +

+ )} {providerForm.type === "bfl-image" && isBltcyBflBaseUrl(providerForm.baseUrl) && (

@@ -15852,6 +15914,17 @@ export function AdminSettingsClient() { 快速添加 Visionary 图片预设 )} + {provider.type === "atlascloud" && ( + + )} {provider.type === "bltcy-video" && ( )} {provider.type === "bltcy-video" && ( @@ -17417,6 +17438,11 @@ export function AdminSettingsClient() { capabilities: (model as any).capabilities, supportsImageGen: model.supportsImageGen, }); + const is3DModel = is3DModelDescriptor({ + providerType: provider?.type, + modelId: model.modelId, + capabilities: (model as any).capabilities, + }); const isImageModel = isImageGenerationModelDescriptor({ providerType: provider?.type, modelId: model.modelId, @@ -17479,6 +17505,11 @@ export function AdminSettingsClient() { Agent )} + {is3DModel && ( + + 3D模型 + + )} {isImageModel && ( 图片模型 @@ -17489,7 +17520,7 @@ export function AdminSettingsClient() { 视频模型 )} - {!isImageModel && !isVideoModel && ( + {!is3DModel && !isImageModel && !isVideoModel && ( 对话模型 diff --git a/app/api/admin/atlascloud-models/route.ts b/app/api/admin/atlascloud-models/route.ts index ba4803a76..74636b29d 100644 --- a/app/api/admin/atlascloud-models/route.ts +++ b/app/api/admin/atlascloud-models/route.ts @@ -79,7 +79,7 @@ export async function POST(request: Request) { avatarUrl: preset.avatarUrl || null, description: preset.description, supportsVision: preset.supportsVision, - supportsImageGen: true, + supportsImageGen: preset.supportsImageGen, isEnabled: true, isHidden: false, sortOrder: preset.sortOrder, diff --git a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx index 14d633723..06d23ce15 100644 --- a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx +++ b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx @@ -2013,6 +2013,7 @@ export default function CanvasEditorLegacyPage({ useState(false); // 面板模式: 'chat' 聊天面板, 'fileList' 文件列表 const [panelMode, setPanelMode] = useState<"chat" | "fileList">("chat"); + const chatPanelCanvasIdRef = useRef(null); const [viewportWidth, setViewportWidth] = useState(0); const [horizontalFloatingToolbarWidth, setHorizontalFloatingToolbarWidth] = useState(0); @@ -2036,6 +2037,16 @@ export default function CanvasEditorLegacyPage({ Partial> >({}); + useLayoutEffect(() => { + if (!id || isVideoMode || chatPanelCanvasIdRef.current === id) { + return; + } + + chatPanelCanvasIdRef.current = id; + setPanelMode("chat"); + setIsPanelOpen(true); + }, [id, isVideoMode]); + useEffect(() => { if (!isVideoMode) { setVideoCanvasInsertMenu(null); diff --git a/lib/ai/atlascloud-models.ts b/lib/ai/atlascloud-models.ts index 2ab430267..21f10e4a0 100644 --- a/lib/ai/atlascloud-models.ts +++ b/lib/ai/atlascloud-models.ts @@ -77,7 +77,7 @@ export interface AtlasCloudImageModelPreset { avatarUrl?: string; description: string; supportsVision: boolean; - supportsImageGen: true; + supportsImageGen: boolean; sortOrder: number; capabilities: Record; defaultParams: Record; @@ -86,6 +86,8 @@ export interface AtlasCloudImageModelPreset { type JsonRecord = Record; +export type AtlasCloudMediaKind = "image" | "video" | "3d"; + function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) @@ -98,6 +100,67 @@ function uniqueStrings(values: unknown[]) { ); } +function normalizeIntentToken(value: unknown) { + return String(value || "") + .trim() + .toLowerCase() + .replaceAll("_", "-"); +} + +export function inferAtlasCloudMediaKind( + catalogModel: AtlasCloudCatalogModel +): AtlasCloudMediaKind { + const catalogType = normalizeIntentToken(catalogModel.type); + const intentTokens = uniqueStrings([ + catalogModel.model, + catalogModel.displayName, + ...(catalogModel.categories || []), + ...(catalogModel.tags || []), + ]).map(normalizeIntentToken); + const intentText = intentTokens.join(" "); + + if ( + /(?:^|[\s/.-])(?:text|image)-to-3d(?:$|[\s/.-])/.test(intentText) || + /(?:^|[\s/.-])(?:hunyuan3d|seed3d|tripo-h3(?:\.\d+)?)(?:$|[\s/.-])/.test( + intentText + ) + ) { + return "3d"; + } + + if ( + catalogType === "video" || + intentTokens.some((token) => + [ + "text-to-video", + "image-to-video", + "video-to-video", + "audio-to-video", + ].includes(token) + ) + ) { + return "video"; + } + + return "image"; +} + +function derive3DSupportedModes(catalogModel: AtlasCloudCatalogModel) { + const intentText = uniqueStrings([ + catalogModel.model, + catalogModel.displayName, + ...(catalogModel.categories || []), + ...(catalogModel.tags || []), + ]) + .map(normalizeIntentToken) + .join(" "); + const modes = [ + ...(intentText.includes("text-to-3d") ? ["text-to-3d"] : []), + ...(intentText.includes("image-to-3d") ? ["image-to-3d"] : []), + ]; + return modes.length > 0 ? modes : ["3d"]; +} + function findFirstKey(properties: JsonRecord, candidates: string[]) { return candidates.find((key) => key in properties); } @@ -361,15 +424,22 @@ export function createAtlasCloudImagePreset( const categories = uniqueStrings(catalogModel.categories || []).map((value) => value.toLowerCase() ); - const supportedModes = [ - ...(categories.some((value) => value.includes("text-to-image")) - ? ["text-to-image"] - : []), - ...(imageInputKey || categories.some((value) => value.includes("image-to-image")) - ? ["image-edit", "reference-images"] - : []), - ]; - if (supportedModes.length === 0) supportedModes.push("text-to-image"); + const mediaKind = inferAtlasCloudMediaKind(catalogModel); + const supportedModes = + mediaKind === "3d" + ? derive3DSupportedModes(catalogModel) + : [ + ...(categories.some((value) => value.includes("text-to-image")) + ? ["text-to-image"] + : []), + ...(imageInputKey || + categories.some((value) => value.includes("image-to-image")) + ? ["image-edit", "reference-images"] + : []), + ]; + if (mediaKind === "image" && supportedModes.length === 0) { + supportedModes.push("text-to-image"); + } const runtimeConfig: AtlasCloudRuntimeConfig = { submitEndpoint: getPostEndpoint(schema), @@ -409,12 +479,20 @@ export function createAtlasCloudImagePreset( avatarUrl: catalogModel.avatar || catalogModel.logo || undefined, description: catalogModel.profile || - `Atlas Cloud 图片模型 ${catalogModel.displayName || catalogModel.model}`, + `Atlas Cloud ${mediaKind === "3d" ? "3D" : mediaKind === "video" ? "视频" : "图片"}模型 ${catalogModel.displayName || catalogModel.model}`, supportsVision: Boolean(imageInputKey), - supportsImageGen: true, + supportsImageGen: mediaKind === "image", sortOrder: Math.max(0, 20_000 - index), capabilities: { - kind: imageInputKey ? "image-edit" : "text-to-image", + kind: + mediaKind === "3d" + ? "3d" + : mediaKind === "video" + ? "video" + : imageInputKey + ? "image-edit" + : "text-to-image", + mediaType: mediaKind, supportedModes, maxReferenceImages, supportsMultiImageReference: maxReferenceImages > 1, diff --git a/lib/ai/model-classification.ts b/lib/ai/model-classification.ts index c861c6947..8e45c9552 100644 --- a/lib/ai/model-classification.ts +++ b/lib/ai/model-classification.ts @@ -21,12 +21,17 @@ const STATIC_IMAGE_PROVIDER_TYPES = new Set([ "openai-image", "bananaimage", "visionary", - "atlascloud", "stepfun-image", "midjourney", "volcano-vision", ]); +const STATIC_3D_PROVIDER_TYPES = new Set([ + "tripo-3d", + "worldlabs-3d", + "kittycad-cad", +]); + const RUNWAY_IMAGE_MODEL_IDS = new Set([ "gen4_image", "gen4_image_turbo", @@ -47,10 +52,42 @@ const RUNWAY_VIDEO_MODEL_PREFIXES = [ const VIDEO_MODE_KEYS = new Set([ "text-to-video", + "image-to-video", + "video-to-video", + "audio-to-video", "first-frame", "first-last-frame", ]); +const VIDEO_KIND_KEYS = new Set([ + "video", + "text-to-video", + "image-to-video", + "video-to-video", + "audio-to-video", +]); + +const THREE_D_MODE_KEYS = new Set([ + "3d", + "text-to-3d", + "text_to_3d", + "image-to-3d", + "image_to_3d", + "mesh-generation", + "mesh_generation", + "cad", +]); + +const THREE_D_KIND_KEYS = new Set([ + "3d", + "model-3d", + "model_3d", + "mesh", + "mesh-generation", + "mesh_generation", + "cad", +]); + const TEXT_TO_IMAGE_MODE_KEYS = new Set([ "text-to-image", "text2image", @@ -107,6 +144,48 @@ function hasVideoModes(capabilities: unknown) { return supportedModes.some((mode) => VIDEO_MODE_KEYS.has(String(mode || "").trim())); } +function hasVideoKind(capabilities: unknown) { + return ["kind", "mediaType", "modelType", "outputType"].some((key) => + VIDEO_KIND_KEYS.has(getStringField(capabilities, key)) + ); +} + +function has3DModes(capabilities: unknown) { + const supportedModes = getStringArrayField(capabilities, "supportedModes"); + return supportedModes.some((mode) => THREE_D_MODE_KEYS.has(mode)); +} + +function has3DKind(capabilities: unknown) { + return ["kind", "mediaType", "modelType", "outputType"].some((key) => + THREE_D_KIND_KEYS.has(getStringField(capabilities, key)) + ); +} + +function modelIdLooks3D(modelId?: string | null) { + const normalized = normalizeText(modelId); + if (!normalized) return false; + return ( + /(?:^|[/_.-])(?:text|image)[_-]to[_-]3d(?:$|[/_.-])/.test(normalized) || + /(?:^|[/_.-])(?:hunyuan3d|seed3d|tripo-h3(?:\.\d+)?)(?:$|[/_.-])/.test( + normalized + ) + ); +} + +export function is3DModelDescriptor(params: { + providerType?: string | null; + modelId?: string | null; + capabilities?: unknown; +}): boolean { + const providerType = normalizeText(params.providerType); + return ( + STATIC_3D_PROVIDER_TYPES.has(providerType) || + has3DModes(params.capabilities) || + has3DKind(params.capabilities) || + modelIdLooks3D(params.modelId) + ); +} + export function inferRunwayModelKind(modelId?: string | null): "image" | "video" | null { const normalized = normalizeText(modelId); if (!normalized) return null; @@ -129,11 +208,15 @@ export function isVideoModelDescriptor(params: { }): boolean { const providerType = normalizeText(params.providerType); + if (is3DModelDescriptor(params)) { + return false; + } + if (isStaticVideoProviderType(providerType)) { return true; } - if (hasVideoModes(params.capabilities)) { + if (hasVideoModes(params.capabilities) || hasVideoKind(params.capabilities)) { return true; } @@ -156,6 +239,10 @@ export function isImageGenerationModelDescriptor(params: { }): boolean { const providerType = normalizeText(params.providerType); + if (is3DModelDescriptor(params)) { + return false; + } + if (providerType === "runway") { const runwayModelKind = inferRunwayModelKind(params.modelId); if (runwayModelKind === "image") return true; diff --git a/tests/routes/model-classification.test.ts b/tests/routes/model-classification.test.ts index 04cc84f1a..0289d8932 100644 --- a/tests/routes/model-classification.test.ts +++ b/tests/routes/model-classification.test.ts @@ -1,35 +1,95 @@ import { expect, test } from "@playwright/test"; import { + createAtlasCloudImagePreset, + inferAtlasCloudMediaKind, +} from "@/lib/ai/atlascloud-models"; +import { + is3DModelDescriptor, isImageGenerationModelDescriptor, isVideoModelDescriptor, } from "@/lib/ai/model-classification"; -test.describe("model classification", () => { - test("keeps multi-reference BananaImage models classified as image models", () => { +const atlasHunyuan3D = { + model: "tencent/hunyuan3d-pro/text-to-3d", + type: "Image", + displayName: "Hunyuan 3D Pro Text-to-3D", + categories: ["TEXT-TO-3D"], + tags: ["TEXT-TO-3D", "NEW"], +}; + +test.describe("AI model media classification", () => { + test("keeps Atlas 3D entries out of image and video categories", () => { + expect(inferAtlasCloudMediaKind(atlasHunyuan3D)).toBe("3d"); + const descriptor = { - providerType: "bananaimage", - modelId: "gemini-3-pro-image-preview", + providerType: "atlascloud", + modelId: atlasHunyuan3D.model, capabilities: { - supportedModes: [ - "text-to-image", - "image-to-image", - "reference-images", - ], + kind: "3d", + mediaType: "3d", + supportedModes: ["text-to-3d"], + }, + supportsImageGen: false, + }; + + expect(is3DModelDescriptor(descriptor)).toBe(true); + expect(isImageGenerationModelDescriptor(descriptor)).toBe(false); + expect(isVideoModelDescriptor(descriptor)).toBe(false); + }); + + test("writes 3D Atlas presets with model-level capabilities", () => { + const preset = createAtlasCloudImagePreset( + atlasHunyuan3D, + { + components: { + schemas: { + Input: { + type: "object", + properties: { + model: { type: "string" }, + prompt: { type: "string" }, + }, + required: ["model", "prompt"], + }, + }, + }, + }, + 0 + ); + + expect(preset.supportsImageGen).toBe(false); + expect(preset.capabilities).toMatchObject({ + kind: "3d", + mediaType: "3d", + supportedModes: ["text-to-3d"], + }); + }); + + test("still distinguishes Atlas image and video descriptors", () => { + const imageDescriptor = { + providerType: "atlascloud", + modelId: "black-forest-labs/flux-2-pro/text-to-image", + capabilities: { + kind: "text-to-image", + mediaType: "image", + supportedModes: ["text-to-image"], }, supportsImageGen: true, }; + const videoDescriptor = { + providerType: "atlascloud", + modelId: "veed/fabric-1.0/image-to-video", + capabilities: { + kind: "video", + mediaType: "video", + supportedModes: ["image-to-video"], + }, + supportsImageGen: false, + }; - expect(isVideoModelDescriptor(descriptor)).toBe(false); - expect(isImageGenerationModelDescriptor(descriptor)).toBe(true); - }); - - test("still recognizes explicit video generation modes", () => { - expect( - isVideoModelDescriptor({ - providerType: "custom", - modelId: "custom-video-model", - capabilities: { supportedModes: ["text-to-video"] }, - }) - ).toBe(true); + expect(isImageGenerationModelDescriptor(imageDescriptor)).toBe(true); + expect(isVideoModelDescriptor(imageDescriptor)).toBe(false); + expect(isVideoModelDescriptor(videoDescriptor)).toBe(true); + expect(isImageGenerationModelDescriptor(videoDescriptor)).toBe(false); }); }); From 758be188cf333363cf7a7f60253ac0b4d54cd767 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 21 Jul 2026 11:59:11 +0800 Subject: [PATCH 03/83] test: preserve image model classification coverage --- tests/routes/model-classification.test.ts | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/routes/model-classification.test.ts b/tests/routes/model-classification.test.ts index 0289d8932..c0b50796b 100644 --- a/tests/routes/model-classification.test.ts +++ b/tests/routes/model-classification.test.ts @@ -18,6 +18,34 @@ const atlasHunyuan3D = { }; test.describe("AI model media classification", () => { + test("keeps multi-reference BananaImage models classified as image models", () => { + const descriptor = { + providerType: "bananaimage", + modelId: "gemini-3-pro-image-preview", + capabilities: { + supportedModes: [ + "text-to-image", + "image-to-image", + "reference-images", + ], + }, + supportsImageGen: true, + }; + + expect(isVideoModelDescriptor(descriptor)).toBe(false); + expect(isImageGenerationModelDescriptor(descriptor)).toBe(true); + }); + + test("still recognizes explicit video generation modes", () => { + expect( + isVideoModelDescriptor({ + providerType: "custom", + modelId: "custom-video-model", + capabilities: { supportedModes: ["text-to-video"] }, + }) + ).toBe(true); + }); + test("keeps Atlas 3D entries out of image and video categories", () => { expect(inferAtlasCloudMediaKind(atlasHunyuan3D)).toBe("3d"); From 75cecdfcadbbb7d3b6aabfa00ca63fab15bc89e1 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 22 Jul 2026 10:25:32 +0800 Subject: [PATCH 04/83] fix: validate AtlasCloud frontend model requests --- app/(chat)/api/files/upload/route.ts | 20 +- components/canvas/CanvasChatPanel.tsx | 206 +++++++++++++- .../canvas/chat-panel/canvas-chat-session.ts | 4 + lib/ai/atlascloud-models.ts | 49 ++++ lib/ai/atlascloud-request.ts | 209 ++++++++++++++ lib/ai/providers/atlascloud.ts | 268 ++++-------------- scripts/sync-atlascloud-models.ts | 92 ++++++ scripts/verify-atlascloud-frontend.ts | 204 +++++++++++++ tests/routes/model-classification.test.ts | 68 +++++ 9 files changed, 908 insertions(+), 212 deletions(-) create mode 100644 lib/ai/atlascloud-request.ts create mode 100644 scripts/sync-atlascloud-models.ts create mode 100644 scripts/verify-atlascloud-frontend.ts diff --git a/app/(chat)/api/files/upload/route.ts b/app/(chat)/api/files/upload/route.ts index 6c11efe0b..0a5297cb1 100644 --- a/app/(chat)/api/files/upload/route.ts +++ b/app/(chat)/api/files/upload/route.ts @@ -18,6 +18,11 @@ const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([ "image/heic", "image/heif", ]); +const ALLOWED_ATLASCLOUD_VIDEO_TYPES = new Set([ + "video/mp4", + "video/quicktime", + "video/webm", +]); const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]); @@ -197,6 +202,9 @@ export async function POST(request: Request) { let filename = originalFilename; let contentType = originalContentType; const uploadPurpose = String(formData.get("purpose") || ""); + const isAtlasCloudReferenceVideo = + uploadPurpose === "canvas-video-process" && + ALLOWED_ATLASCLOUD_VIDEO_TYPES.has(contentType); const clientUploadId = String(formData.get("clientUploadId") || "").trim() || null; const isSvgUpload = contentType === "image/svg+xml"; @@ -227,7 +235,10 @@ export async function POST(request: Request) { return NextResponse.json({ error: errorMessage }, { status: 400 }); } - if (!ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType)) { + if ( + !ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) && + !isAtlasCloudReferenceVideo + ) { const errorMessage = createUnsupportedImageTypeMessage(contentType, filename); logger.warn("[Upload API] Unsupported upload content type", { contentType, @@ -361,18 +372,19 @@ export async function POST(request: Request) { try { const asset = await upsertMediaAsset({ userId: session.user.id, - mediaType: "image", + mediaType: isAtlasCloudReferenceVideo ? "video" : "image", sourceType: "upload", status: "ready", originalUrl: data.url, - previewUrl: data.url, - thumbnailUrl: data.url, + previewUrl: isAtlasCloudReferenceVideo ? null : data.url, + thumbnailUrl: isAtlasCloudReferenceVideo ? null : data.url, mimeType: data.contentType || contentType || null, displayName: filename, sourceTable: "upload", metadata: { pathname: data.pathname, size: optimizedBuffer.length, + purpose: uploadPurpose || null, clientUploadId, convertedFrom, }, diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 5a8000c56..8c35ab4a2 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -2197,6 +2197,39 @@ function isLikelyVideoAttachment(url: string, contentType: string) { ); } +function collectMessageVideoAttachments(message: ChatMessage) { + const attachments: PickedObjectAttachment[] = []; + const seenUrls = new Set(); + const append = (value: any, fallbackName: string) => { + const url = String(value?.url || value?.data || "").trim(); + if (!isValidChatHttpUrl(url) || seenUrls.has(url)) return; + const contentType = normalizeChatMediaType( + value?.contentType || value?.mediaType || value?.mimeType + ); + if (!isLikelyVideoAttachment(url, contentType)) return; + const attachment = sanitizeChatAttachment({ + url, + name: normalizeChatAttachmentName(value?.name || fallbackName), + contentType, + assetId: + typeof value?.assetId === "string" && value.assetId.trim() + ? value.assetId.trim() + : undefined, + } as PickedObjectAttachment); + if (!attachment) return; + seenUrls.add(url); + attachments.push(attachment); + }; + + (message.parts || []).forEach((part: any, index: number) => { + if (part?.type === "file") append(part, `参考视频-${index + 1}.mp4`); + }); + ((message as any).attachments || []).forEach((attachment: any, index: number) => { + append(attachment, `参考视频-${index + 1}.mp4`); + }); + return attachments; +} + function buildCopiedMessageAttachment( value: any, fallbackName: string @@ -4206,6 +4239,7 @@ export function CanvasChatPanel({ videoToolCapability?: VideoToolCapabilityContract | null; imageGenConfig?: { maxImages?: number; + maxVideoReferences?: number; supportsSize?: boolean; supports1K?: boolean; supports2K?: boolean; @@ -4234,6 +4268,7 @@ export function CanvasChatPanel({ isMultimodal?: boolean; imageGenConfig?: { maxImages?: number; + maxVideoReferences?: number; supportsSize?: boolean; supports1K?: boolean; supports2K?: boolean; @@ -4731,6 +4766,25 @@ export function CanvasChatPanel({ [visualModels] ); const currentModel = allModels.find((model) => model.id === selectedModelId); + const atlasVideoReferenceLimit = + currentModel?.providerType === "atlascloud" + ? Math.max( + 0, + Math.min( + 4, + Number( + currentModel.capabilities?.maxVideoReferences || + currentModel.imageGenConfig?.maxVideoReferences || + 0 + ) + ) + ) + : 0; + const supportsAtlasVideoReference = atlasVideoReferenceLimit > 0; + const requiresAtlasVideoReference = Boolean( + supportsAtlasVideoReference && + currentModel?.capabilities?.requiresVideoReference === true + ); const isCanvasVideoModel = useCallback( (model?: (typeof allModels)[number] | null) => Boolean( @@ -11295,6 +11349,20 @@ export function CanvasChatPanel({ let effectiveAttachments = attachments .map((attachment) => sanitizeChatAttachment(attachment)) .filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment)); + if (supportsAtlasVideoReference) { + let keptVideoCount = 0; + effectiveAttachments = effectiveAttachments.filter((attachment) => { + if (!isLikelyVideoAttachment(attachment.url, attachment.contentType)) { + return true; + } + keptVideoCount += 1; + return keptVideoCount <= atlasVideoReferenceLimit; + }); + if (requiresAtlasVideoReference && keptVideoCount === 0) { + toast.error("当前模型需要先上传 1 个参考视频"); + return; + } + } if (hasAgentUploadSlots) { if (effectiveAttachments.length > agentRequiredUploadCount) { effectiveAttachments = effectiveAttachments.slice(0, agentRequiredUploadCount); @@ -11918,6 +11986,57 @@ export function CanvasChatPanel({ } }, [clearComposerUploadProgress, updateComposerUploadProgress]); + const uploadVideoFileForComposerAttachment = useCallback(async ( + file: File, + progressOptions?: CanvasUploadProgressOptions + ): Promise => { + if (!file.type.startsWith("video/")) { + toast.error("请选择视频文件"); + return null; + } + if (file.size > 15 * 1024 * 1024) { + toast.error("参考视频大小不能超过 15MB"); + return null; + } + + try { + const formData = new FormData(); + formData.append("file", file); + formData.append("purpose", "canvas-video-process"); + if (progressOptions) { + updateComposerUploadProgress(progressOptions); + } + const response = await fetchUploadWithRetry( + formData, + progressOptions + ? { + onProgress: (progress) => { + updateComposerUploadProgress(progressOptions, progress); + }, + } + : undefined + ); + if (!response.ok) { + throw new Error("上传失败"); + } + + const data = await response.json(); + clearComposerUploadProgress(250); + return { + imageUrl: String(data.url || ""), + name: file.name || `参考视频-${Date.now()}.mp4`, + contentType: file.type || "video/mp4", + assetId: + typeof data.assetId === "string" && data.assetId.trim() + ? data.assetId.trim() + : undefined, + }; + } catch (_error) { + toast.error("参考视频上传失败"); + return null; + } + }, [clearComposerUploadProgress, updateComposerUploadProgress]); + const handlePastedImages = useCallback(async (files: File[]) => { if (files.length === 0) return; @@ -12250,6 +12369,58 @@ export function CanvasChatPanel({ e.target.value = ""; return; } + + if (supportsAtlasVideoReference) { + const selectedVideoFiles = selectedFiles.filter((file) => + file.type.startsWith("video/") + ); + const existingVideoCount = attachments.filter((attachment) => + isLikelyVideoAttachment(attachment.url, attachment.contentType) + ).length; + const availableVideoSlots = Math.max( + 0, + atlasVideoReferenceLimit - existingVideoCount + ); + const acceptedVideoFiles = selectedVideoFiles.slice(0, availableVideoSlots); + + if (selectedVideoFiles.length > acceptedVideoFiles.length) { + toast.info(`当前模型最多支持 ${atlasVideoReferenceLimit} 个参考视频`); + } + if (acceptedVideoFiles.length > 0) { + let uploadedCount = 0; + try { + for (const [index, file] of acceptedVideoFiles.entries()) { + const uploaded = await uploadVideoFileForComposerAttachment(file, { + title: "正在上传参考视频", + fileName: file.name, + fileIndex: index, + fileTotal: acceptedVideoFiles.length, + target: "composer-reference", + }); + if (!uploaded?.imageUrl) continue; + addUploadedImageAttachmentToComposer(uploaded); + uploadedCount += 1; + } + } finally { + clearComposerUploadProgress(450); + } + if (uploadedCount > 0) { + toast.success( + uploadedCount === 1 + ? "参考视频已添加到对话" + : `已添加 ${uploadedCount} 个参考视频` + ); + } + } + + selectedFiles = selectedFiles.filter( + (file) => !file.type.startsWith("video/") + ); + if (selectedFiles.length === 0) { + e.target.value = ""; + return; + } + } let firstImageUrl: string | null = null; // 记录本次上传开始时的节点数量，用于计算新图片位置 @@ -13802,6 +13973,10 @@ export function CanvasChatPanel({ message.role === "user" ? collectMessageImageAttachments(message as any) : []; + const userVideoAttachments = + message.role === "user" + ? collectMessageVideoAttachments(message as any) + : []; const assistantTextParts = message.role === "assistant" ? (message.parts || []).filter((part: any) => part.type === "text" && part.text) : []; @@ -13954,6 +14129,25 @@ export function CanvasChatPanel({ ); }); + userVideoAttachments.forEach((attachment, idx) => { + renderedUserParts.push( + + + ); + }); + message.parts?.forEach((part: any, idx: number) => { if (part.type === 'text' && part.text) { const text = String(part.text) @@ -15619,8 +15813,8 @@ export function CanvasChatPanel({ className="relative flex h-8 w-8 shrink-0 items-center justify-center overflow-hidden rounded-full border border-dashed border-[#d9dde4] bg-[#f3f3f4] text-[#8f97a3] transition-colors hover:border-[#c6ccd6] hover:bg-[#ececee] hover:text-[#20242a] disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/12 dark:bg-white/8 dark:text-zinc-400 dark:hover:border-white/20 dark:hover:bg-white/12 dark:hover:text-white" onClick={() => fileInputRef.current?.click()} disabled={isSubmitting} - title="添加图片" - aria-label="添加图片" + title={supportsAtlasVideoReference ? "添加图片或参考视频" : "添加图片"} + aria-label={supportsAtlasVideoReference ? "添加图片或参考视频" : "添加图片"} > {composerReferenceUploadProgress ? ( @@ -15926,7 +16120,13 @@ export function CanvasChatPanel({ ; imageInputKey?: string; imageInputMode?: "array" | "single"; + videoClipInput?: { + key: string; + maxItems: number; + required: boolean; + itemDefaults: Record; + }; resolutionKey?: string; aspectRatioKey?: string; outputCountKey?: string; aspectRatioValueMap: Record; maxReferenceImages: number; + maxVideoReferences: number; maxImages: number; defaults: Record; enums: Record>; @@ -196,6 +203,39 @@ function getImageInputs(properties: JsonRecord, requiredFields: string[]) { .sort((a, b) => Number(b.required) - Number(a.required)); } +function getVideoClipInput(properties: JsonRecord, requiredFields: string[]) { + const entry = Object.entries(properties).find(([key, property]) => { + const normalizedKey = key.toLowerCase(); + const record = asRecord(property); + const itemProperties = asRecord(asRecord(record.items).properties); + return ( + record.type === "array" && + normalizedKey.includes("video") && + ("url" in itemProperties || normalizedKey.includes("clip")) + ); + }); + if (!entry) return undefined; + + const [key, property] = entry; + const record = asRecord(property); + const itemProperties = asRecord(asRecord(record.items).properties); + const itemDefaults: Record = {}; + for (const [itemKey, itemProperty] of Object.entries(itemProperties)) { + if (itemKey === "url") continue; + const value = asRecord(itemProperty).default; + if (["string", "number", "boolean"].includes(typeof value)) { + itemDefaults[itemKey] = value; + } + } + + return { + key, + maxItems: Math.max(1, Math.min(4, Number(record.maxItems || 1))), + required: requiredFields.includes(key), + itemDefaults, + }; +} + function getEnumValues(property: unknown) { const record = asRecord(property); const values = Array.isArray(record.enum) ? record.enum : []; @@ -396,6 +436,7 @@ export function createAtlasCloudImagePreset( const properties = asRecord(input.properties); const requiredFields = uniqueStrings(Array.isArray(input.required) ? input.required : []); const imageInputs = getImageInputs(properties, requiredFields); + const videoClipInput = getVideoClipInput(properties, requiredFields); const imageInputKey = imageInputs[0]?.key; const resolutionKey = RESOLUTION_KEYS.find((key) => { if (!(key in properties)) return false; @@ -420,6 +461,7 @@ export function createAtlasCloudImagePreset( 14, imageInputs.reduce((total, input) => total + input.maxItems, 0) ); + const maxVideoReferences = videoClipInput?.maxItems || 0; const maxImages = deriveMaxImages(properties, outputCountKey); const categories = uniqueStrings(catalogModel.categories || []).map((value) => value.toLowerCase() @@ -436,6 +478,7 @@ export function createAtlasCloudImagePreset( categories.some((value) => value.includes("image-to-image")) ? ["image-edit", "reference-images"] : []), + ...(videoClipInput ? ["reference-video"] : []), ]; if (mediaKind === "image" && supportedModes.length === 0) { supportedModes.push("text-to-image"); @@ -449,11 +492,13 @@ export function createAtlasCloudImagePreset( imageInputs, imageInputKey, imageInputMode: imageProperty.type === "array" ? "array" : "single", + videoClipInput, resolutionKey, aspectRatioKey, outputCountKey, aspectRatioValueMap: aspectRatioConfig.map, maxReferenceImages, + maxVideoReferences, maxImages, defaults: buildScalarDefaults(properties), enums: buildEnums(properties), @@ -495,7 +540,10 @@ export function createAtlasCloudImagePreset( mediaType: mediaKind, supportedModes, maxReferenceImages, + maxVideoReferences, supportsMultiImageReference: maxReferenceImages > 1, + supportsVideoReference: maxVideoReferences > 0, + requiresVideoReference: videoClipInput?.required === true, provider: "atlascloud", schemaUrl: catalogModel.schema || null, }, @@ -509,6 +557,7 @@ export function createAtlasCloudImagePreset( apiFormat: "atlascloud-unified-image", maxImages, maxReferenceImages, + maxVideoReferences, supportsSize, supports1K: supportedImageSizes.includes("1K"), supports2K: supportedImageSizes.includes("2K"), diff --git a/lib/ai/atlascloud-request.ts b/lib/ai/atlascloud-request.ts new file mode 100644 index 000000000..4607c4853 --- /dev/null +++ b/lib/ai/atlascloud-request.ts @@ -0,0 +1,209 @@ +import type { AtlasCloudRuntimeConfig } from "@/lib/ai/atlascloud-models"; + +export interface AtlasCloudImageParams { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + negativePrompt?: string; + promptExtend?: boolean; + quality?: string; + seed?: number; + watermark?: boolean; +} + +export interface AtlasCloudModelConfig { + maxReferenceImages?: number; + maxVideoReferences?: number; + supportedAspectRatios?: string[]; + supportedImageSizes?: string[]; + atlasCloud?: AtlasCloudRuntimeConfig; +} + +function normalizeAspectRatio( + requested: string | undefined, + supported: string[] | undefined +) { + const values = (supported || []).map(String); + const normalized = String(requested || "auto").trim().toLowerCase(); + const exact = values.find((value) => value.toLowerCase() === normalized); + if (exact) return exact; + return ( + values.find((value) => value === "auto") || + values.find((value) => value === "1:1") || + values[0] || + "1:1" + ); +} + +function normalizeImageSize(requested: string | undefined, supported?: string[]) { + const values = (supported || []).map((value) => String(value).trim().toUpperCase()); + const normalized = String(requested || "2K").trim().toUpperCase(); + if (values.includes(normalized)) return normalized; + return values.includes("2K") ? "2K" : values[0] || normalized; +} + +function matchEnumValue(values: Array | undefined, value: unknown) { + if (!values?.length) return value; + const normalized = String(value || "").trim().toLowerCase(); + return values.find( + (item) => String(item).trim().toLowerCase() === normalized + ); +} + +function parseRatio(value: string) { + const match = value.match(/^(\d+):(\d+)$/); + if (!match) return 1; + return Number(match[1]) / Number(match[2]); +} + +function pickClosestSizeValue( + map: Record>, + imageSize: string, + aspectRatio: string +) { + const tiers = [imageSize, "2K", "1K", "4K"]; + const values = tiers.map((tier) => map[tier]).find(Boolean); + if (!values) return undefined; + if (values[aspectRatio]) return values[aspectRatio]; + const target = parseRatio(aspectRatio); + return Object.entries(values).sort( + ([a], [b]) => + Math.abs(Math.log(target / parseRatio(a))) - + Math.abs(Math.log(target / parseRatio(b))) + )[0]?.[1]; +} + +export function createAtlasCloudRequestBody(params: { + modelId: string; + prompt: string; + referenceImages: string[]; + referenceVideos?: string[]; + imageParams?: AtlasCloudImageParams; + modelConfig: AtlasCloudModelConfig; +}) { + const runtime = params.modelConfig.atlasCloud; + if (!runtime) { + throw new Error("Atlas Cloud 模型运行参数缺失，请重新同步模型"); + } + const body: Record = { ...runtime.defaults }; + if ( + runtime.propertyNames.includes("model") || + runtime.requiredFields.includes("model") + ) { + body.model = params.modelId; + } + if ( + runtime.propertyNames.includes("prompt") || + runtime.requiredFields.includes("prompt") + ) { + body.prompt = params.prompt; + } + + const aspectRatio = normalizeAspectRatio( + params.imageParams?.aspectRatio, + params.modelConfig.supportedAspectRatios + ); + const imageSize = normalizeImageSize( + params.imageParams?.imageSize, + params.modelConfig.supportedImageSizes + ); + + if (runtime.aspectRatioKey) { + body[runtime.aspectRatioKey] = + runtime.aspectRatioValueMap?.[aspectRatio] || + matchEnumValue(runtime.enums[runtime.aspectRatioKey], aspectRatio) || + aspectRatio; + } + if (runtime.resolutionKey) { + body[runtime.resolutionKey] = + matchEnumValue(runtime.enums[runtime.resolutionKey], imageSize.toLowerCase()) || + matchEnumValue(runtime.enums[runtime.resolutionKey], imageSize) || + imageSize.toLowerCase(); + } else { + const sizeValue = pickClosestSizeValue( + runtime.sizeValueMap, + imageSize, + aspectRatio === "auto" ? "1:1" : aspectRatio + ); + if (sizeValue) body.size = sizeValue; + } + + const imageInputs = runtime.imageInputs?.length + ? runtime.imageInputs + : runtime.imageInputKey + ? [ + { + key: runtime.imageInputKey, + mode: runtime.imageInputMode || ("single" as const), + maxItems: Math.max(1, runtime.maxReferenceImages || 1), + required: runtime.requiredFields.includes(runtime.imageInputKey), + }, + ] + : []; + let referenceCursor = 0; + for (const input of imageInputs) { + const values = params.referenceImages.slice( + referenceCursor, + referenceCursor + input.maxItems + ); + if (values.length > 0) { + body[input.key] = input.mode === "single" ? values[0] : values; + referenceCursor += values.length; + } else if (input.required) { + throw new Error( + `该 Atlas Cloud 模型至少需要 ${referenceCursor + 1} 张参考图` + ); + } + } + if (imageInputs.length === 0 && params.referenceImages.length > 0) { + throw new Error("当前 Atlas Cloud 模型不支持参考图，请选择图片编辑模型"); + } + + const referenceVideos = params.referenceVideos || []; + if (runtime.videoClipInput) { + const values = referenceVideos.slice(0, runtime.videoClipInput.maxItems); + if (values.length > 0) { + body[runtime.videoClipInput.key] = values.map((url) => ({ + ...runtime.videoClipInput?.itemDefaults, + url, + })); + } else if (runtime.videoClipInput.required) { + throw new Error("该 Atlas Cloud 模型至少需要 1 个参考视频"); + } + } else if (referenceVideos.length > 0) { + throw new Error("当前 Atlas Cloud 模型不支持参考视频"); + } + + if (runtime.outputCountKey) { + const requestedCount = Math.max(1, params.imageParams?.imageCount || 1); + body[runtime.outputCountKey] = Math.min(runtime.maxImages, requestedCount); + } + const optionalParams: Array<[string, unknown]> = [ + ["negative_prompt", params.imageParams?.negativePrompt], + ["prompt_extend", params.imageParams?.promptExtend], + ["quality", params.imageParams?.quality], + ["seed", params.imageParams?.seed], + ["watermark", params.imageParams?.watermark], + ]; + for (const [key, value] of optionalParams) { + if (value !== undefined && runtime.propertyNames.includes(key)) { + body[key] = matchEnumValue(runtime.enums[key], value) ?? value; + } + } + if (runtime.propertyNames.includes("enable_sync_mode")) { + body.enable_sync_mode = false; + } + if (runtime.propertyNames.includes("enable_base64_output")) { + body.enable_base64_output = false; + } + + const unsupportedRequiredFields = runtime.requiredFields.filter( + (key) => body[key] === undefined + ); + if (unsupportedRequiredFields.length > 0) { + throw new Error( + `当前界面暂不支持该 Atlas Cloud 模型的必填参数：${unsupportedRequiredFields.join(", ")}` + ); + } + return { body, imageSize }; +} diff --git a/lib/ai/providers/atlascloud.ts b/lib/ai/providers/atlascloud.ts index e55f82e09..81a0c79f2 100644 --- a/lib/ai/providers/atlascloud.ts +++ b/lib/ai/providers/atlascloud.ts @@ -1,10 +1,12 @@ import "server-only"; import sharp from "sharp"; +import { ATLASCLOUD_BASE_URL } from "@/lib/ai/atlascloud-models"; import { - ATLASCLOUD_BASE_URL, - type AtlasCloudRuntimeConfig, -} from "@/lib/ai/atlascloud-models"; + createAtlasCloudRequestBody, + type AtlasCloudImageParams, + type AtlasCloudModelConfig, +} from "@/lib/ai/atlascloud-request"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; @@ -13,24 +15,6 @@ interface AtlasCloudConfig { baseURL?: string; } -interface AtlasCloudImageParams { - aspectRatio?: string; - imageSize?: string; - imageCount?: number; - negativePrompt?: string; - promptExtend?: boolean; - quality?: string; - seed?: number; - watermark?: boolean; -} - -interface AtlasCloudModelConfig { - maxReferenceImages?: number; - supportedAspectRatios?: string[]; - supportedImageSizes?: string[]; - atlasCloud?: AtlasCloudRuntimeConfig; -} - interface UploadedImage { thumbnail: string; original: string; @@ -116,18 +100,19 @@ function extractOutputUrls(payload: any) { return urls; } -function extractPromptAndImages( +function extractPromptAndMedia( messages: Array<{ role: string; content?: any; parts?: any[] }> ) { const message = messages.filter((item) => item.role === "user").pop(); if (!message) throw new Error("未找到用户消息"); const texts: string[] = []; const imageUrls: string[] = []; + const videoUrls: string[] = []; const pushText = (value: unknown) => { if (typeof value === "string" && value.trim()) texts.push(value.trim()); }; - const pushImage = (value: unknown) => { + const pushMedia = (value: unknown, mediaType?: unknown) => { const url = String( typeof value === "string" ? value @@ -137,24 +122,31 @@ function extractPromptAndImages( (value as any)?.src || "" ).trim(); - if ( - url && - (/^https?:\/\//i.test(url) || url.startsWith("data:image/")) && - !imageUrls.includes(url) - ) { - imageUrls.push(url); - } + if (!url || !/^(?:https?:\/\/|data:)/i.test(url)) return; + const normalizedMediaType = String(mediaType || "").toLowerCase(); + const isVideo = + normalizedMediaType.startsWith("video/") || + url.startsWith("data:video/") || + /\.(?:mp4|mov|webm|m4v)(?:[?#].*)?$/i.test(url); + const target = isVideo ? videoUrls : imageUrls; + if (!target.includes(url)) target.push(url); }; const visitPart = (part: any) => { if (!part) return; if (typeof part === "string") return pushText(part); if (part.type === "text" || typeof part.text === "string") pushText(part.text); - if (part.type === "image_url") pushImage(part.image_url); - if (part.type === "image") pushImage(part.image || part.url); - if (part.type === "file") pushImage(part.url || part.file || part.data); + if (part.type === "image_url") pushMedia(part.image_url, "image/jpeg"); + if (part.type === "image") pushMedia(part.image || part.url, "image/jpeg"); + if (part.type === "file") { + pushMedia( + part.url || part.file || part.data, + part.mediaType || part.contentType || part.mimeType + ); + } if (part.inlineData?.data) { - pushImage( - `data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}` + pushMedia( + `data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}`, + part.inlineData.mimeType ); } }; @@ -165,31 +157,31 @@ function extractPromptAndImages( const prompt = texts.join("\n").trim(); if (!prompt) throw new Error("请输入图片生成或编辑要求"); - return { prompt, imageUrls }; + return { prompt, imageUrls, videoUrls }; } function parseDataUrl(value: string) { const match = value.match(/^data:([^;,]+);base64,(.+)$/s); - if (!match) throw new Error("Atlas Cloud 参考图 Base64 格式无效"); + if (!match) throw new Error("Atlas Cloud 参考素材 Base64 格式无效"); return { mimeType: match[1] || "image/png", buffer: Buffer.from(match[2] || "", "base64"), }; } -async function loadImageSource(imageUrl: string) { - if (imageUrl.startsWith("data:")) return parseDataUrl(imageUrl); - const response = await fetch(imageUrl, { +async function loadMediaSource(mediaUrl: string) { + if (mediaUrl.startsWith("data:")) return parseDataUrl(mediaUrl); + const response = await fetch(mediaUrl, { cache: "no-store", redirect: "follow", signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS), headers: { - Accept: "image/avif,image/webp,image/png,image/jpeg,image/*,*/*;q=0.8", + Accept: "image/*,video/*,*/*;q=0.8", "User-Agent": "yedir-ai-atlascloud/1.0", }, }); if (!response.ok) { - throw new Error(`Atlas Cloud 参考图下载失败（HTTP ${response.status}）`); + throw new Error(`Atlas Cloud 参考素材下载失败（HTTP ${response.status}）`); } return { mimeType: response.headers.get("content-type")?.split(";")[0] || "image/png", @@ -201,16 +193,19 @@ function detectExtension(mimeType: string) { if (/jpe?g/i.test(mimeType)) return "jpg"; if (/webp/i.test(mimeType)) return "webp"; if (/gif/i.test(mimeType)) return "gif"; + if (/quicktime/i.test(mimeType)) return "mov"; + if (/webm/i.test(mimeType)) return "webm"; + if (/mp4/i.test(mimeType)) return "mp4"; return "png"; } -async function uploadReferenceImage( +async function uploadReferenceMedia( baseURL: string | undefined, apiKey: string, - imageUrl: string + mediaUrl: string ) { - if (/^https:\/\//i.test(imageUrl)) return imageUrl; - const source = await loadImageSource(imageUrl); + if (/^https:\/\//i.test(mediaUrl)) return mediaUrl; + const source = await loadMediaSource(mediaUrl); const form = new FormData(); form.append( "file", @@ -226,171 +221,17 @@ async function uploadReferenceImage( const payload = await parseJsonResponse(response); if (!response.ok) { throw new Error( - `${extractErrorMessage(payload, "Atlas Cloud 参考图上传失败")}（HTTP ${response.status}）` + `${extractErrorMessage(payload, "Atlas Cloud 参考素材上传失败")}（HTTP ${response.status}）` ); } const data = payload?.data && typeof payload.data === "object" ? payload.data : payload; const url = String( data?.url || data?.file_url || data?.fileUrl || data?.media_url || data?.mediaUrl || "" ).trim(); - if (!url) throw new Error("Atlas Cloud 参考图上传成功但未返回文件地址"); + if (!url) throw new Error("Atlas Cloud 参考素材上传成功但未返回文件地址"); return url; } -function normalizeAspectRatio( - requested: string | undefined, - supported: string[] | undefined -) { - const values = (supported || []).map(String); - const normalized = String(requested || "auto").trim().toLowerCase(); - const exact = values.find((value) => value.toLowerCase() === normalized); - if (exact) return exact; - return values.find((value) => value === "auto") || values.find((value) => value === "1:1") || values[0] || "1:1"; -} - -function normalizeImageSize(requested: string | undefined, supported?: string[]) { - const values = (supported || []).map((value) => String(value).trim().toUpperCase()); - const normalized = String(requested || "2K").trim().toUpperCase(); - if (values.includes(normalized)) return normalized; - return values.includes("2K") ? "2K" : values[0] || normalized; -} - -function matchEnumValue(values: Array | undefined, value: unknown) { - if (!values?.length) return value; - const normalized = String(value || "").trim().toLowerCase(); - return values.find((item) => String(item).trim().toLowerCase() === normalized); -} - -function parseRatio(value: string) { - const match = value.match(/^(\d+):(\d+)$/); - if (!match) return 1; - return Number(match[1]) / Number(match[2]); -} - -function pickClosestSizeValue( - map: Record>, - imageSize: string, - aspectRatio: string -) { - const tiers = [imageSize, "2K", "1K", "4K"]; - const values = tiers.map((tier) => map[tier]).find(Boolean); - if (!values) return undefined; - if (values[aspectRatio]) return values[aspectRatio]; - const target = parseRatio(aspectRatio); - return Object.entries(values).sort( - ([a], [b]) => - Math.abs(Math.log(target / parseRatio(a))) - - Math.abs(Math.log(target / parseRatio(b))) - )[0]?.[1]; -} - -function createRequestBody(params: { - modelId: string; - prompt: string; - referenceImages: string[]; - imageParams?: AtlasCloudImageParams; - modelConfig: AtlasCloudModelConfig; -}) { - const runtime = params.modelConfig.atlasCloud; - if (!runtime) throw new Error("Atlas Cloud 模型运行参数缺失，请重新同步模型"); - const body: Record = { - ...runtime.defaults, - }; - if (runtime.propertyNames.includes("model")) body.model = params.modelId; - if (runtime.propertyNames.includes("prompt")) body.prompt = params.prompt; - const aspectRatio = normalizeAspectRatio( - params.imageParams?.aspectRatio, - params.modelConfig.supportedAspectRatios - ); - const imageSize = normalizeImageSize( - params.imageParams?.imageSize, - params.modelConfig.supportedImageSizes - ); - - if (runtime.aspectRatioKey) { - body[runtime.aspectRatioKey] = - runtime.aspectRatioValueMap?.[aspectRatio] || - matchEnumValue(runtime.enums[runtime.aspectRatioKey], aspectRatio) || aspectRatio; - } - if (runtime.resolutionKey) { - const resolutionValue = - matchEnumValue(runtime.enums[runtime.resolutionKey], imageSize.toLowerCase()) || - matchEnumValue(runtime.enums[runtime.resolutionKey], imageSize) || - imageSize.toLowerCase(); - body[runtime.resolutionKey] = resolutionValue; - } else { - const sizeValue = pickClosestSizeValue( - runtime.sizeValueMap, - imageSize, - aspectRatio === "auto" ? "1:1" : aspectRatio - ); - if (sizeValue) body.size = sizeValue; - } - - const imageInputs = runtime.imageInputs?.length - ? runtime.imageInputs - : runtime.imageInputKey - ? [ - { - key: runtime.imageInputKey, - mode: runtime.imageInputMode || "single", - maxItems: Math.max(1, runtime.maxReferenceImages || 1), - required: runtime.requiredFields.includes(runtime.imageInputKey), - }, - ] - : []; - let referenceCursor = 0; - for (const input of imageInputs) { - const values = params.referenceImages.slice( - referenceCursor, - referenceCursor + input.maxItems - ); - if (values.length > 0) { - body[input.key] = input.mode === "single" ? values[0] : values; - referenceCursor += values.length; - } else if (input.required) { - throw new Error( - `该 Atlas Cloud 模型至少需要 ${referenceCursor + 1} 张参考图` - ); - } - } - if (imageInputs.length === 0 && params.referenceImages.length > 0) { - throw new Error("当前 Atlas Cloud 模型不支持参考图，请选择图片编辑模型"); - } - - if (runtime.outputCountKey) { - const requestedCount = Math.max(1, params.imageParams?.imageCount || 1); - body[runtime.outputCountKey] = Math.min(runtime.maxImages, requestedCount); - } - const optionalParams: Array<[string, unknown]> = [ - ["negative_prompt", params.imageParams?.negativePrompt], - ["prompt_extend", params.imageParams?.promptExtend], - ["quality", params.imageParams?.quality], - ["seed", params.imageParams?.seed], - ["watermark", params.imageParams?.watermark], - ]; - for (const [key, value] of optionalParams) { - if (value !== undefined && runtime.propertyNames.includes(key)) { - body[key] = matchEnumValue(runtime.enums[key], value) ?? value; - } - } - if (runtime.propertyNames.includes("enable_sync_mode")) { - body.enable_sync_mode = false; - } - if (runtime.propertyNames.includes("enable_base64_output")) { - body.enable_base64_output = false; - } - const unsupportedRequiredFields = runtime.requiredFields.filter( - (key) => body[key] === undefined - ); - if (unsupportedRequiredFields.length > 0) { - throw new Error( - `当前界面暂不支持该 Atlas Cloud 模型的必填参数：${unsupportedRequiredFields.join(", ")}` - ); - } - return { body, imageSize }; -} - async function submitTask(params: { apiKey: string; baseURL?: string; @@ -540,7 +381,9 @@ export function createAtlasCloud({ apiKey, baseURL }: AtlasCloudConfig) { if (!modelConfig.atlasCloud) { throw new Error("Atlas Cloud 模型能力配置缺失，请在系统设置中重新同步模型"); } - const { prompt, imageUrls } = extractPromptAndImages(options.messages); + const { prompt, imageUrls, videoUrls } = extractPromptAndMedia( + options.messages + ); const maxReferenceImages = Math.max( 0, Math.min(14, modelConfig.maxReferenceImages ?? modelConfig.atlasCloud.maxReferenceImages) @@ -548,12 +391,27 @@ export function createAtlasCloud({ apiKey, baseURL }: AtlasCloudConfig) { const referenceImages = await Promise.all( imageUrls .slice(0, maxReferenceImages) - .map((url) => uploadReferenceImage(baseURL, apiKey, url)) + .map((url) => uploadReferenceMedia(baseURL, apiKey, url)) ); - const { body, imageSize } = createRequestBody({ + const maxVideoReferences = Math.max( + 0, + Math.min( + 4, + modelConfig.maxVideoReferences ?? + modelConfig.atlasCloud.maxVideoReferences ?? + 0 + ) + ); + const referenceVideos = await Promise.all( + videoUrls + .slice(0, maxVideoReferences) + .map((url) => uploadReferenceMedia(baseURL, apiKey, url)) + ); + const { body, imageSize } = createAtlasCloudRequestBody({ modelId: runtimeModelId, prompt, referenceImages, + referenceVideos, imageParams: options.imageParams, modelConfig, }); diff --git a/scripts/sync-atlascloud-models.ts b/scripts/sync-atlascloud-models.ts new file mode 100644 index 000000000..0455116e6 --- /dev/null +++ b/scripts/sync-atlascloud-models.ts @@ -0,0 +1,92 @@ +import { and, eq } from "drizzle-orm"; +import { fetchAtlasCloudImagePresets } from "../lib/ai/atlascloud-models"; +import { db } from "../lib/db"; +import { aiModel, aiProvider } from "../lib/db/schema"; + +async function main() { + const providers = await db + .select() + .from(aiProvider) + .where(and(eq(aiProvider.type, "atlascloud"), eq(aiProvider.isEnabled, true))); + if (providers.length === 0) { + throw new Error("没有找到已启用的 AtlasCloud 服务商"); + } + + const catalog = await fetchAtlasCloudImagePresets(); + if (catalog.errors.length > 0) { + throw new Error( + `AtlasCloud 目录有 ${catalog.errors.length} 个 schema 加载失败：${catalog.errors + .map((item) => item.modelId) + .join(", ")}` + ); + } + + const summaries = []; + for (const provider of providers) { + let added = 0; + let updated = 0; + for (const preset of catalog.presets) { + const existing = await db.query.aiModel.findFirst({ + where: and( + eq(aiModel.providerId, provider.id), + eq(aiModel.modelId, preset.modelId) + ), + }); + const values = { + displayName: preset.displayName, + avatarUrl: preset.avatarUrl || null, + description: preset.description, + supportsVision: preset.supportsVision, + supportsImageGen: preset.supportsImageGen, + isEnabled: true, + isHidden: false, + sortOrder: preset.sortOrder, + capabilities: preset.capabilities, + defaultParams: preset.defaultParams, + imageGenConfig: preset.imageGenConfig, + updatedAt: new Date(), + }; + + if (existing) { + await db + .update(aiModel) + .set(values) + .where(eq(aiModel.id, existing.id)); + updated += 1; + } else { + await db.insert(aiModel).values({ + providerId: provider.id, + modelId: preset.modelId, + ...values, + isDefault: false, + isSubModel: false, + parentModelId: null, + }); + added += 1; + } + } + summaries.push({ + providerId: provider.id, + providerName: provider.name, + added, + updated, + }); + } + + console.log( + JSON.stringify( + { + catalogModels: catalog.catalogCount, + availableModels: catalog.presets.length, + providers: summaries, + }, + null, + 2 + ) + ); +} + +void main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/scripts/verify-atlascloud-frontend.ts b/scripts/verify-atlascloud-frontend.ts new file mode 100644 index 000000000..2bfe269bd --- /dev/null +++ b/scripts/verify-atlascloud-frontend.ts @@ -0,0 +1,204 @@ +import { fetchAtlasCloudImagePresets } from "../lib/ai/atlascloud-models"; +import { createAtlasCloudRequestBody } from "../lib/ai/atlascloud-request"; +import { + is3DModelDescriptor, + isImageGenerationModelDescriptor, + isVideoModelDescriptor, +} from "../lib/ai/model-classification"; + +const FAKE_IMAGES = Array.from( + { length: 14 }, + (_, index) => `https://example.com/reference-${index + 1}.png` +); +const FAKE_VIDEOS = Array.from( + { length: 4 }, + (_, index) => `https://example.com/reference-${index + 1}.mp4` +); + +type Failure = { + modelId: string; + scenario: string; + error: string; +}; + +function validateBody( + body: Record, + runtime: any +) { + const missing = runtime.requiredFields.filter( + (key: string) => body[key] === undefined + ); + if (missing.length > 0) { + throw new Error(`缺少必填参数：${missing.join(", ")}`); + } + for (const [key, values] of Object.entries( + runtime.enums as Record> + )) { + if (body[key] === undefined || !values.length) continue; + if (!values.some((value) => String(value) === String(body[key]))) { + throw new Error(`${key}=${String(body[key])} 不在官方枚举中`); + } + } + if (runtime.videoClipInput && body[runtime.videoClipInput.key] !== undefined) { + const clips = body[runtime.videoClipInput.key]; + if (!Array.isArray(clips) || clips.length === 0) { + throw new Error("video_clips 请求格式无效"); + } + for (const clip of clips) { + if (!clip || typeof clip !== "object" || !("url" in clip)) { + throw new Error("video_clips 缺少 url"); + } + } + } +} + +async function main() { + const catalog = await fetchAtlasCloudImagePresets(); + const failures: Failure[] = catalog.errors.map((item) => ({ + modelId: item.modelId, + scenario: "catalog-schema", + error: item.error, + })); + let simulatedCases = 0; + let imageModels = 0; + let model3dModels = 0; + let videoReferenceModels = 0; + + for (const preset of catalog.presets) { + const capabilities = preset.capabilities as Record; + const descriptor = { + providerType: "atlascloud", + modelId: preset.modelId, + capabilities, + supportsImageGen: preset.supportsImageGen, + }; + const mediaType = String(capabilities.mediaType || ""); + + if (mediaType === "3d") { + model3dModels += 1; + simulatedCases += 1; + if ( + !is3DModelDescriptor(descriptor) || + isImageGenerationModelDescriptor(descriptor) || + isVideoModelDescriptor(descriptor) + ) { + failures.push({ + modelId: preset.modelId, + scenario: "3d-classification", + error: "3D 模型被错误归入图片或视频模型", + }); + } + continue; + } + + imageModels += 1; + if ( + !isImageGenerationModelDescriptor(descriptor) || + is3DModelDescriptor(descriptor) || + isVideoModelDescriptor(descriptor) + ) { + failures.push({ + modelId: preset.modelId, + scenario: "image-classification", + error: "图片模型分类不正确", + }); + continue; + } + + const imageGenConfig = preset.imageGenConfig as Record; + const runtime = imageGenConfig.atlasCloud; + if (!runtime) { + failures.push({ + modelId: preset.modelId, + scenario: "runtime-config", + error: "缺少 AtlasCloud 运行配置", + }); + continue; + } + + const ratios = imageGenConfig.supportedAspectRatios?.length + ? imageGenConfig.supportedAspectRatios + : [String(preset.defaultParams.aspectRatio || "1:1")]; + const sizes = imageGenConfig.supportedImageSizes?.length + ? imageGenConfig.supportedImageSizes + : [String(preset.defaultParams.imageSize || "1K")]; + const counts = runtime.outputCountKey && runtime.maxImages > 1 + ? [1, runtime.maxImages] + : [1]; + const requiredImageCount = runtime.imageInputs + .filter((input: any) => input.required) + .reduce((total: number, input: any) => total + input.maxItems, 0); + const imageReferenceCounts = new Set([ + requiredImageCount, + ...(runtime.maxReferenceImages > requiredImageCount + ? [runtime.maxReferenceImages] + : []), + ]); + const requiredVideoCount = runtime.videoClipInput?.required + ? runtime.videoClipInput.maxItems + : 0; + const videoReferenceCounts = new Set([ + requiredVideoCount, + ...(runtime.maxVideoReferences > requiredVideoCount + ? [runtime.maxVideoReferences] + : []), + ]); + if (runtime.maxVideoReferences > 0) videoReferenceModels += 1; + + for (const ratio of ratios) { + for (const size of sizes) { + for (const imageCount of counts) { + for (const referenceImageCount of imageReferenceCounts) { + for (const referenceVideoCount of videoReferenceCounts) { + const scenario = [ + `ratio=${ratio}`, + `size=${size}`, + `count=${imageCount}`, + `images=${referenceImageCount}`, + `videos=${referenceVideoCount}`, + ].join(","); + simulatedCases += 1; + try { + const { body } = createAtlasCloudRequestBody({ + modelId: preset.modelId, + prompt: "AtlasCloud 前端请求模拟测试", + referenceImages: FAKE_IMAGES.slice(0, referenceImageCount), + referenceVideos: FAKE_VIDEOS.slice(0, referenceVideoCount), + imageParams: { + aspectRatio: ratio, + imageSize: size, + imageCount, + }, + modelConfig: imageGenConfig, + }); + validateBody(body, runtime); + } catch (error) { + failures.push({ + modelId: preset.modelId, + scenario, + error: error instanceof Error ? error.message : String(error), + }); + } + } + } + } + } + } + } + + const summary = { + catalogModels: catalog.catalogCount, + availableModels: catalog.presets.length, + imageModels, + model3dModels, + videoReferenceModels, + simulatedCases, + passedCases: simulatedCases - failures.length, + failedCases: failures.length, + failures, + }; + console.log(JSON.stringify(summary, null, 2)); + if (failures.length > 0) process.exitCode = 1; +} + +void main(); diff --git a/tests/routes/model-classification.test.ts b/tests/routes/model-classification.test.ts index c0b50796b..9bbf17e91 100644 --- a/tests/routes/model-classification.test.ts +++ b/tests/routes/model-classification.test.ts @@ -3,6 +3,7 @@ import { createAtlasCloudImagePreset, inferAtlasCloudMediaKind, } from "@/lib/ai/atlascloud-models"; +import { createAtlasCloudRequestBody } from "@/lib/ai/atlascloud-request"; import { is3DModelDescriptor, isImageGenerationModelDescriptor, @@ -120,4 +121,71 @@ test.describe("AI model media classification", () => { expect(isVideoModelDescriptor(videoDescriptor)).toBe(true); expect(isImageGenerationModelDescriptor(videoDescriptor)).toBe(false); }); + + test("maps Atlas reference-video image models to a valid frontend request", () => { + const preset = createAtlasCloudImagePreset( + { + model: "google/nano-banana-2/reference-to-image", + type: "Image", + displayName: "Nano Banana 2 Reference to Image", + categories: ["IMAGE-TO-IMAGE"], + }, + { + paths: { + "/api/v1/model/generateImage": { post: {} }, + "/api/v1/model/prediction/{request_id}": { get: {} }, + }, + components: { + schemas: { + Input: { + type: "object", + properties: { + model: { type: "string" }, + prompt: { type: "string" }, + resolution: { type: "string", enum: ["1k", "2k", "4k"] }, + video_clips: { + type: "array", + minItems: 1, + maxItems: 1, + items: { + type: "object", + properties: { + url: { type: "string" }, + start: { type: "number", default: 0 }, + ends: { type: "number", default: 0 }, + fps: { type: "number", default: 1 }, + }, + }, + }, + }, + required: ["model", "prompt", "video_clips"], + }, + }, + }, + } + ); + + expect(preset.capabilities).toMatchObject({ + mediaType: "image", + supportsVideoReference: true, + requiresVideoReference: true, + maxVideoReferences: 1, + }); + const { body } = createAtlasCloudRequestBody({ + modelId: preset.modelId, + prompt: "参考视频生成商品图", + referenceImages: [], + referenceVideos: ["https://example.com/reference.mp4"], + imageParams: { imageSize: "2K", aspectRatio: "1:1" }, + modelConfig: preset.imageGenConfig, + }); + expect(body.video_clips).toEqual([ + { + url: "https://example.com/reference.mp4", + start: 0, + ends: 0, + fps: 1, + }, + ]); + }); }); From 1a91b8ada93f124a4dc8179dd12da84abc3d6ee5 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 22 Jul 2026 10:34:45 +0800 Subject: [PATCH 05/83] fix: hide stale AtlasCloud catalog entries --- scripts/sync-atlascloud-models.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scripts/sync-atlascloud-models.ts b/scripts/sync-atlascloud-models.ts index 0455116e6..1de03d9ac 100644 --- a/scripts/sync-atlascloud-models.ts +++ b/scripts/sync-atlascloud-models.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { fetchAtlasCloudImagePresets } from "../lib/ai/atlascloud-models"; import { db } from "../lib/db"; import { aiModel, aiProvider } from "../lib/db/schema"; @@ -22,6 +22,9 @@ async function main() { } const summaries = []; + const catalogModelIds = new Set( + catalog.presets.map((preset) => preset.modelId) + ); for (const provider of providers) { let added = 0; let updated = 0; @@ -65,11 +68,34 @@ async function main() { added += 1; } } + const providerModels = await db + .select({ modelId: aiModel.modelId }) + .from(aiModel) + .where(eq(aiModel.providerId, provider.id)); + const staleModelIds = providerModels + .map((model) => model.modelId) + .filter((modelId) => !catalogModelIds.has(modelId)); + if (staleModelIds.length > 0) { + await db + .update(aiModel) + .set({ + isEnabled: false, + isHidden: true, + updatedAt: new Date(), + }) + .where( + and( + eq(aiModel.providerId, provider.id), + inArray(aiModel.modelId, staleModelIds) + ) + ); + } summaries.push({ providerId: provider.id, providerName: provider.name, added, updated, + disabledStale: staleModelIds.length, }); } From 6ccf04a786a33ef4b2bb5e80b2a4263997679664 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 22 Jul 2026 10:53:19 +0800 Subject: [PATCH 06/83] fix: preserve AtlasCloud model aliases --- app/api/admin/atlascloud-models/route.ts | 11 +++++++++-- lib/ai/atlascloud-models.ts | 11 +++++++++++ scripts/sync-atlascloud-models.ts | 11 +++++++++-- tests/routes/model-classification.test.ts | 18 ++++++++++++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/app/api/admin/atlascloud-models/route.ts b/app/api/admin/atlascloud-models/route.ts index 74636b29d..44ae614a1 100644 --- a/app/api/admin/atlascloud-models/route.ts +++ b/app/api/admin/atlascloud-models/route.ts @@ -1,6 +1,9 @@ import { and, eq } from "drizzle-orm"; import { NextResponse } from "next/server"; -import { fetchAtlasCloudImagePresets } from "@/lib/ai/atlascloud-models"; +import { + fetchAtlasCloudImagePresets, + resolveAtlasCloudDisplayName, +} from "@/lib/ai/atlascloud-models"; import { db } from "@/lib/db"; import { getAiProviderById } from "@/lib/db/queries"; import { type AiModel, aiModel } from "@/lib/db/schema"; @@ -75,7 +78,11 @@ export async function POST(request: Request) { ), }); const values = { - displayName: preset.displayName, + displayName: resolveAtlasCloudDisplayName( + existing?.displayName, + preset.displayName, + preset.modelId + ), avatarUrl: preset.avatarUrl || null, description: preset.description, supportsVision: preset.supportsVision, diff --git a/lib/ai/atlascloud-models.ts b/lib/ai/atlascloud-models.ts index 91998fb70..342865628 100644 --- a/lib/ai/atlascloud-models.ts +++ b/lib/ai/atlascloud-models.ts @@ -95,6 +95,17 @@ type JsonRecord = Record; export type AtlasCloudMediaKind = "image" | "video" | "3d"; +export function resolveAtlasCloudDisplayName( + existingDisplayName: unknown, + catalogDisplayName: unknown, + modelId: string +) { + const existing = String(existingDisplayName || "").trim(); + if (existing) return existing; + + return String(catalogDisplayName || "").trim() || modelId; +} + function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) diff --git a/scripts/sync-atlascloud-models.ts b/scripts/sync-atlascloud-models.ts index 1de03d9ac..dabcf5309 100644 --- a/scripts/sync-atlascloud-models.ts +++ b/scripts/sync-atlascloud-models.ts @@ -1,5 +1,8 @@ import { and, eq, inArray } from "drizzle-orm"; -import { fetchAtlasCloudImagePresets } from "../lib/ai/atlascloud-models"; +import { + fetchAtlasCloudImagePresets, + resolveAtlasCloudDisplayName, +} from "../lib/ai/atlascloud-models"; import { db } from "../lib/db"; import { aiModel, aiProvider } from "../lib/db/schema"; @@ -36,7 +39,11 @@ async function main() { ), }); const values = { - displayName: preset.displayName, + displayName: resolveAtlasCloudDisplayName( + existing?.displayName, + preset.displayName, + preset.modelId + ), avatarUrl: preset.avatarUrl || null, description: preset.description, supportsVision: preset.supportsVision, diff --git a/tests/routes/model-classification.test.ts b/tests/routes/model-classification.test.ts index 9bbf17e91..f8b8a697a 100644 --- a/tests/routes/model-classification.test.ts +++ b/tests/routes/model-classification.test.ts @@ -2,6 +2,7 @@ import { expect, test } from "@playwright/test"; import { createAtlasCloudImagePreset, inferAtlasCloudMediaKind, + resolveAtlasCloudDisplayName, } from "@/lib/ai/atlascloud-models"; import { createAtlasCloudRequestBody } from "@/lib/ai/atlascloud-request"; import { @@ -19,6 +20,23 @@ const atlasHunyuan3D = { }; test.describe("AI model media classification", () => { + test("preserves administrator-defined AtlasCloud display names", () => { + expect( + resolveAtlasCloudDisplayName( + "视觉 Flux Kontext Pro", + "Flux Kontext Pro", + "black-forest-labs/flux-kontext-pro" + ) + ).toBe("视觉 Flux Kontext Pro"); + expect( + resolveAtlasCloudDisplayName( + "", + "Flux Kontext Pro", + "black-forest-labs/flux-kontext-pro" + ) + ).toBe("Flux Kontext Pro"); + }); + test("keeps multi-reference BananaImage models classified as image models", () => { const descriptor = { providerType: "bananaimage", From 974a4b1b59c602e258f0fec29657b7b2875b8540 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 22 Jul 2026 11:42:39 +0800 Subject: [PATCH 07/83] chore: rename hidden model control --- app/(admin)/settings/admin-settings-client.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 328c66183..ed16889a0 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -16464,10 +16464,7 @@ export function AdminSettingsClient() { }) } /> - +

开启后，该模型不会出现在前端，也不会参与第三方生图 From 3096eb67526085c8bfdbd017fe795cd57f7f44a5 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 23 Jul 2026 10:52:29 +0800 Subject: [PATCH 08/83] feat: add AtlasCloud video model support --- .../settings/admin-settings-client.tsx | 12 +- app/api/admin/atlascloud-models/route.ts | 48 ++- lib/ai/atlascloud-models.ts | 281 ++++++++++++++++-- lib/ai/atlascloud-request.ts | 123 ++++++++ lib/ai/providers/atlascloud.ts | 77 ++++- lib/ai/tools/generate-video.ts | 36 ++- scripts/sync-atlascloud-models.ts | 71 ++--- tests/routes/model-classification.test.ts | 101 ++++++- 8 files changed, 667 insertions(+), 82 deletions(-) diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index ed16889a0..ce81cfce2 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -1587,7 +1587,7 @@ const PROVIDER_TYPES = [ { value: "openai-image", label: "OpenAI 图片兼容" }, { value: "bananaimage", label: "BananaImage 生图" }, { value: "visionary", label: "Visionary 生图" }, - { value: "atlascloud", label: "Atlas Cloud 图片 / 3D 模型" }, + { value: "atlascloud", label: "Atlas Cloud 图片 / 视频 / 3D 模型" }, { value: "stepfun-image", label: "阶跃星辰 Step Image Edit" }, { value: "bltcy-video", label: "柏拉图统一视频" }, { value: "grsai-gpt-image", label: "Grsai GPT Image" }, @@ -6473,7 +6473,7 @@ export function AdminSettingsClient() { } catch (modelError) { console.error("[Auto Add Atlas Cloud Models] Error:", modelError); toast.warning( - 'Atlas Cloud 模型同步失败，请手动点击"同步 Atlas Cloud 图片 / 3D 模型"按钮' + 'Atlas Cloud 模型同步失败，请手动点击"同步 Atlas Cloud 模型"按钮' ); } } @@ -7534,7 +7534,7 @@ export function AdminSettingsClient() { if ( !options?.skipConfirm && !confirm( - "将从 Atlas Cloud 官方目录同步全部开放图片与 3D 模型，并读取每个模型的参数 Schema。同步数量较多，可能需要几十秒。\n\n继续吗？" + "将从 Atlas Cloud 官方目录同步图片、视频与 3D 模型，并读取每个模型的参数 Schema。已删除的模型不会被恢复。同步数量较多，可能需要几十秒。\n\n继续吗？" ) ) { return; @@ -7553,7 +7553,7 @@ export function AdminSettingsClient() { const totalSynced = Number(data.addedCount || 0) + Number(data.updatedCount || 0); toast.success( - `已同步 ${totalSynced} 个 Atlas Cloud 图片 / 3D 模型（官方目录共 ${Number(data.counts?.total || totalSynced)} 个）` + `已同步 ${totalSynced} 个 Atlas Cloud 图片 / 视频 / 3D 模型（官方目录共 ${Number(data.counts?.total || totalSynced)} 个）` ); mutateModels(); if (data.errors?.length) { @@ -7561,7 +7561,7 @@ export function AdminSettingsClient() { } } catch (error: any) { console.error("[Quick Add Atlas Cloud Models] Error:", error); - toast.error(error.message || "同步 Atlas Cloud 图片 / 3D 模型失败"); + toast.error(error.message || "同步 Atlas Cloud 模型失败"); throw error; } }; @@ -15943,7 +15943,7 @@ export function AdminSettingsClient() { variant="secondary" > - 同步 Atlas Cloud 图片 / 3D 模型 + 同步 Atlas Cloud 模型 )} {provider.type === "bltcy-video" && ( diff --git a/app/api/admin/atlascloud-models/route.ts b/app/api/admin/atlascloud-models/route.ts index 44ae614a1..965957638 100644 --- a/app/api/admin/atlascloud-models/route.ts +++ b/app/api/admin/atlascloud-models/route.ts @@ -1,8 +1,9 @@ import { and, eq } from "drizzle-orm"; import { NextResponse } from "next/server"; import { - fetchAtlasCloudImagePresets, + fetchAtlasCloudModelPresets, resolveAtlasCloudDisplayName, + shouldInsertMissingAtlasCloudPreset, } from "@/lib/ai/atlascloud-models"; import { db } from "@/lib/db"; import { getAiProviderById } from "@/lib/db/queries"; @@ -15,7 +16,7 @@ export const maxDuration = 300; export async function GET() { try { await requireAdmin(); - const result = await fetchAtlasCloudImagePresets(); + const result = await fetchAtlasCloudModelPresets(); return NextResponse.json({ presets: result.presets, errors: result.errors.length > 0 ? result.errors : undefined, @@ -32,7 +33,7 @@ export async function GET() { error: error instanceof Error ? error.message - : "Failed to fetch Atlas Cloud image models", + : "Failed to fetch Atlas Cloud models", }, { status: 500 } ); @@ -61,22 +62,39 @@ export async function POST(request: Request) { ); } - const catalog = await fetchAtlasCloudImagePresets(); + const catalog = await fetchAtlasCloudModelPresets(); const presets = requestedModelIds ? catalog.presets.filter((preset) => requestedModelIds.has(preset.modelId)) : catalog.presets; const addedModels: AiModel[] = []; const updatedModels: AiModel[] = []; const errors = [...catalog.errors]; + const providerModels = await db.query.aiModel.findMany({ + where: eq(aiModel.providerId, providerId), + }); + const existingByModelId = new Map( + providerModels.map((model) => [model.modelId, model]) + ); + const existingMediaKinds = new Set( + providerModels.map((model) => { + const capabilities = model.capabilities as Record | null; + return String(capabilities?.mediaType || "image"); + }) + ); + let preservedDeletedCount = 0; for (const preset of presets) { try { - const existing = await db.query.aiModel.findFirst({ - where: and( - eq(aiModel.providerId, providerId), - eq(aiModel.modelId, preset.modelId) - ), - }); + const existing = existingByModelId.get(preset.modelId); + const presetCapabilities = preset.capabilities as Record; + const mediaKind = String(presetCapabilities.mediaType || "image"); + if ( + !existing && + !shouldInsertMissingAtlasCloudPreset(existingMediaKinds, mediaKind) + ) { + preservedDeletedCount += 1; + continue; + } const values = { displayName: resolveAtlasCloudDisplayName( existing?.displayName, @@ -87,9 +105,7 @@ export async function POST(request: Request) { description: preset.description, supportsVision: preset.supportsVision, supportsImageGen: preset.supportsImageGen, - isEnabled: true, - isHidden: false, - sortOrder: preset.sortOrder, + sortOrder: existing?.sortOrder ?? preset.sortOrder, capabilities: preset.capabilities, defaultParams: preset.defaultParams, imageGenConfig: preset.imageGenConfig, @@ -115,6 +131,8 @@ export async function POST(request: Request) { providerId, modelId: preset.modelId, ...values, + isEnabled: true, + isHidden: false, isDefault: false, isSubModel: false, parentModelId: null, @@ -145,6 +163,8 @@ export async function POST(request: Request) { requested: presets.length, total: catalog.catalogCount, available: catalog.presets.length, + skippedUnsupported: catalog.skipped.length, + preservedDeleted: preservedDeletedCount, }, }); } catch (error) { @@ -154,7 +174,7 @@ export async function POST(request: Request) { error: error instanceof Error ? error.message - : "Failed to sync Atlas Cloud image models", + : "Failed to sync Atlas Cloud models", }, { status: 500 } ); diff --git a/lib/ai/atlascloud-models.ts b/lib/ai/atlascloud-models.ts index 342865628..5a2cc4970 100644 --- a/lib/ai/atlascloud-models.ts +++ b/lib/ai/atlascloud-models.ts @@ -29,8 +29,25 @@ const IMAGE_INPUT_KEYS = [ "reference_image", ]; const RESOLUTION_KEYS = ["resolution", "image_size", "output_resolution"]; -const ASPECT_RATIO_KEYS = ["aspect_ratio", "aspectRatio"]; +const VIDEO_RESOLUTION_KEYS = [ + "resolution", + "output_resolution", + "target_resolution", +]; +const ASPECT_RATIO_KEYS = ["aspect_ratio", "aspectRatio", "ratio"]; const OUTPUT_COUNT_KEYS = ["num_images", "number_of_images", "n", "output_count"]; +const VIDEO_DURATION_KEYS = ["duration", "video_duration", "seconds"]; +const VIDEO_REQUEST_KEYS = new Set([ + "model", + "prompt", + ...VIDEO_RESOLUTION_KEYS, + ...ASPECT_RATIO_KEYS, + ...VIDEO_DURATION_KEYS, + "seed", + "watermark", + "camera_fixed", + "return_last_frame", +]); export interface AtlasCloudCatalogModel { uuid?: string; @@ -67,14 +84,17 @@ export interface AtlasCloudRuntimeConfig { itemDefaults: Record; }; resolutionKey?: string; + durationKey?: string; aspectRatioKey?: string; outputCountKey?: string; aspectRatioValueMap: Record; + resolutionValueMap?: Record; maxReferenceImages: number; maxVideoReferences: number; maxImages: number; defaults: Record; enums: Record>; + numericRanges?: Record; sizeValueMap: Record>; } @@ -106,6 +126,13 @@ export function resolveAtlasCloudDisplayName( return String(catalogDisplayName || "").trim() || modelId; } +export function shouldInsertMissingAtlasCloudPreset( + existingMediaKinds: ReadonlySet, + mediaKind: string +) { + return !existingMediaKinds.has(mediaKind); +} + function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) @@ -342,6 +369,7 @@ function closestCommonRatio(ratio: string) { function normalizeAspectRatioEnum(value: unknown) { const normalized = String(value || "").trim().toLowerCase(); if (normalized === "auto") return "auto"; + if (normalized === "adaptive") return "adaptive"; if (/^\d+:\d+$/.test(normalized)) return normalized; if (normalized.startsWith("square")) return "1:1"; const namedRatio = normalized.match( @@ -352,6 +380,115 @@ function normalizeAspectRatioEnum(value: unknown) { return orientation === "portrait" ? `${second}:${first}` : `${first}:${second}`; } +function normalizeVideoResolution(value: unknown) { + const normalized = String(value || "").trim().toLowerCase(); + const match = normalized.match(/(?:^|[^0-9])(480|720|1080)p(?:$|[^0-9])/); + return match?.[1] ? `${match[1]}p` : ""; +} + +function buildVideoResolutionValueMap( + properties: JsonRecord, + resolutionKey?: string +) { + if (!resolutionKey) return {}; + const map: Record = {}; + for (const value of getEnumValues(properties[resolutionKey])) { + const normalized = normalizeVideoResolution(value); + if (!normalized) continue; + const raw = String(value); + const current = map[normalized]; + if (!current || raw.toLowerCase() === normalized) { + map[normalized] = raw; + } + } + return map; +} + +function buildNumericRanges(properties: JsonRecord) { + const ranges: Record = {}; + for (const [key, property] of Object.entries(properties)) { + const record = asRecord(property); + const min = Number(record.minimum ?? record.min); + const max = Number(record.maximum ?? record.max); + const range = { + ...(Number.isFinite(min) ? { min } : {}), + ...(Number.isFinite(max) ? { max } : {}), + }; + if (Object.keys(range).length > 0) ranges[key] = range; + } + return ranges; +} + +function deriveVideoDurationOptions( + properties: JsonRecord, + durationKey?: string +) { + if (!durationKey) return []; + return Array.from( + new Set( + getEnumValues(properties[durationKey]) + .map(Number) + .filter((value) => Number.isFinite(value) && value > 0) + .map((value) => Math.round(value)) + ) + ).sort((a, b) => a - b); +} + +function deriveAtlasCloudVideoModes(params: { + catalogModel: AtlasCloudCatalogModel; + properties: JsonRecord; + requiredFields: string[]; + imageInputs: AtlasCloudRuntimeConfig["imageInputs"]; +}) { + const { catalogModel, properties, requiredFields, imageInputs } = params; + const intentText = uniqueStrings([ + catalogModel.model, + catalogModel.displayName, + ...(catalogModel.categories || []), + ...(catalogModel.tags || []), + ]) + .map(normalizeIntentToken) + .join(" "); + const imageKeys = new Set(imageInputs.map((input) => input.key)); + const hasUnsupportedRequiredInput = requiredFields.some((key) => { + if (imageKeys.has(key) || VIDEO_REQUEST_KEYS.has(key)) return false; + const defaultValue = asRecord(properties[key]).default; + return !["string", "number", "boolean"].includes(typeof defaultValue); + }); + const requiredImages = imageInputs.filter((input) => input.required); + const singleImageInputs = imageInputs.filter((input) => input.mode === "single"); + const arrayImageInput = imageInputs.find( + (input) => input.mode === "array" && input.maxItems > 1 + ); + const modes: string[] = []; + + if ( + !hasUnsupportedRequiredInput && + requiredImages.length === 0 && + ("prompt" in properties || intentText.includes("text-to-video")) + ) { + modes.push("text-to-video"); + } + if (!hasUnsupportedRequiredInput && imageInputs.length > 0) { + if (requiredImages.length <= 1) modes.push("first-frame"); + if ( + singleImageInputs.length >= 2 || + /(?:first|start).*(?:last|end)|(?:last|end).*(?:first|start)/.test(intentText) + ) { + modes.push("first-last-frame"); + } + if ( + arrayImageInput || + imageInputs.length > 2 || + /reference-to-video|reference-images/.test(intentText) + ) { + modes.push("reference-images"); + } + } + + return uniqueStrings(modes); +} + function buildAspectRatioValueMap(properties: JsonRecord) { for (const key of [...ASPECT_RATIO_KEYS, "image_size"]) { if (!(key in properties)) continue; @@ -449,20 +586,37 @@ export function createAtlasCloudImagePreset( const imageInputs = getImageInputs(properties, requiredFields); const videoClipInput = getVideoClipInput(properties, requiredFields); const imageInputKey = imageInputs[0]?.key; - const resolutionKey = RESOLUTION_KEYS.find((key) => { + const mediaKind = inferAtlasCloudMediaKind(catalogModel); + const resolutionCandidates = + mediaKind === "video" ? VIDEO_RESOLUTION_KEYS : RESOLUTION_KEYS; + const resolutionKey = resolutionCandidates.find((key) => { if (!(key in properties)) return false; const values = getEnumValues(properties[key]); - return values.length === 0 || values.some((value) => Boolean(normalizeKSize(value))); + return ( + values.length === 0 || + values.some((value) => + mediaKind === "video" + ? Boolean(normalizeVideoResolution(value)) + : Boolean(normalizeKSize(value)) + ) + ); }); + const durationKey = findFirstKey(properties, VIDEO_DURATION_KEYS); const aspectRatioConfig = buildAspectRatioValueMap(properties); const aspectRatioKey = aspectRatioConfig.key; const outputCountKey = findFirstKey(properties, OUTPUT_COUNT_KEYS); const sizeValueMap = buildSizeValueMap(getEnumValues(properties.size)); - const supportedImageSizes = deriveSupportedImageSizes( + const supportedImageSizes = + mediaKind === "video" + ? [] + : deriveSupportedImageSizes(properties, resolutionKey, sizeValueMap); + const resolutionValueMap = buildVideoResolutionValueMap( properties, - resolutionKey, - sizeValueMap + mediaKind === "video" ? resolutionKey : undefined ); + const supportedVideoResolutions = Object.keys(resolutionValueMap); + const durationOptions = deriveVideoDurationOptions(properties, durationKey); + const numericRanges = buildNumericRanges(properties); const supportedAspectRatios = deriveAspectRatios( aspectRatioConfig.map, sizeValueMap @@ -477,10 +631,16 @@ export function createAtlasCloudImagePreset( const categories = uniqueStrings(catalogModel.categories || []).map((value) => value.toLowerCase() ); - const mediaKind = inferAtlasCloudMediaKind(catalogModel); const supportedModes = mediaKind === "3d" ? derive3DSupportedModes(catalogModel) + : mediaKind === "video" + ? deriveAtlasCloudVideoModes({ + catalogModel, + properties, + requiredFields, + imageInputs, + }) : [ ...(categories.some((value) => value.includes("text-to-image")) ? ["text-to-image"] @@ -505,14 +665,17 @@ export function createAtlasCloudImagePreset( imageInputMode: imageProperty.type === "array" ? "array" : "single", videoClipInput, resolutionKey, + durationKey, aspectRatioKey, outputCountKey, aspectRatioValueMap: aspectRatioConfig.map, + resolutionValueMap, maxReferenceImages, maxVideoReferences, maxImages, defaults: buildScalarDefaults(properties), enums: buildEnums(properties), + numericRanges, sizeValueMap, }; @@ -524,10 +687,23 @@ export function createAtlasCloudImagePreset( supportedImageSizes[0] || "1K"; const defaultAspectRatio = - supportedAspectRatios.find((value) => value === "auto") || + supportedAspectRatios.find((value) => + mediaKind === "video" ? value === "adaptive" : value === "auto" + ) || supportedAspectRatios.find((value) => value === "1:1") || supportedAspectRatios[0] || - "1:1"; + (mediaKind === "video" ? "16:9" : "1:1"); + const defaultVideoResolution = + supportedVideoResolutions.find((value) => value === "720p") || + supportedVideoResolutions[0] || + "720p"; + const durationRange = numericRanges[durationKey || ""] || {}; + const defaultDuration = + Number(runtimeConfig.defaults[durationKey || ""]) || + durationOptions.find((value) => value === 5) || + durationOptions[0] || + durationRange.min || + 5; return { modelId: catalogModel.model, @@ -550,6 +726,27 @@ export function createAtlasCloudImagePreset( : "text-to-image", mediaType: mediaKind, supportedModes, + ...(mediaKind === "video" + ? { + supportedResolutions: supportedVideoResolutions, + supportedRatios: supportedAspectRatios, + durationOptions, + durationRange: { + min: + durationOptions[0] || + durationRange.min || + defaultDuration, + max: + durationOptions[durationOptions.length - 1] || + durationRange.max || + defaultDuration, + }, + supportWatermark: "watermark" in properties, + supportCameraFixed: "camera_fixed" in properties, + supportSeed: "seed" in properties, + supportReturnLastFrame: "return_last_frame" in properties, + } + : {}), maxReferenceImages, maxVideoReferences, supportsMultiImageReference: maxReferenceImages > 1, @@ -559,13 +756,24 @@ export function createAtlasCloudImagePreset( schemaUrl: catalogModel.schema || null, }, defaultParams: { - imageSize: defaultImageSize, - aspectRatio: defaultAspectRatio, - imageCount: 1, + ...(mediaKind === "video" + ? { + resolution: defaultVideoResolution, + ratio: defaultAspectRatio, + duration: defaultDuration, + } + : { + imageSize: defaultImageSize, + aspectRatio: defaultAspectRatio, + imageCount: 1, + }), runtimeModelId: catalogModel.model, }, imageGenConfig: { - apiFormat: "atlascloud-unified-image", + apiFormat: + mediaKind === "video" + ? "atlascloud-unified-video" + : "atlascloud-unified-image", maxImages, maxReferenceImages, maxVideoReferences, @@ -575,6 +783,7 @@ export function createAtlasCloudImagePreset( supports4K: supportedImageSizes.includes("4K"), supportedImageSizes, supportedAspectRatios, + videoDurationOptions: durationOptions, supportsSeed: "seed" in properties, supportsNegativePrompt: "negative_prompt" in properties, supportsWatermark: "watermark" in properties, @@ -602,7 +811,7 @@ async function mapWithConcurrency( return results; } -export async function fetchAtlasCloudImagePresets() { +async function fetchAtlasCloudPresets(mediaTypes: Array<"image" | "video">) { const catalogResponse = await fetch(ATLASCLOUD_CATALOG_URL, { cache: "no-store", headers: { Accept: "application/json", "X-Source-From": "website" }, @@ -618,7 +827,9 @@ export async function fetchAtlasCloudImagePresets() { const models = catalogModels .filter( (model: AtlasCloudCatalogModel) => - String(model?.type || "").toLowerCase() === "image" && + mediaTypes.includes( + String(model?.type || "").toLowerCase() as "image" | "video" + ) && model?.display_console !== false && Boolean(model?.model) && Boolean(model?.schema) @@ -638,10 +849,28 @@ export async function fetchAtlasCloudImagePresets() { if (!schemaResponse.ok) { throw new Error(`HTTP ${schemaResponse.status}`); } - return { - preset: createAtlasCloudImagePreset(model, await schemaResponse.json(), index), - error: null, - }; + const preset = createAtlasCloudImagePreset( + model, + await schemaResponse.json(), + index + ); + const supportedModes = Array.isArray(preset.capabilities.supportedModes) + ? preset.capabilities.supportedModes + : []; + if ( + preset.capabilities.mediaType === "video" && + supportedModes.length === 0 + ) { + return { + preset: null, + error: null, + skipped: { + modelId: model.model, + reason: "当前视频工具不支持该模型要求的视频或音频输入", + }, + }; + } + return { preset, error: null, skipped: null }; } catch (error) { return { preset: null, @@ -649,6 +878,7 @@ export async function fetchAtlasCloudImagePresets() { modelId: model.model, error: error instanceof Error ? error.message : String(error), }, + skipped: null, }; } }); @@ -656,6 +886,19 @@ export async function fetchAtlasCloudImagePresets() { return { presets: settled.flatMap((result) => (result.preset ? [result.preset] : [])), errors: settled.flatMap((result) => (result.error ? [result.error] : [])), + skipped: settled.flatMap((result) => (result.skipped ? [result.skipped] : [])), catalogCount: models.length, }; } + +export function fetchAtlasCloudImagePresets() { + return fetchAtlasCloudPresets(["image"]); +} + +export function fetchAtlasCloudVideoPresets() { + return fetchAtlasCloudPresets(["video"]); +} + +export function fetchAtlasCloudModelPresets() { + return fetchAtlasCloudPresets(["image", "video"]); +} diff --git a/lib/ai/atlascloud-request.ts b/lib/ai/atlascloud-request.ts index 4607c4853..79455fe64 100644 --- a/lib/ai/atlascloud-request.ts +++ b/lib/ai/atlascloud-request.ts @@ -1,4 +1,5 @@ import type { AtlasCloudRuntimeConfig } from "@/lib/ai/atlascloud-models"; +import type { VideoGenerationOptions } from "@/types/ai"; export interface AtlasCloudImageParams { aspectRatio?: string; @@ -19,6 +20,10 @@ export interface AtlasCloudModelConfig { atlasCloud?: AtlasCloudRuntimeConfig; } +export interface AtlasCloudVideoModelConfig extends AtlasCloudModelConfig { + videoDurationOptions?: number[]; +} + function normalizeAspectRatio( requested: string | undefined, supported: string[] | undefined @@ -207,3 +212,121 @@ export function createAtlasCloudRequestBody(params: { } return { body, imageSize }; } + +function setAtlasCloudImages(params: { + body: Record; + runtime: AtlasCloudRuntimeConfig; + images: string[]; +}) { + const imageInputs = params.runtime.imageInputs?.length + ? params.runtime.imageInputs + : params.runtime.imageInputKey + ? [ + { + key: params.runtime.imageInputKey, + mode: params.runtime.imageInputMode || ("single" as const), + maxItems: Math.max(1, params.runtime.maxReferenceImages || 1), + required: params.runtime.requiredFields.includes( + params.runtime.imageInputKey + ), + }, + ] + : []; + let cursor = 0; + for (const input of imageInputs) { + const values = params.images.slice(cursor, cursor + input.maxItems); + if (values.length > 0) { + params.body[input.key] = input.mode === "single" ? values[0] : values; + cursor += values.length; + } else if (input.required) { + throw new Error(`该 Atlas Cloud 视频模型至少需要 ${cursor + 1} 张参考图`); + } + } + if (imageInputs.length === 0 && params.images.length > 0) { + throw new Error("当前 Atlas Cloud 视频模型不支持参考图"); + } +} + +export function createAtlasCloudVideoRequestBody(params: { + modelId: string; + options: VideoGenerationOptions; + referenceImages: string[]; + modelConfig: AtlasCloudVideoModelConfig; +}) { + const runtime = params.modelConfig.atlasCloud; + if (!runtime) { + throw new Error("Atlas Cloud 视频模型运行参数缺失，请重新同步模型"); + } + const body: Record = { ...runtime.defaults }; + if ( + runtime.propertyNames.includes("model") || + runtime.requiredFields.includes("model") + ) { + body.model = params.modelId; + } + if ( + runtime.propertyNames.includes("prompt") || + runtime.requiredFields.includes("prompt") + ) { + body.prompt = params.options.prompt; + } + + if (runtime.resolutionKey && params.options.resolution) { + body[runtime.resolutionKey] = + runtime.resolutionValueMap?.[params.options.resolution] || + matchEnumValue( + runtime.enums[runtime.resolutionKey], + params.options.resolution + ) || + params.options.resolution; + } + if (runtime.aspectRatioKey && params.options.ratio) { + const requestedRatio = params.options.ratio; + body[runtime.aspectRatioKey] = + runtime.aspectRatioValueMap?.[requestedRatio] || + runtime.aspectRatioValueMap?.[ + requestedRatio === "adaptive" ? "auto" : requestedRatio + ] || + matchEnumValue(runtime.enums[runtime.aspectRatioKey], requestedRatio) || + requestedRatio; + } + if (runtime.durationKey && params.options.duration !== undefined) { + body[runtime.durationKey] = + matchEnumValue(runtime.enums[runtime.durationKey], params.options.duration) ?? + params.options.duration; + } + + setAtlasCloudImages({ + body, + runtime, + images: params.referenceImages, + }); + + const optionalParams: Array<[string, unknown]> = [ + ["seed", params.options.seed], + ["watermark", params.options.watermark], + ["camera_fixed", params.options.cameraFixed], + ["return_last_frame", params.options.returnLastFrame], + ]; + for (const [key, value] of optionalParams) { + if (value !== undefined && runtime.propertyNames.includes(key)) { + body[key] = matchEnumValue(runtime.enums[key], value) ?? value; + } + } + if (runtime.propertyNames.includes("enable_sync_mode")) { + body.enable_sync_mode = false; + } + if (runtime.propertyNames.includes("enable_base64_output")) { + body.enable_base64_output = false; + } + + const unsupportedRequiredFields = runtime.requiredFields.filter( + (key) => body[key] === undefined + ); + if (unsupportedRequiredFields.length > 0) { + throw new Error( + `当前视频界面暂不支持该 Atlas Cloud 模型的必填参数：${unsupportedRequiredFields.join(", ")}` + ); + } + return { body }; +} diff --git a/lib/ai/providers/atlascloud.ts b/lib/ai/providers/atlascloud.ts index 81a0c79f2..6a7bbb623 100644 --- a/lib/ai/providers/atlascloud.ts +++ b/lib/ai/providers/atlascloud.ts @@ -4,11 +4,14 @@ import sharp from "sharp"; import { ATLASCLOUD_BASE_URL } from "@/lib/ai/atlascloud-models"; import { createAtlasCloudRequestBody, + createAtlasCloudVideoRequestBody, type AtlasCloudImageParams, type AtlasCloudModelConfig, + type AtlasCloudVideoModelConfig, } from "@/lib/ai/atlascloud-request"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; +import type { VideoGenerationOptions } from "@/types/ai"; interface AtlasCloudConfig { apiKey: string; @@ -82,6 +85,10 @@ function extractOutputUrls(payload: any) { prediction?.outputs, prediction?.output, prediction?.images, + prediction?.video, + prediction?.video_url, + prediction?.videoUrl, + prediction?.url, prediction?.urls?.outputs, ]; const urls: string[] = []; @@ -91,8 +98,16 @@ function extractOutputUrls(payload: any) { const url = typeof value === "string" ? value - : String(value?.url || value?.image_url || value?.src || ""); - if (/^(?:https?:\/\/|data:image\/)/i.test(url) && !urls.includes(url)) { + : String( + value?.url || + value?.video_url || + value?.videoUrl || + value?.image_url || + value?.file_url || + value?.src || + "" + ); + if (/^(?:https?:\/\/|data:(?:image|video)\/)/i.test(url) && !urls.includes(url)) { urls.push(url); } } @@ -358,6 +373,64 @@ async function archiveOutputImage(imageUrl: string): Promise { ); } +export async function executeAtlasCloudVideoGeneration(params: { + apiKey: string; + baseURL?: string; + modelId: string; + modelConfig: AtlasCloudVideoModelConfig; + options: VideoGenerationOptions; +}) { + const runtime = params.modelConfig.atlasCloud; + if (!runtime) { + throw new Error("Atlas Cloud 视频模型能力配置缺失，请重新同步模型"); + } + const inputImages = [ + ...(params.options.firstFrameImage ? [params.options.firstFrameImage] : []), + ...(params.options.lastFrameImage ? [params.options.lastFrameImage] : []), + ...(params.options.referenceImages || []), + ]; + const uniqueImages = Array.from(new Set(inputImages)).slice( + 0, + Math.max(0, Math.min(14, runtime.maxReferenceImages || 0)) + ); + const referenceImages = await Promise.all( + uniqueImages.map((url) => uploadReferenceMedia(params.baseURL, params.apiKey, url)) + ); + const { body } = createAtlasCloudVideoRequestBody({ + modelId: params.modelId, + options: params.options, + referenceImages, + modelConfig: params.modelConfig, + }); + const submitted = await submitTask({ + apiKey: params.apiKey, + baseURL: params.baseURL, + endpoint: runtime.submitEndpoint, + body, + }); + const outputUrls = submitted.outputs.length + ? submitted.outputs + : await waitForTask({ + apiKey: params.apiKey, + baseURL: params.baseURL, + endpoint: runtime.pollEndpoint, + taskId: submitted.taskId, + }); + const videoUrl = outputUrls[0]; + if (!videoUrl) { + throw new Error("Atlas Cloud 视频任务完成但未返回视频地址"); + } + return { + url: videoUrl, + metadata: { + provider: "atlascloud", + taskId: submitted.taskId || null, + temporaryVideoUrl: videoUrl, + request: body, + }, + }; +} + function formatMarkdown(images: UploadedImage[], imageSize: string) { return images .map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`) diff --git a/lib/ai/tools/generate-video.ts b/lib/ai/tools/generate-video.ts index 150971222..0bae08c25 100644 --- a/lib/ai/tools/generate-video.ts +++ b/lib/ai/tools/generate-video.ts @@ -6,6 +6,7 @@ import { tool } from "ai"; import { z } from "zod"; import { doGenerateStream } from "@/lib/ai/providers/volcano-video"; +import { executeAtlasCloudVideoGeneration } from "@/lib/ai/providers/atlascloud"; import { getEnabledAiModelsWithProvider, getSubModelsByParentModelId @@ -109,8 +110,22 @@ function isVolcanoVideoModel(item: any) { return item?.provider?.type === "volcano-video" || item?.provider?.type === "doubao-video"; } +function isAtlasCloudVideoModel(item: any) { + const capabilities = item?.model?.capabilities; + return ( + item?.provider?.type === "atlascloud" && + capabilities && + typeof capabilities === "object" && + String((capabilities as Record).mediaType || "") === "video" + ); +} + function isDirectExecutableVideoModel(item: any) { - return isVolcanoVideoModel(item) || isAgnesVideoModel(item); + return ( + isVolcanoVideoModel(item) || + isAgnesVideoModel(item) || + isAtlasCloudVideoModel(item) + ); } /** @@ -275,6 +290,7 @@ function validateParamsAgainstCapabilities( // 检查分辨率 if ( params.resolution && + capabilities.supportedResolutions.length > 0 && !capabilities.supportedResolutions.includes(params.resolution) ) { throw new Error( @@ -283,7 +299,11 @@ function validateParamsAgainstCapabilities( } // 检查宽高比 - if (params.ratio && !capabilities.supportedRatios.includes(params.ratio)) { + if ( + params.ratio && + capabilities.supportedRatios.length > 0 && + !capabilities.supportedRatios.includes(params.ratio) + ) { throw new Error( `模型不支持 ${params.ratio} 宽高比，支持的宽高比：${capabilities.supportedRatios.join(", ")}` ); @@ -621,6 +641,18 @@ export async function executeDirectVideoGeneration( ...(agnesResult.metadata || {}), temporaryVideoUrl: agnesResult.url, }; + } else if (isAtlasCloudVideoModel(model)) { + const atlasResult = await executeAtlasCloudVideoGeneration({ + apiKey: model.provider.apiKey, + baseURL: model.provider.baseUrl, + modelId: model.model.modelId, + modelConfig: model.model.imageGenConfig || {}, + options, + }); + videoMetadata = { + ...(atlasResult.metadata || {}), + temporaryVideoUrl: atlasResult.url, + }; } else { for await (const chunk of doGenerateStream(model.provider.apiKey, model.provider.baseUrl, options)) { await params.onProgress?.(chunk); diff --git a/scripts/sync-atlascloud-models.ts b/scripts/sync-atlascloud-models.ts index dabcf5309..55a57cbd7 100644 --- a/scripts/sync-atlascloud-models.ts +++ b/scripts/sync-atlascloud-models.ts @@ -1,7 +1,8 @@ -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { - fetchAtlasCloudImagePresets, + fetchAtlasCloudModelPresets, resolveAtlasCloudDisplayName, + shouldInsertMissingAtlasCloudPreset, } from "../lib/ai/atlascloud-models"; import { db } from "../lib/db"; import { aiModel, aiProvider } from "../lib/db/schema"; @@ -15,7 +16,7 @@ async function main() { throw new Error("没有找到已启用的 AtlasCloud 服务商"); } - const catalog = await fetchAtlasCloudImagePresets(); + const catalog = await fetchAtlasCloudModelPresets(); if (catalog.errors.length > 0) { throw new Error( `AtlasCloud 目录有 ${catalog.errors.length} 个 schema 加载失败：${catalog.errors @@ -25,19 +26,34 @@ async function main() { } const summaries = []; - const catalogModelIds = new Set( - catalog.presets.map((preset) => preset.modelId) - ); for (const provider of providers) { let added = 0; let updated = 0; + let preservedDeleted = 0; + const providerModels = await db + .select() + .from(aiModel) + .where(eq(aiModel.providerId, provider.id)); + const existingByModelId = new Map( + providerModels.map((model) => [model.modelId, model]) + ); + const existingMediaKinds = new Set( + providerModels.map((model) => { + const capabilities = model.capabilities as Record | null; + return String(capabilities?.mediaType || "image"); + }) + ); for (const preset of catalog.presets) { - const existing = await db.query.aiModel.findFirst({ - where: and( - eq(aiModel.providerId, provider.id), - eq(aiModel.modelId, preset.modelId) - ), - }); + const existing = existingByModelId.get(preset.modelId); + const presetCapabilities = preset.capabilities as Record; + const mediaKind = String(presetCapabilities.mediaType || "image"); + if ( + !existing && + !shouldInsertMissingAtlasCloudPreset(existingMediaKinds, mediaKind) + ) { + preservedDeleted += 1; + continue; + } const values = { displayName: resolveAtlasCloudDisplayName( existing?.displayName, @@ -48,9 +64,7 @@ async function main() { description: preset.description, supportsVision: preset.supportsVision, supportsImageGen: preset.supportsImageGen, - isEnabled: true, - isHidden: false, - sortOrder: preset.sortOrder, + sortOrder: existing?.sortOrder ?? preset.sortOrder, capabilities: preset.capabilities, defaultParams: preset.defaultParams, imageGenConfig: preset.imageGenConfig, @@ -68,6 +82,8 @@ async function main() { providerId: provider.id, modelId: preset.modelId, ...values, + isEnabled: true, + isHidden: false, isDefault: false, isSubModel: false, parentModelId: null, @@ -75,34 +91,12 @@ async function main() { added += 1; } } - const providerModels = await db - .select({ modelId: aiModel.modelId }) - .from(aiModel) - .where(eq(aiModel.providerId, provider.id)); - const staleModelIds = providerModels - .map((model) => model.modelId) - .filter((modelId) => !catalogModelIds.has(modelId)); - if (staleModelIds.length > 0) { - await db - .update(aiModel) - .set({ - isEnabled: false, - isHidden: true, - updatedAt: new Date(), - }) - .where( - and( - eq(aiModel.providerId, provider.id), - inArray(aiModel.modelId, staleModelIds) - ) - ); - } summaries.push({ providerId: provider.id, providerName: provider.name, added, updated, - disabledStale: staleModelIds.length, + preservedDeleted, }); } @@ -111,6 +105,7 @@ async function main() { { catalogModels: catalog.catalogCount, availableModels: catalog.presets.length, + skippedUnsupportedVideoModels: catalog.skipped.length, providers: summaries, }, null, diff --git a/tests/routes/model-classification.test.ts b/tests/routes/model-classification.test.ts index f8b8a697a..d8039ed87 100644 --- a/tests/routes/model-classification.test.ts +++ b/tests/routes/model-classification.test.ts @@ -3,8 +3,12 @@ import { createAtlasCloudImagePreset, inferAtlasCloudMediaKind, resolveAtlasCloudDisplayName, + shouldInsertMissingAtlasCloudPreset, } from "@/lib/ai/atlascloud-models"; -import { createAtlasCloudRequestBody } from "@/lib/ai/atlascloud-request"; +import { + createAtlasCloudRequestBody, + createAtlasCloudVideoRequestBody, +} from "@/lib/ai/atlascloud-request"; import { is3DModelDescriptor, isImageGenerationModelDescriptor, @@ -37,6 +41,13 @@ test.describe("AI model media classification", () => { ).toBe("Flux Kontext Pro"); }); + test("adds a missing Atlas media kind without restoring deleted models in an existing kind", () => { + const existingKinds = new Set(["image", "3d"]); + expect(shouldInsertMissingAtlasCloudPreset(existingKinds, "video")).toBe(true); + expect(shouldInsertMissingAtlasCloudPreset(existingKinds, "image")).toBe(false); + expect(shouldInsertMissingAtlasCloudPreset(existingKinds, "3d")).toBe(false); + }); + test("keeps multi-reference BananaImage models classified as image models", () => { const descriptor = { providerType: "bananaimage", @@ -206,4 +217,92 @@ test.describe("AI model media classification", () => { }, ]); }); + + test("maps Atlas video schema capabilities and frontend parameters", () => { + const preset = createAtlasCloudImagePreset( + { + model: "bytedance/seedance-2.0-mini/image-to-video", + type: "Video", + displayName: "Seedance 2.0 Mini Image to Video", + categories: ["IMAGE-TO-VIDEO"], + }, + { + paths: { + "/api/v1/model/generateVideo": { post: {} }, + "/api/v1/model/prediction/{request_id}": { get: {} }, + }, + components: { + schemas: { + Input: { + type: "object", + properties: { + model: { type: "string" }, + prompt: { type: "string" }, + image: { type: "string", "x-format": "image" }, + last_image: { type: "string", "x-format": "image" }, + resolution: { + type: "string", + enum: ["480p", "720p", "1080p"], + default: "720p", + }, + ratio: { + type: "string", + enum: ["16:9", "9:16", "adaptive"], + default: "adaptive", + }, + duration: { + type: "integer", + enum: [5, 10], + default: 5, + }, + seed: { type: "integer" }, + watermark: { type: "boolean", default: false }, + }, + required: ["model", "prompt", "image"], + }, + }, + }, + } + ); + + expect(preset.supportsImageGen).toBe(false); + expect(preset.capabilities).toMatchObject({ + kind: "video", + mediaType: "video", + supportedModes: ["first-frame", "first-last-frame"], + supportedResolutions: ["480p", "720p", "1080p"], + supportedRatios: ["16:9", "9:16", "adaptive"], + durationOptions: [5, 10], + }); + + const { body } = createAtlasCloudVideoRequestBody({ + modelId: preset.modelId, + options: { + model: preset.modelId, + prompt: "让商品缓慢旋转", + mode: "first-last-frame", + firstFrameImage: "https://example.com/first.png", + lastFrameImage: "https://example.com/last.png", + resolution: "1080p", + ratio: "9:16", + duration: 10, + watermark: true, + }, + referenceImages: [ + "https://example.com/first.png", + "https://example.com/last.png", + ], + modelConfig: preset.imageGenConfig, + }); + expect(body).toMatchObject({ + model: preset.modelId, + prompt: "让商品缓慢旋转", + image: "https://example.com/first.png", + last_image: "https://example.com/last.png", + resolution: "1080p", + ratio: "9:16", + duration: 10, + watermark: true, + }); + }); }); From 62c686a26bc0b117fec019592ddbd9211f4163bd Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 24 Jul 2026 13:51:06 +0800 Subject: [PATCH 09/83] fix: fail over on upstream image provider errors --- lib/ai/providers/gemini-image.ts | 81 +++++++++++++++++++++++++++----- lib/ai/runtime/model-health.ts | 49 +++++++++++++++++-- 2 files changed, 115 insertions(+), 15 deletions(-) diff --git a/lib/ai/providers/gemini-image.ts b/lib/ai/providers/gemini-image.ts index b3c5c5fb1..b507a3940 100644 --- a/lib/ai/providers/gemini-image.ts +++ b/lib/ai/providers/gemini-image.ts @@ -716,12 +716,11 @@ async function sendGrsaiGptImageRequest(params: { response.status, responseText.substring(0, 1000) ); + const providerError = getGrsaiErrorMessage(data); const errorMessage = - data?.error?.message || - data?.message || - data?.error || - responseText || - `${response.status} ${response.statusText}`; + providerError !== "未知错误" + ? providerError + : responseText || `${response.status} ${response.statusText}`; throw new Error(`图片生成失败: ${errorMessage}`); } @@ -754,6 +753,13 @@ async function sendGrsaiGptImageRequest(params: { timeoutLabel: "Grsai GPT Image", }); } + + const providerError = getGrsaiErrorMessage(data); + throw new Error( + `图片生成失败: 未返回任务 ID 或可用图片${ + providerError !== "未知错误" ? `: ${providerError}` : "" + }` + ); } return data; @@ -857,18 +863,46 @@ function getGrsaiProgress(payload: any): number | null { function getGrsaiErrorMessage(payload: any): string { const normalized = normalizeGrsaiPayload(payload); + const normalizePart = (part: unknown) => { + if (typeof part === "string" || typeof part === "number") { + return String(part).trim(); + } + if (!isRecord(part)) return ""; + return String( + part.message || + part.msg || + part.detail || + part.reason || + part.code || + part.type || + "" + ).trim(); + }; const messages = [ normalized?.failure_reason, + normalized?.error?.message, + normalized?.error?.msg, + normalized?.error?.code, normalized?.error, normalized?.message, + normalized?.msg, + normalized?.detail, + normalized?.reason, + normalized?.code, + payload?.error?.message, + payload?.error?.msg, + payload?.error?.code, payload?.msg, payload?.message, payload?.error, + payload?.detail, + payload?.reason, + payload?.code, ] - .map((part) => String(part || "").trim()) + .map(normalizePart) .filter(Boolean); - return messages.join(" / ") || "未知错误"; + return Array.from(new Set(messages)).join(" / ") || "未知错误"; } function hasGrsaiImages(payload: any): boolean { @@ -899,18 +933,36 @@ async function fetchWithTimeout( async function readJsonResponse(response: Response, label: string): Promise { const text = await response.text(); + const payload = parseGrsaiJsonOrStreamResponse(text); if (!response.ok) { logger.error(`${label} failed:`, response.status, text.substring(0, 500)); - throw new Error(`${label} 接口错误：${response.status} ${response.statusText}`); + const providerError = getGrsaiErrorMessage(payload); + const detail = + providerError !== "未知错误" + ? providerError + : String(payload?.raw || text || "").trim().slice(0, 300); + throw new Error( + `${label} 接口错误：${response.status} ${response.statusText}${ + detail ? `: ${detail}` : "" + }` + ); } - try { - return JSON.parse(text); - } catch (error) { + if (isRecord(payload) && !("raw" in payload)) { + return payload; + } + + if (!isRecord(payload)) { + return payload; + } + + if ("raw" in payload) { logger.error(`${label} returned non-JSON response:`, text.substring(0, 500)); throw new Error(`${label} 返回格式异常，未能解析任务结果`); } + + return payload; } async function pollGrsaiNanoBananaResult(params: { @@ -1047,11 +1099,16 @@ async function submitAndPollGrsaiNanoBanana(params: { const taskId = getGrsaiTaskId(submitPayload); if (!taskId) { + const providerError = getGrsaiErrorMessage(submitPayload); logger.error( `${params.apiLogPrefix} Async submit response has no task id:`, JSON.stringify(submitPayload).substring(0, 500) ); - throw new Error("GRSAI Nano Banana 异步任务提交失败：未返回任务 ID"); + throw new Error( + `GRSAI Nano Banana 异步任务提交失败：未返回任务 ID${ + providerError !== "未知错误" ? `: ${providerError}` : "" + }` + ); } logger.debug(`${params.apiLogPrefix} Async task submitted:`, { taskId }); diff --git a/lib/ai/runtime/model-health.ts b/lib/ai/runtime/model-health.ts index 765cf910c..8ab443831 100644 --- a/lib/ai/runtime/model-health.ts +++ b/lib/ai/runtime/model-health.ts @@ -151,7 +151,10 @@ export function classifyAiModelHealthFailure(error: unknown): { if ( normalized.includes("insufficient balance") || + normalized.includes("insufficient credit") || normalized.includes("credits insufficient") || + normalized.includes("credit insufficient") || + normalized.includes("not enough credit") || normalized.includes("quota") || normalized.includes("余额") || normalized.includes("额度") || @@ -209,10 +212,38 @@ export function classifyAiModelHealthFailure(error: unknown): { }; } + if ( + normalized.includes("未返回任务 id") || + normalized.includes("未返回任务id") || + normalized.includes("no task id") || + normalized.includes("missing task id") || + normalized.includes("provider error") || + normalized.includes("provider_error") || + normalized.includes("provider request failed") || + normalized.includes("provider task failed") || + normalized.includes("upstream error") || + normalized.includes("upstream_error") || + normalized.includes("upstream request failed") || + normalized.includes("上游服务错误") || + normalized.includes("上游接口错误") + ) { + return { + eventType: "unavailable", + severity: "critical", + counted: true, + reason: "provider_execution_failed", + }; + } + if ( normalized.includes("download") || normalized.includes("no image url") || normalized.includes("no image") || + normalized.includes("empty result") || + normalized.includes("empty response") || + normalized.includes("空结果") || + normalized.includes("返回结果为空") || + normalized.includes("响应为空") || normalized.includes("未返回可用图片") || normalized.includes("未包含可用图片") || normalized.includes("未解析到图片") || @@ -245,14 +276,21 @@ export function shouldFailoverAiModelError(error: unknown) { normalized.includes("snapgen 任务已完成但未返回图片地址") || normalized.includes("snapgen 未返回可用图片"); - if (imageProviderTaskFailed) { - return true; - } + const providerExecutionFailed = + classification.reason === "provider_execution_failed" || + normalized.includes("未返回任务 id") || + normalized.includes("未返回任务id") || + normalized.includes("no task id") || + normalized.includes("missing task id"); if (classification.reason === "user_cancelled" || classification.reason === "non_provider_failure") { return false; } + if (imageProviderTaskFailed || providerExecutionFailed) { + return true; + } + if ( classification.eventType === "timeout" || classification.reason === "transport_failure_no_provider_response" @@ -263,6 +301,11 @@ export function shouldFailoverAiModelError(error: unknown) { if (classification.eventType === "download_error") { return ( normalized.includes("no image") || + normalized.includes("empty result") || + normalized.includes("empty response") || + normalized.includes("空结果") || + normalized.includes("返回结果为空") || + normalized.includes("响应为空") || normalized.includes("未返回可用图片") || normalized.includes("未包含可用图片") || normalized.includes("未解析到图片") || From c889bfc381f999b862b817abc10c5a32d5bf0b97 Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 24 Jul 2026 14:10:13 +0800 Subject: [PATCH 10/83] feat: add product planning v2 claw skill --- lib/canvas-skill-tags.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/canvas-skill-tags.ts b/lib/canvas-skill-tags.ts index 5e2932df3..f80312712 100644 --- a/lib/canvas-skill-tags.ts +++ b/lib/canvas-skill-tags.ts @@ -1,6 +1,6 @@ export const CANVAS_SKILL_TAGS_SETTING_KEY = "canvas_skill_tags"; export const CANVAS_SKILL_TAG_ORDER_SETTING_KEY_PREFIX = "canvas_skill_tag_order:"; -export const CANVAS_SKILL_TAGS_CONFIG_VERSION = 4; +export const CANVAS_SKILL_TAGS_CONFIG_VERSION = 5; export const DEFAULT_CANVAS_SKILL_TAGS_TITLE = "Claw 快捷技能"; export type CanvasSkillTagSettings = { @@ -44,6 +44,14 @@ export const DEFAULT_CANVAS_SKILL_TAGS: CanvasSkillTag[] = [ prompt: "【技能路由：product-asset-api】用途：查询商品详情页、主图、SKU 图、白底图下载链接。触发关键词：详情页、详情页面、详情素材、详情整套、任务详情、SKU图、主图、白底图、白底、SKU:、SPU:、货号 + 详情/图片/白底需求。必须要求用户提供 SKU 或 SPU；提到白底/白底图时查白底图，其他详情/主图/SKU图查任务详情素材。输出下载链接，不默认展开图片预览；命中多组候选时默认展示最新/最相关 1 组，并提示可展开全部详情链接；未命中时说明 SKU/SPU 可能不正确或素材未入库。", }, + { + id: "product-detail-planning-v2", + name: "商品策划案 V2", + iconName: "sparkles", + placeholder: "输入 SKU/SPU、产品名或上传产品图，开始商品策划案 V2", + prompt: + "【技能路由：product-detail-planning-v2】用途：通过可恢复的分步对话完成商品详情页全案策划，覆盖产品事实、竞品、真实评价与问大家、内容策划、视觉策划、详情页提示词、五张主图、主图视频、AI 买家秀，以及最终 HTML 和 17 子表 Excel 交付。触发关键词：商品策划案、详情页策划、完整详情策划、新品详情策划、竞品详情分析、详情页改版、主图策划、SKU图策划、主图视频策划、AI买家秀策划、继续详情策划、新版商品策划案、商品策划案V2。通用“商品策划案”默认进入 V2，只有用户明确要求旧版/V1时才使用兼容流程。公司已有商品可提供 SKU/SPU；新品可以提供产品名、现有资料并上传产品图。每轮只处理当前节点，完整展示当前节点数据、系统最优推荐和可选动作，但不得替用户确认或跨节点推进；图片只证明可见结构，不得自动证明材质、参数、检测或服务承诺。全部必需节点完成后生成 HTML 策划案和固定 17 子表 Excel，并返回查看或下载入口。", + }, { id: "install-media-db", name: "安装资料", From 80d05e177a420173f5b116f3c3d56c00d80a7cb5 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 27 Jul 2026 10:22:03 +0800 Subject: [PATCH 11/83] fix: keep sent Claw messages out of composer --- app/api/canvas/clawd-chat/route.ts | 6 -- components/canvas/CanvasChatPanel.tsx | 59 ++++++-------------- lib/openclaw-chat/request-lifecycle.ts | 16 ++++++ playwright.helpers.config.ts | 1 + tests/routes/clawd-request-lifecycle.test.ts | 51 +++++++++++++++++ 5 files changed, 84 insertions(+), 49 deletions(-) create mode 100644 lib/openclaw-chat/request-lifecycle.ts create mode 100644 tests/routes/clawd-request-lifecycle.test.ts diff --git a/app/api/canvas/clawd-chat/route.ts b/app/api/canvas/clawd-chat/route.ts index 01479493a..d360ece3e 100644 --- a/app/api/canvas/clawd-chat/route.ts +++ b/app/api/canvas/clawd-chat/route.ts @@ -15,7 +15,6 @@ import { } from "@/lib/openclaw-chat/config"; import { buildClawdCancelKey, - cancelClawdRequest, clearClawdRequestCancellation, isClawdRequestCanceled, registerActiveClawdRequest, @@ -259,10 +258,6 @@ export async function POST(request: Request) { const controller = new AbortController(); registerActiveClawdRequest(cancelKey, controller, config.timeoutMs); const timeout = setTimeout(() => controller.abort(), config.timeoutMs); - const handleRequestAbort = () => { - cancelClawdRequest(cancelKey); - }; - request.signal.addEventListener("abort", handleRequestAbort, { once: true }); const response = await fetch(endpoint, { method: "POST", headers: { @@ -302,7 +297,6 @@ export async function POST(request: Request) { signal: controller.signal, }).finally(() => { clearTimeout(timeout); - request.signal.removeEventListener("abort", handleRequestAbort); unregisterActiveClawdRequest(cancelKey); }); diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 8c35ab4a2..5f8f3529d 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -101,6 +101,10 @@ import { normalizeCanvasSkillTags, type CanvasSkillTag, } from "@/lib/canvas-skill-tags"; +import { + isClawdTransportInterruption, + shouldRestoreClawdComposerAfterFailure, +} from "@/lib/openclaw-chat/request-lifecycle"; import { isCanvasImageReadyForAi, isCanvasImageUploadPending, @@ -11199,8 +11203,7 @@ export function CanvasChatPanel({ }); const responseData = await response.json().catch(() => null); if (response.status === 499 || responseData?.canceled === true) { - canceledClawdMessageIdsRef.current.add(userMessageId); - throw new DOMException("Claw request canceled", "AbortError"); + throw new Error("Claw request interrupted"); } if (!response.ok) { console.error("[CanvasChat] Claw API returned error:", { @@ -11251,10 +11254,13 @@ export function CanvasChatPanel({ void mutateClawdHistory(); hasHydratedHistoryRef.current = true; } catch (error) { - const wasCanceled = - canceledClawdMessageIdsRef.current.has(userMessageId) || - (error instanceof Error && error.name === "AbortError"); - if (wasCanceled) { + const explicitlyCanceled = + canceledClawdMessageIdsRef.current.has(userMessageId); + if ( + shouldRestoreClawdComposerAfterFailure({ + explicitlyCanceled, + }) + ) { if (pendingClawdRequestRestoreRef.current === clawdRequestRestoreSnapshot) { pendingClawdRequestRestoreRef.current = null; restorePendingClawdRequestToComposer(clawdRequestRestoreSnapshot); @@ -11263,44 +11269,11 @@ export function CanvasChatPanel({ return; } console.error("[CanvasChat] Claw request failed:", error); - pendingClawdSubmissionRef.current = null; - if (recentClawdSubmissionRef.current === clawdSubmissionSnapshot) { - recentClawdSubmissionRef.current = null; - writeStoredClawdComposerSubmissionSnapshot(null); - } - setMessages((previousMessages) => - previousMessages.filter((message) => message.id !== userMessageId) + toast.error( + isClawdTransportInterruption(error) + ? "连接已中断，任务仍可能在后台处理，请稍后查看 Claw 历史" + : CLAW_UPGRADE_RETRY_MESSAGE ); - toast.error(CLAW_UPGRADE_RETRY_MESSAGE); - if (!isProgrammaticClawdQuickSend) { - setClawdTableFiles(tableFilesSnapshot); - setAttachments(attachmentsSnapshot); - if (clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt) { - setSelectedSkillTagId(clawdSkillTagForRequest.id); - pendingSelectedClawdSkillTagRef.current = clawdSkillTagForRequest; - } - const editableDiv = textareaRef.current as HTMLDivElement | null; - const currentEditorText = (editableDiv?.textContent || "") - .replace(/\u200b/g, " ") - .replace(/\s+/g, " ") - .trim(); - if (!currentEditorText) { - setInput(textContent); - if (editableDiv) { - editableDiv.textContent = textContent; - window.setTimeout(() => { - attachmentsSnapshot.forEach((attachment, index) => { - if (!attachment.url) return; - insertImageTagInEditor( - attachment.url, - normalizeChatAttachmentName(attachment.name), - index - ); - }); - }, 0); - } - } - } } finally { pendingClawdSubmissionRef.current = null; if (pendingClawdRequestRestoreRef.current === clawdRequestRestoreSnapshot) { diff --git a/lib/openclaw-chat/request-lifecycle.ts b/lib/openclaw-chat/request-lifecycle.ts new file mode 100644 index 000000000..91bbc9f08 --- /dev/null +++ b/lib/openclaw-chat/request-lifecycle.ts @@ -0,0 +1,16 @@ +export function shouldRestoreClawdComposerAfterFailure({ + explicitlyCanceled, +}: { + explicitlyCanceled: boolean; +}) { + return explicitlyCanceled; +} + +export function isClawdTransportInterruption(error: unknown) { + if (!(error instanceof Error)) return false; + + return ( + error.name === "AbortError" || + /aborted|failed to fetch|fetch failed|network|interrupted/i.test(error.message) + ); +} diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index 6862b52e1..1c10d4ef4 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ /routes\/canvas-scale-adjust-intent\.test\.ts/, /routes\/canvas-runtime-internal-endpoint\.test\.ts/, /routes\/model-classification\.test\.ts/, + /routes\/clawd-request-lifecycle\.test\.ts/, ], fullyParallel: true, forbidOnly: !!process.env.CI, diff --git a/tests/routes/clawd-request-lifecycle.test.ts b/tests/routes/clawd-request-lifecycle.test.ts new file mode 100644 index 000000000..e501f02ae --- /dev/null +++ b/tests/routes/clawd-request-lifecycle.test.ts @@ -0,0 +1,51 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { expect, test } from "@playwright/test"; +import { + isClawdTransportInterruption, + shouldRestoreClawdComposerAfterFailure, +} from "@/lib/openclaw-chat/request-lifecycle"; + +function readSource(path: string) { + return readFileSync(resolve(process.cwd(), path), "utf8"); +} + +test.describe("Claw request lifecycle", () => { + test("restores the composer only after an explicit user cancellation", () => { + expect( + shouldRestoreClawdComposerAfterFailure({ explicitlyCanceled: true }) + ).toBe(true); + expect( + shouldRestoreClawdComposerAfterFailure({ explicitlyCanceled: false }) + ).toBe(false); + }); + + test("recognizes browser and network interruptions without treating them as edits", () => { + expect( + isClawdTransportInterruption( + new DOMException("The operation was aborted", "AbortError") + ) + ).toBe(true); + expect(isClawdTransportInterruption(new Error("fetch failed"))).toBe(true); + expect(isClawdTransportInterruption(new Error("provider unavailable"))).toBe( + false + ); + }); + + test("does not cancel the upstream request when the browser connection closes", () => { + const source = readSource("app/api/canvas/clawd-chat/route.ts"); + + expect(source).not.toContain('request.signal.addEventListener("abort"'); + expect(source).not.toContain("cancelClawdRequest(cancelKey)"); + expect(source).toContain("registerActiveClawdRequest(cancelKey"); + }); + + test("does not classify every AbortError as an explicit stop action", () => { + const source = readSource("components/canvas/CanvasChatPanel.tsx"); + + expect(source).toContain("shouldRestoreClawdComposerAfterFailure"); + expect(source).not.toContain( + '(error instanceof Error && error.name === "AbortError")' + ); + }); +}); From de8b7670b222330bc2c1f76806c6f978f885c95f Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 27 Jul 2026 17:30:01 +0800 Subject: [PATCH 12/83] fix: support Claw AVIF and TIFF uploads --- app/(chat)/api/files/upload/route.ts | 79 ++++++++++++++++++++++++++- components/canvas/CanvasChatPanel.tsx | 46 +++++++++++++--- lib/storage/service.ts | 5 ++ 3 files changed, 119 insertions(+), 11 deletions(-) diff --git a/app/(chat)/api/files/upload/route.ts b/app/(chat)/api/files/upload/route.ts index 0a5297cb1..831484ff4 100644 --- a/app/(chat)/api/files/upload/route.ts +++ b/app/(chat)/api/files/upload/route.ts @@ -14,6 +14,9 @@ const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([ "image/jpg", "image/png", "image/webp", + "image/avif", + "image/tiff", + "image/x-tiff", "image/svg+xml", "image/heic", "image/heif", @@ -25,6 +28,11 @@ const ALLOWED_ATLASCLOUD_VIDEO_TYPES = new Set([ ]); const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]); +const SHARP_COMPATIBILITY_UPLOAD_TYPES = new Set([ + "image/avif", + "image/tiff", + "image/x-tiff", +]); function normalizeUploadContentType(contentType: string, filename: string) { const normalized = contentType.trim().toLowerCase(); @@ -40,8 +48,16 @@ function createUnsupportedImageTypeMessage(contentType: string, filename: string if (contentType === "image/heic" || contentType === "image/heif" || extension === "heic" || extension === "heif") { return "HEIC/HEIF 图片转码失败，请在手机相册中导出为 JPG/PNG 后再上传。"; } + if ( + SHARP_COMPATIBILITY_UPLOAD_TYPES.has(contentType) || + extension === "avif" || + extension === "tif" || + extension === "tiff" + ) { + return "AVIF/TIFF 图片转码失败，请导出为 JPG/PNG 后再上传。"; + } - return "File type should be JPEG, PNG, WebP or SVG"; + return "图片格式不支持，请上传 JPG、PNG、WebP、HEIC、AVIF 或 TIFF 图片。"; } function isHeicUpload(contentType: string, filename: string) { @@ -53,10 +69,20 @@ function isHeicUpload(contentType: string, filename: string) { ); } +function isSharpCompatibilityUpload(contentType: string, filename: string) { + const extension = filename.toLowerCase().split(".").pop() || ""; + return ( + SHARP_COMPATIBILITY_UPLOAD_TYPES.has(contentType) || + extension === "avif" || + extension === "tif" || + extension === "tiff" + ); +} + function toJpegFilename(filename: string) { const trimmed = filename.trim() || `upload-${Date.now()}`; - if (/\.(heic|heif)$/i.test(trimmed)) { - return trimmed.replace(/\.(heic|heif)$/i, ".jpg"); + if (/\.(avif|heic|heif|tif|tiff)$/i.test(trimmed)) { + return trimmed.replace(/\.(avif|heic|heif|tif|tiff)$/i, ".jpg"); } if (/\.[a-z0-9]+$/i.test(trimmed)) { return trimmed.replace(/\.[a-z0-9]+$/i, ".jpg"); @@ -76,6 +102,21 @@ async function convertHeicToJpeg(buffer: Buffer): Promise { } return Buffer.from(output); } + +async function convertSharpCompatibilityImageToJpeg( + buffer: Buffer +): Promise { + const sharp = (await import("sharp")).default; + return sharp(buffer, { animated: false, failOn: "none" }) + .rotate() + .flatten({ background: "#ffffff" }) + .jpeg({ + quality: 92, + mozjpeg: true, + progressive: true, + }) + .toBuffer(); +} /** * 使用Sharp智能优化图片（保持原格式） * 策略：避免双重压缩，只在需要时优化 @@ -288,6 +329,38 @@ export async function POST(request: Request) { { status: 400 } ); } + } else if (isSharpCompatibilityUpload(contentType, filename)) { + try { + const convertedBuffer = + await convertSharpCompatibilityImageToJpeg(uploadBuffer); + logger.info("[Upload API] Compatibility image converted to JPEG", { + clientUploadId, + originalContentType: contentType, + originalSize: uploadBuffer.length, + convertedSize: convertedBuffer.length, + }); + uploadBuffer = convertedBuffer; + filename = toJpegFilename(filename); + convertedFrom = contentType; + contentType = "image/jpeg"; + } catch (conversionError) { + logger.warn("[Upload API] Compatibility image conversion failed", { + clientUploadId, + contentType, + size: uploadBuffer.length, + error: + conversionError instanceof Error + ? conversionError.message + : String(conversionError), + }); + return NextResponse.json( + { + error: createUnsupportedImageTypeMessage(contentType, filename), + clientUploadId, + }, + { status: 400 } + ); + } } if (uploadPurpose === "agent-workflow-preview" && contentType !== "image/svg+xml") { diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 5f8f3529d..acb17211f 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -415,6 +415,19 @@ function isClawdChatMessage(message: unknown) { const CLAWD_TABLE_ACCEPT = ".xlsx,.csv,.tsv,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,text/csv,text/tab-separated-values"; const CLAWD_TABLE_ALLOWED_EXTENSIONS = new Set(["xlsx", "csv", "tsv"]); +const CLAWD_IMAGE_ACCEPT = + ".jpg,.jpeg,.png,.webp,.heic,.heif,.avif,.tif,.tiff,image/jpeg,image/png,image/webp,image/heic,image/heif,image/avif,image/tiff"; +const CLAWD_IMAGE_ALLOWED_EXTENSIONS = new Set([ + "jpg", + "jpeg", + "png", + "webp", + "heic", + "heif", + "avif", + "tif", + "tiff", +]); const CLAWD_TABLE_MAX_FILES = 3; const CLAWD_TABLE_MAX_BYTES = 10 * 1024 * 1024; const CLAWD_PROCESSING_HINT = @@ -444,6 +457,13 @@ function isClawdTableFile(file: File) { ); } +function isClawdImageFile(file: File) { + return ( + file.type.startsWith("image/") || + CLAWD_IMAGE_ALLOWED_EXTENSIONS.has(getClawdTableFileExtension(file.name)) + ); +} + function normalizeClawdComposerText(value: string) { return String(value || "") .replace(/\u200b/g, " ") @@ -11906,7 +11926,7 @@ export function CanvasChatPanel({ width?: number; height?: number; } | null> => { - if (!file.type.startsWith("image/")) { + if (!isClawdImageFile(file)) { toast.error("请选择图片文件"); return null; } @@ -11936,7 +11956,15 @@ export function CanvasChatPanel({ } : undefined); if (!response.ok) { - throw new Error("上传失败"); + const payload = (await response.json().catch(() => null)) as { + error?: string; + details?: string; + } | null; + throw new Error( + payload?.error?.trim() || + payload?.details?.trim() || + "图片上传失败" + ); } const data = await response.json(); @@ -11954,7 +11982,11 @@ export function CanvasChatPanel({ assetId, }; } catch (_error) { - toast.error("图片上传失败"); + toast.error( + _error instanceof Error && _error.message + ? _error.message + : "图片上传失败" + ); return null; } }, [clearComposerUploadProgress, updateComposerUploadProgress]); @@ -12289,11 +12321,9 @@ export function CanvasChatPanel({ if (isClawdMode) { const tableFiles = selectedFiles.filter((file) => isClawdTableFile(file)); - const imageFiles = selectedFiles.filter((file) => - file.type.startsWith("image/") - ); + const imageFiles = selectedFiles.filter((file) => isClawdImageFile(file)); const unsupportedFiles = selectedFiles.filter( - (file) => !isClawdTableFile(file) && !file.type.startsWith("image/") + (file) => !isClawdTableFile(file) && !isClawdImageFile(file) ); unsupportedFiles.forEach((file) => { @@ -16095,7 +16125,7 @@ export function CanvasChatPanel({ type="file" accept={ isClawdMode - ? `image/*,${CLAWD_TABLE_ACCEPT}` + ? `${CLAWD_IMAGE_ACCEPT},${CLAWD_TABLE_ACCEPT}` : supportsAtlasVideoReference ? "image/*,video/mp4,video/quicktime,video/webm" : "image/*" diff --git a/lib/storage/service.ts b/lib/storage/service.ts index 8027d503c..7e6da8547 100644 --- a/lib/storage/service.ts +++ b/lib/storage/service.ts @@ -109,6 +109,11 @@ export function getContentType(filename: string): string { png: "image/png", gif: "image/gif", webp: "image/webp", + avif: "image/avif", + tif: "image/tiff", + tiff: "image/tiff", + heic: "image/heic", + heif: "image/heif", svg: "image/svg+xml", psd: "image/vnd.adobe.photoshop", psb: "application/octet-stream", From 11deb3bba668003fc27f1aa611377ca18c8e1f60 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 27 Jul 2026 18:24:20 +0800 Subject: [PATCH 13/83] perf: streamline HiCAD startup --- deploy/nginx-hicad-zoo-ws.conf.template | 1 + playwright.helpers.config.ts | 1 + scripts/vendor-zoo-modeling-app.mjs | 31 ++++++++++++- .../routes/hicad-embedded-performance.test.ts | 43 +++++++++++++++++++ vendor/zoo-modeling-app/localization.ts | 1 + 5 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/routes/hicad-embedded-performance.test.ts diff --git a/deploy/nginx-hicad-zoo-ws.conf.template b/deploy/nginx-hicad-zoo-ws.conf.template index 3106772db..a68500f8d 100644 --- a/deploy/nginx-hicad-zoo-ws.conf.template +++ b/deploy/nginx-hicad-zoo-ws.conf.template @@ -17,6 +17,7 @@ location ^~ /vendor/zoo-modeling-app/fonts/ { location = /vendor/zoo-modeling-app/kcl_wasm_lib_bg.wasm { root /var/www/ai-chatbot/public; try_files $uri =404; + gzip_static on; add_header Cache-Control "public, max-age=3600" always; } diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index 1c10d4ef4..f30d2518d 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ /routes\/canvas-runtime-internal-endpoint\.test\.ts/, /routes\/model-classification\.test\.ts/, /routes\/clawd-request-lifecycle\.test\.ts/, + /routes\/hicad-embedded-performance\.test\.ts/, ], fullyParallel: true, forbidOnly: !!process.env.CI, diff --git a/scripts/vendor-zoo-modeling-app.mjs b/scripts/vendor-zoo-modeling-app.mjs index 871816cfe..391df5af4 100644 --- a/scripts/vendor-zoo-modeling-app.mjs +++ b/scripts/vendor-zoo-modeling-app.mjs @@ -140,6 +140,12 @@ function patchOfficialSource() { " Zoo Design Studio", " HiCAD" ); + replaceSource( + "index.html", + ' ', + ` + ` + ); replaceSource( "src/Router.tsx", " createBrowserRouter,\n createHashRouter,", @@ -225,6 +231,19 @@ function patchOfficialSource() { ` className="cursor-pointer"\n id="freeze-frame"`, ` className="absolute inset-0 h-full w-full cursor-pointer"\n id="freeze-frame"` ); + replaceSource( + "src/components/OpenedProject.tsx", + "import { getMlEphantProjectReloadBehavior } from '@src/components/openedProjectUtils'\n", + "import { getMlEphantProjectReloadBehavior } from '@src/components/openedProjectUtils'\nimport env from '@src/env'\n" + ); + replaceSource( + "src/components/OpenedProject.tsx", + ` const needsOnboarded = + !window.electron &&`, + ` const needsOnboarded = + env().VITE_ZOO_API_TOKEN !== '${proxyToken}' && + !window.electron &&` + ); replaceSource( "src/index.tsx", "import { markOnce } from '@src/lib/performance'\n", @@ -357,7 +376,7 @@ function ensureWasm() { function precompressVersionedAssets(assetsDir) { for (const entry of readdirSync(assetsDir, { withFileTypes: true })) { - if (!entry.isFile() || !/\.(?:css|js)$/.test(entry.name)) { + if (!entry.isFile() || !/\.(?:css|js|wasm)$/.test(entry.name)) { continue; } const assetPath = path.join(assetsDir, entry.name); @@ -365,6 +384,15 @@ function precompressVersionedAssets(assetsDir) { } } +function precompressRootWasm(buildDir) { + const wasmPath = path.join(buildDir, "kcl_wasm_lib_bg.wasm"); + if (!existsSync(wasmPath)) return; + writeFileSync( + `${wasmPath}.gz`, + gzipSync(readFileSync(wasmPath), { level: 9 }) + ); +} + async function main() { ensureOfficialSource(); ensureDependencies(); @@ -393,6 +421,7 @@ async function main() { throw new Error("Zoo Design Studio build did not produce index.html"); } precompressVersionedAssets(path.join(buildDir, "assets")); + precompressRootWasm(buildDir); cacheBustZooModelingAppAssets(buildDir); rmSync(outputDir, { force: true, recursive: true }); diff --git a/tests/routes/hicad-embedded-performance.test.ts b/tests/routes/hicad-embedded-performance.test.ts new file mode 100644 index 000000000..40e936a5f --- /dev/null +++ b/tests/routes/hicad-embedded-performance.test.ts @@ -0,0 +1,43 @@ +import { expect, test } from "@playwright/test"; +import fs from "node:fs"; +import path from "node:path"; + +const projectRoot = process.cwd(); + +test.describe("HiCAD embedded startup", () => { + test("suppresses the upstream onboarding chooser in embedded mode", () => { + const vendorScript = fs.readFileSync( + path.join(projectRoot, "scripts/vendor-zoo-modeling-app.mjs"), + "utf8" + ); + const localization = fs.readFileSync( + path.join(projectRoot, "vendor/zoo-modeling-app/localization.ts"), + "utf8" + ); + + expect(vendorScript).toMatch( + /env\(\)\.VITE_ZOO_API_TOKEN !== '\$\{proxyToken\}'/ + ); + expect(localization).toContain("[data-testid='onboarding-toast']"); + }); + + test("preloads and serves the CAD wasm using a precompressed asset", () => { + const vendorScript = fs.readFileSync( + path.join(projectRoot, "scripts/vendor-zoo-modeling-app.mjs"), + "utf8" + ); + const nginxTemplate = fs.readFileSync( + path.join(projectRoot, "deploy/nginx-hicad-zoo-ws.conf.template"), + "utf8" + ); + + expect(vendorScript).toContain( + 'rel="preload" href="./kcl_wasm_lib_bg.wasm"' + ); + expect(vendorScript).toContain("/\\.(?:css|js|wasm)$/"); + expect(vendorScript).toContain("precompressRootWasm(buildDir)"); + expect(nginxTemplate).toMatch( + /location = \/vendor\/zoo-modeling-app\/kcl_wasm_lib_bg\.wasm \{[\s\S]*?gzip_static on;/ + ); + }); +}); diff --git a/vendor/zoo-modeling-app/localization.ts b/vendor/zoo-modeling-app/localization.ts index 7d8782b22..37f02b274 100644 --- a/vendor/zoo-modeling-app/localization.ts +++ b/vendor/zoo-modeling-app/localization.ts @@ -199,6 +199,7 @@ export function installHiCadLocalization() { } html[data-hicad-embedded='true'] [data-testid='download-desktop-app'], html[data-hicad-embedded='true'] [data-testid='billing-remaining-bar'], + html[data-hicad-embedded='true'] [data-testid='onboarding-toast'], html[data-hicad-embedded='true'] [data-hicad-official-account-link] { display: none !important; } From f65b9e047261840c9650b97a5d59b3c8439daeb6 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 27 Jul 2026 19:20:20 +0800 Subject: [PATCH 14/83] fix: frame complete HiCAD models after generation --- scripts/vendor-zoo-modeling-app.mjs | 19 +++++++++++++++++++ .../routes/hicad-embedded-performance.test.ts | 14 ++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/scripts/vendor-zoo-modeling-app.mjs b/scripts/vendor-zoo-modeling-app.mjs index 391df5af4..bc596e1f6 100644 --- a/scripts/vendor-zoo-modeling-app.mjs +++ b/scripts/vendor-zoo-modeling-app.mjs @@ -269,6 +269,25 @@ function patchOfficialSource() { ` // We need a padding of 0.1 for zoom_to_fit for all E2E tests since they were originally\n // written with zoom_to_fit with padding 0.1\n const padding = 0.1\n if (isPlaywright()) {`, ` // Keep the official E2E framing stable while giving HiCAD users enough\n // breathing room to see the complete model after generation and file loads.\n const padding = isPlaywright() ? 0.1 : 0.35\n if (isPlaywright()) {` ); + replaceSource( + "src/lib/resetCameraPosition.ts", + ` await engineViewIsometric({ + engineCommandManager, + padding, + })`, + ` await engineViewIsometric({ + engineCommandManager, + padding, + }) + + // view_isometric changes the direction but can preserve the previous + // close-up distance. Refit against every generated entity so newly + // completed models open fully framed instead of showing a local detail. + await engineStreamZoomToFit({ + engineCommandManager, + padding, + })` + ); replaceSource( "src/lib/commandBarConfigs/standardViewsConfig.ts", "engineStreamZoomToFit({ engineCommandManager, padding: 0.1 })", diff --git a/tests/routes/hicad-embedded-performance.test.ts b/tests/routes/hicad-embedded-performance.test.ts index 40e936a5f..32f56b122 100644 --- a/tests/routes/hicad-embedded-performance.test.ts +++ b/tests/routes/hicad-embedded-performance.test.ts @@ -40,4 +40,18 @@ test.describe("HiCAD embedded startup", () => { /location = \/vendor\/zoo-modeling-app\/kcl_wasm_lib_bg\.wasm \{[\s\S]*?gzip_static on;/ ); }); + + test("fits the complete generated model after selecting the isometric view", () => { + const vendorScript = fs.readFileSync( + path.join(projectRoot, "scripts/vendor-zoo-modeling-app.mjs"), + "utf8" + ); + + expect(vendorScript).toMatch( + /await engineViewIsometric\(\{[\s\S]*?await engineStreamZoomToFit\(\{\s*engineCommandManager,\s*padding,/ + ); + expect(vendorScript).toContain( + "Refit against every generated entity so newly" + ); + }); }); From 240352362ae016d6d7a20d8174ee55159b1d6053 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 27 Jul 2026 19:33:41 +0800 Subject: [PATCH 15/83] feat: upgrade embedded HiCAD to latest official source --- scripts/vendor-zoo-modeling-app.mjs | 4 ++-- tests/routes/hicad-embedded-performance.test.ts | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/vendor-zoo-modeling-app.mjs b/scripts/vendor-zoo-modeling-app.mjs index bc596e1f6..ebd939eb8 100644 --- a/scripts/vendor-zoo-modeling-app.mjs +++ b/scripts/vendor-zoo-modeling-app.mjs @@ -18,7 +18,7 @@ import { cacheBustZooModelingAppAssets } from "./cache-bust-zoo-modeling-app-ass const repoUrl = "https://github.com/KittyCAD/modeling-app.git"; const repoRef = process.env.ZOO_MODELING_APP_REF || - "6f87f4e66c07c67f09d8f7db284496a3ae3332e7"; + "f2b345dbb56a851e10d8a742732ab73ee139f025"; const repoTarballUrls = [ `https://ghfast.top/https://github.com/KittyCAD/modeling-app/archive/${repoRef}.tar.gz`, `https://github.com/KittyCAD/modeling-app/archive/${repoRef}.tar.gz`, @@ -167,7 +167,7 @@ function patchOfficialSource() { ` ws.addEventListener('open', () => {\n if (token !== '${proxyToken}') {\n ws.send(\n JSON.stringify({\n type: 'headers',\n headers: {\n Authorization: 'Bearer ' + token,\n },\n })\n )\n }\n resolve(ws)\n })` ); replaceSource( - "src/network/websocketConnection.ts", + "src/lib/engineConnection/websocketConnection.ts", " if (token) {", ` if (token && token !== '${proxyToken}') {` ); diff --git a/tests/routes/hicad-embedded-performance.test.ts b/tests/routes/hicad-embedded-performance.test.ts index 32f56b122..184850ad8 100644 --- a/tests/routes/hicad-embedded-performance.test.ts +++ b/tests/routes/hicad-embedded-performance.test.ts @@ -3,8 +3,19 @@ import fs from "node:fs"; import path from "node:path"; const projectRoot = process.cwd(); +const latestReviewedOfficialCommit = + "f2b345dbb56a851e10d8a742732ab73ee139f025"; test.describe("HiCAD embedded startup", () => { + test("pins the latest reviewed official Zoo Design Studio source", () => { + const vendorScript = fs.readFileSync( + path.join(projectRoot, "scripts/vendor-zoo-modeling-app.mjs"), + "utf8" + ); + + expect(vendorScript).toContain(latestReviewedOfficialCommit); + }); + test("suppresses the upstream onboarding chooser in embedded mode", () => { const vendorScript = fs.readFileSync( path.join(projectRoot, "scripts/vendor-zoo-modeling-app.mjs"), From 08d2503e11fd5a46eef96f8f48dd857113092781 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 27 Jul 2026 19:41:51 +0800 Subject: [PATCH 16/83] fix: use site favicon in HiCAD --- scripts/vendor-zoo-modeling-app.mjs | 28 +++++++++++++++++++ .../routes/hicad-embedded-performance.test.ts | 20 +++++++++++++ 2 files changed, 48 insertions(+) diff --git a/scripts/vendor-zoo-modeling-app.mjs b/scripts/vendor-zoo-modeling-app.mjs index ebd939eb8..56cc5a91d 100644 --- a/scripts/vendor-zoo-modeling-app.mjs +++ b/scripts/vendor-zoo-modeling-app.mjs @@ -30,6 +30,7 @@ const sourceRefFile = path.join(sourceDir, ".hicad-source-ref"); const outputDir = path.join(root, "public/vendor/zoo-modeling-app"); const customizationDir = path.join(root, "vendor/zoo-modeling-app"); const proxyToken = "hicad-session"; +const brandedFaviconUrl = "/api/favicon?surface=hicad"; function run(command, args, options = {}) { const result = spawnSync(command, args, { @@ -112,6 +113,22 @@ function replaceSource(relativePath, before, after) { writeFileSync(filePath, source.replace(before, after)); } +function patchOfficialManifest() { + const manifestPath = path.join(sourceDir, "public/manifest.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + + manifest.short_name = "HiCAD"; + manifest.name = "HiCAD"; + manifest.icons = [ + { + src: brandedFaviconUrl, + sizes: "any", + }, + ]; + + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); +} + function patchOfficialSource() { copyFileSync( path.join(customizationDir, "Logo.tsx"), @@ -130,6 +147,17 @@ function patchOfficialSource() { ' ', ' ' ); + replaceSource( + "index.html", + ' ', + ` ` + ); + replaceSource( + "index.html", + ' ', + ` ` + ); + patchOfficialManifest(); replaceSource( "index.html", ' \n', diff --git a/tests/routes/hicad-embedded-performance.test.ts b/tests/routes/hicad-embedded-performance.test.ts index 184850ad8..3f6ac4df0 100644 --- a/tests/routes/hicad-embedded-performance.test.ts +++ b/tests/routes/hicad-embedded-performance.test.ts @@ -16,6 +16,26 @@ test.describe("HiCAD embedded startup", () => { expect(vendorScript).toContain(latestReviewedOfficialCommit); }); + test("uses the website favicon across embedded browser surfaces", () => { + const vendorScript = fs.readFileSync( + path.join(projectRoot, "scripts/vendor-zoo-modeling-app.mjs"), + "utf8" + ); + + expect(vendorScript).toContain( + 'const brandedFaviconUrl = "/api/favicon?surface=hicad"' + ); + expect(vendorScript).toContain( + '` `' + ); + expect(vendorScript).toContain( + '` `' + ); + expect(vendorScript).toMatch( + /manifest\.icons = \[\s*\{\s*src: brandedFaviconUrl,\s*sizes: "any",/ + ); + }); + test("suppresses the upstream onboarding chooser in embedded mode", () => { const vendorScript = fs.readFileSync( path.join(projectRoot, "scripts/vendor-zoo-modeling-app.mjs"), From 7435e7b0f0a4d3f0d1f95183ba5a5ee6d74c5db1 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 27 Jul 2026 19:55:14 +0800 Subject: [PATCH 17/83] feat: add HiCAD manufacturing drawings --- playwright.helpers.config.ts | 1 + scripts/vendor-zoo-modeling-app.mjs | 18 + .../hicad-manufacturing-extension.test.ts | 138 ++++ .../HicadManufacturingPanel.tsx | 593 ++++++++++++++++++ vendor/zoo-modeling-app/README.md | 31 + vendor/zoo-modeling-app/manufacturing.ts | 401 ++++++++++++ 6 files changed, 1182 insertions(+) create mode 100644 tests/routes/hicad-manufacturing-extension.test.ts create mode 100644 vendor/zoo-modeling-app/HicadManufacturingPanel.tsx create mode 100644 vendor/zoo-modeling-app/README.md create mode 100644 vendor/zoo-modeling-app/manufacturing.ts diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index f30d2518d..b0d63cfa4 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -15,6 +15,7 @@ export default defineConfig({ /routes\/model-classification\.test\.ts/, /routes\/clawd-request-lifecycle\.test\.ts/, /routes\/hicad-embedded-performance\.test\.ts/, + /routes\/hicad-manufacturing-extension\.test\.ts/, ], fullyParallel: true, forbidOnly: !!process.env.CI, diff --git a/scripts/vendor-zoo-modeling-app.mjs b/scripts/vendor-zoo-modeling-app.mjs index 56cc5a91d..3389f0eb2 100644 --- a/scripts/vendor-zoo-modeling-app.mjs +++ b/scripts/vendor-zoo-modeling-app.mjs @@ -134,6 +134,14 @@ function patchOfficialSource() { path.join(customizationDir, "Logo.tsx"), path.join(sourceDir, "src/components/Logo.tsx") ); + copyFileSync( + path.join(customizationDir, "manufacturing.ts"), + path.join(sourceDir, "src/components/manufacturing.ts") + ); + copyFileSync( + path.join(customizationDir, "HicadManufacturingPanel.tsx"), + path.join(sourceDir, "src/components/HicadManufacturingPanel.tsx") + ); copyFileSync( path.join(customizationDir, "localization.ts"), path.join(sourceDir, "src/lib/hicadLocalization.ts") @@ -189,6 +197,16 @@ function patchOfficialSource() { "import { isDesktop } from '@src/lib/isDesktop'\n", "" ); + replaceSource( + "src/components/OpenedProject.tsx", + "import { AppHeader } from '@src/components/AppHeader'\n", + "import { AppHeader } from '@src/components/AppHeader'\nimport { HicadManufacturingPanel } from '@src/components/HicadManufacturingPanel'\n" + ); + replaceSource( + "src/components/OpenedProject.tsx", + " \n {\n ws.send(\n JSON.stringify({\n type: 'headers',\n headers: {\n Authorization: 'Bearer ' + token,\n },\n })\n )\n resolve(ws)\n })`, diff --git a/tests/routes/hicad-manufacturing-extension.test.ts b/tests/routes/hicad-manufacturing-extension.test.ts new file mode 100644 index 000000000..f86ab885d --- /dev/null +++ b/tests/routes/hicad-manufacturing-extension.test.ts @@ -0,0 +1,138 @@ +import { expect, test } from '@playwright/test' +import fs from 'node:fs' +import path from 'node:path' +import { + buildExplodeOffsets, + createBomCsv, + createEngineeringSheetHtml, + groupBodiesIntoBom, + type BodyMeasurement, +} from '../../vendor/zoo-modeling-app/manufacturing' + +const bodies: BodyMeasurement[] = [ + { + id: 'left-leg', + name: 'leg001', + center: { x: -50, y: 0, z: 20 }, + dimensions: { x: 10, y: 10, z: 80 }, + volume: 8_000, + surfaceArea: 3_400, + }, + { + id: 'right-leg', + name: 'leg002', + center: { x: 50, y: 0, z: 20 }, + dimensions: { x: 10, y: 10, z: 80 }, + volume: 8_000, + surfaceArea: 3_400, + }, + { + id: 'top', + name: 'tableTop', + center: { x: 0, y: 0, z: 65 }, + dimensions: { x: 120, y: 60, z: 12 }, + volume: 86_400, + surfaceArea: 18_720, + }, +] + +test.describe('HiCAD manufacturing extension', () => { + test('keeps the extension isolated behind one official registration point', () => { + const projectRoot = process.cwd() + const vendorScript = fs.readFileSync( + path.join(projectRoot, 'scripts/vendor-zoo-modeling-app.mjs'), + 'utf8' + ) + + expect(vendorScript).toContain( + 'vendor/zoo-modeling-app/HicadManufacturingPanel.tsx' + .split('/') + .at(-1) + ) + expect(vendorScript).toContain( + "import { HicadManufacturingPanel } from '@src/components/HicadManufacturingPanel'" + ) + expect(vendorScript).toContain( + "" + ) + }) + + test('creates non-zero reversible offsets for every independent body', () => { + const offsets = buildExplodeOffsets(bodies) + + expect(offsets).toHaveLength(3) + expect(new Set(offsets.map((offset) => offset.id)).size).toBe(3) + expect( + offsets.every( + ({ offset }) => Math.hypot(offset.x, offset.y, offset.z) > 0 + ) + ).toBe(true) + expect(buildExplodeOffsets([bodies[0]])).toEqual([]) + }) + + test('groups matching parts and exports a UTF-8 BOM', () => { + const rows = groupBodiesIntoBom(bodies) + const legRow = rows.find((row) => row.name === 'leg') + + expect(rows).toHaveLength(2) + expect(legRow?.quantity).toBe(2) + expect(legRow?.bodyIds).toEqual(['left-leg', 'right-leg']) + + const csv = createBomCsv('工作台', rows) + expect(csv.startsWith('\uFEFF')).toBe(true) + expect(csv).toContain('"工作台"') + expect(csv).toContain('"生产工艺"') + }) + + test('builds a printable dimensioned engineering sheet safely', () => { + const rows = groupBodiesIntoBom(bodies) + const html = createEngineeringSheetHtml({ + projectName: '<工作台>', + generatedAt: '2026-07-27 20:00', + overallDimensions: { x: 120, y: 60, z: 92 }, + snapshotDataUrl: 'data:image/png;base64,abc', + bomRows: rows, + }) + + expect(html).toContain('<工作台>') + expect(html).not.toContain('<工作台>') + expect(html).toContain('正视图') + expect(html).toContain('俯视图') + expect(html).toContain('右视图') + expect(html).toContain('120 mm') + expect(html).toContain('配件明细表（BOM）') + expect(html).toContain('@page { size: A3 landscape;') + }) + + test('renders the engineering sheet without horizontal clipping', async ({ + page, + }, testInfo) => { + const rows = groupBodiesIntoBom(bodies) + const html = createEngineeringSheetHtml({ + projectName: '工作台生产图', + generatedAt: '2026-07-27 20:00', + overallDimensions: { x: 120, y: 60, z: 92 }, + bomRows: rows, + notes: + '材料、表面处理、公差、连接方式和检验标准须在投产前由工程人员复核。', + }) + + await page.setViewportSize({ width: 1600, height: 900 }) + await page.setContent(html) + + const layout = await page.evaluate(() => ({ + scrollWidth: document.documentElement.scrollWidth, + clientWidth: document.documentElement.clientWidth, + tableCount: document.querySelectorAll('table').length, + dimensionViewCount: document.querySelectorAll('.dimension-view').length, + })) + + expect(layout.scrollWidth).toBeLessThanOrEqual(layout.clientWidth) + expect(layout.tableCount).toBe(1) + expect(layout.dimensionViewCount).toBe(3) + await page.screenshot({ + path: testInfo.outputPath('engineering-sheet.png'), + fullPage: true, + }) + }) +}) diff --git a/vendor/zoo-modeling-app/HicadManufacturingPanel.tsx b/vendor/zoo-modeling-app/HicadManufacturingPanel.tsx new file mode 100644 index 000000000..84bc0b044 --- /dev/null +++ b/vendor/zoo-modeling-app/HicadManufacturingPanel.tsx @@ -0,0 +1,593 @@ +import type { BoundingBox, ModelingCmd } from '@kittycad/lib' +import { useSignals } from '@preact/signals-react/runtime' +import { getBodiesFromArtifactGraph } from '@src/lang/std/artifactGraph' +import { useSingletons } from '@src/lib/boot' +import { isModelingResponse } from '@src/lib/kcSdkGuards' +import { engineStreamZoomToFit, engineViewIsometric } from '@src/lib/utils' +import { uuidv4 } from '@src/lib/utils' +import { useMemo, useRef, useState } from 'react' +import toast from 'react-hot-toast' +import { + buildExplodeOffsets, + createBomCsv, + createEngineeringSheetHtml, + getOverallBounds, + groupBodiesIntoBom, + type BodyMeasurement, + type BomRow, + type ExplodeOffset, +} from './manufacturing' + +type ModelingResponse = { + type: string + data?: unknown +} + +type BomOverrides = Record< + string, + { + material?: string + process?: string + } +> + +function downloadText(filename: string, text: string, type: string) { + const blob = new Blob([text], { type }) + const url = URL.createObjectURL(blob) + const anchor = document.createElement('a') + anchor.href = url + anchor.download = filename + anchor.click() + URL.revokeObjectURL(url) +} + +function collectArtifactIds( + value: unknown, + ids: Set, + seen = new WeakSet(), + depth = 0 +) { + if (depth > 8 || value === null || typeof value !== 'object') return + if (seen.has(value)) return + seen.add(value) + + if ('artifactId' in value && typeof value.artifactId === 'string') { + ids.add(value.artifactId) + } + if ('artifact_id' in value && typeof value.artifact_id === 'string') { + ids.add(value.artifact_id) + } + if ('id' in value && typeof value.id === 'string') { + ids.add(value.id) + } + + for (const nested of Object.values(value)) { + collectArtifactIds(nested, ids, seen, depth + 1) + } +} + +function resolveBodyName( + bodyId: string, + variables: Record, + fallbackIndex: number +) { + for (const [name, value] of Object.entries(variables)) { + const ids = new Set() + collectArtifactIds(value, ids) + if (ids.has(bodyId)) return name + } + return `零件 ${String(fallbackIndex + 1).padStart(2, '0')}` +} + +function modelingResponseData(response: unknown): ModelingResponse { + if (!isModelingResponse(response)) { + throw new Error('HiCAD 引擎未返回有效结果') + } + return response.resp.data.modeling_response as ModelingResponse +} + +function numberValue(data: unknown, key: string) { + if (typeof data !== 'object' || data === null) return undefined + const value = (data as Record)[key] + return typeof value === 'number' ? value : undefined +} + +function formatNumber(value: number | undefined) { + if (value === undefined) return '-' + return Math.round(value * 100) / 100 +} + +function dataUrlFromSnapshot(contents: string) { + return contents.startsWith('data:') + ? contents + : `data:image/png;base64,${contents}` +} + +export function HicadManufacturingPanel({ + projectName, +}: { + projectName: string +}) { + useSignals() + const { kclManager } = useSingletons() + const [open, setOpen] = useState(false) + const [busy, setBusy] = useState(false) + const [bodies, setBodies] = useState([]) + const [snapshot, setSnapshot] = useState() + const [exploded, setExploded] = useState(false) + const [overrides, setOverrides] = useState({}) + const [notes, setNotes] = useState( + '材料、表面处理、公差、连接方式和检验标准须在投产前由工程人员复核。' + ) + const explodeOffsetsRef = useRef([]) + const resolvedProjectName = projectName || 'HiCAD 模型' + + const sendModelingCommand = async (cmd: ModelingCmd) => { + const response = await kclManager.engineCommandManager.sendSceneCommand({ + type: 'modeling_cmd_req', + cmd_id: uuidv4(), + cmd, + }) + return modelingResponseData(response) + } + + const captureSnapshot = async () => { + const response = await sendModelingCommand({ + type: 'take_snapshot', + format: 'png', + }) + if ( + response.type !== 'take_snapshot' || + typeof response.data !== 'object' || + response.data === null || + !('contents' in response.data) || + typeof response.data.contents !== 'string' + ) { + return undefined + } + const dataUrl = dataUrlFromSnapshot(response.data.contents) + setSnapshot(dataUrl) + return dataUrl + } + + const resetExplodeTransforms = async () => { + const offsets = explodeOffsetsRef.current + if (offsets.length === 0) return + + await Promise.allSettled( + offsets.map(({ id, offset }) => + sendModelingCommand({ + type: 'set_object_transform', + object_id: id, + transforms: [ + { + translate: { + property: { + x: -offset.x, + y: -offset.y, + z: -offset.z, + }, + set: false, + origin: { type: 'global' }, + }, + }, + ], + }) + ) + ) + explodeOffsetsRef.current = [] + setExploded(false) + } + + const analyzeModel = async (manageBusy = true) => { + if (manageBusy) setBusy(true) + try { + if (explodeOffsetsRef.current.length > 0) { + await resetExplodeTransforms() + } + + const bodyEntries = [...getBodiesFromArtifactGraph(kclManager.artifactGraph)] + if (bodyEntries.length === 0) { + throw new Error('当前模型还没有可识别的实体') + } + + const variables = kclManager.variablesSignal.value as Record< + string, + unknown + > + const measured = await Promise.all( + bodyEntries.map(async ([bodyId], index): Promise => { + const boundingResponse = await sendModelingCommand({ + type: 'bounding_box', + entity_ids: [bodyId], + output_unit: 'mm', + }) + if ( + boundingResponse.type !== 'bounding_box' || + typeof boundingResponse.data !== 'object' || + boundingResponse.data === null + ) { + throw new Error(`无法读取零件 ${index + 1} 的尺寸`) + } + + const boundingBox = boundingResponse.data as BoundingBox + const [volumeResponse, areaResponse] = await Promise.allSettled([ + sendModelingCommand({ + type: 'volume', + entity_ids: [bodyId], + output_unit: 'mm3', + }), + sendModelingCommand({ + type: 'surface_area', + entity_ids: [bodyId], + output_unit: 'mm2', + }), + ]) + + return { + id: bodyId, + name: resolveBodyName(bodyId, variables, index), + center: boundingBox.center, + dimensions: boundingBox.dimensions, + volume: + volumeResponse.status === 'fulfilled' + ? numberValue(volumeResponse.value.data, 'volume') + : undefined, + surfaceArea: + areaResponse.status === 'fulfilled' + ? numberValue(areaResponse.value.data, 'surface_area') + : undefined, + } + }) + ) + + setBodies(measured) + await engineViewIsometric({ + engineCommandManager: kclManager.engineCommandManager, + padding: 0.22, + }) + await engineStreamZoomToFit({ + engineCommandManager: kclManager.engineCommandManager, + padding: 0.24, + }) + await new Promise((resolve) => setTimeout(resolve, 180)) + await captureSnapshot() + return measured + } catch (error) { + const message = + error instanceof Error ? error.message : '工程数据分析失败' + toast.error(message) + return [] + } finally { + if (manageBusy) setBusy(false) + } + } + + const handleOpen = () => { + setOpen(true) + if (bodies.length === 0) { + void analyzeModel() + } + } + + const toggleExplodedView = async () => { + if (busy) return + setBusy(true) + try { + if (exploded) { + await resetExplodeTransforms() + } else { + const measured = + bodies.length > 0 ? bodies : await analyzeModel(false) + if (measured.length < 2) { + throw new Error( + '爆炸图需要至少两个独立实体；当前模型可能已经布尔合并为一个整体' + ) + } + const offsets = buildExplodeOffsets(measured) + const transformResults = await Promise.allSettled( + offsets.map(({ id, offset }) => + sendModelingCommand({ + type: 'set_object_transform', + object_id: id, + transforms: [ + { + translate: { + property: offset, + set: false, + origin: { type: 'global' }, + }, + }, + ], + }) + ) + ) + const successfulOffsets = offsets.filter( + (_, index) => transformResults[index]?.status === 'fulfilled' + ) + explodeOffsetsRef.current = successfulOffsets + const failedCount = transformResults.length - successfulOffsets.length + if (failedCount > 0) { + await resetExplodeTransforms() + throw new Error( + `${failedCount} 个实体无法移动，已自动复原装配，请检查模型实体` + ) + } + explodeOffsetsRef.current = offsets + setExploded(true) + } + + await engineViewIsometric({ + engineCommandManager: kclManager.engineCommandManager, + padding: 0.18, + }) + await engineStreamZoomToFit({ + engineCommandManager: kclManager.engineCommandManager, + padding: 0.18, + }) + await new Promise((resolve) => setTimeout(resolve, 240)) + await captureSnapshot() + } catch (error) { + toast.error(error instanceof Error ? error.message : '爆炸图生成失败') + } finally { + setBusy(false) + } + } + + const bomRows = useMemo( + () => + groupBodiesIntoBom(bodies).map((row) => ({ + ...row, + ...overrides[row.key], + })), + [bodies, overrides] + ) + const overall = useMemo(() => getOverallBounds(bodies), [bodies]) + + const updateRow = ( + row: BomRow, + field: 'material' | 'process', + value: string + ) => { + setOverrides((current) => ({ + ...current, + [row.key]: { + ...current[row.key], + [field]: value, + }, + })) + } + + const downloadBom = () => { + if (bomRows.length === 0) { + toast.error('请先分析模型') + return + } + downloadText( + `${resolvedProjectName}-BOM.csv`, + createBomCsv(resolvedProjectName, bomRows), + 'text/csv;charset=utf-8' + ) + } + + const printEngineeringSheet = () => { + if (bomRows.length === 0) { + toast.error('请先分析模型') + return + } + const reportWindow = window.open('', '_blank') + if (!reportWindow) { + toast.error('浏览器阻止了工艺图窗口，请允许本站打开新窗口') + return + } + reportWindow.opener = null + reportWindow.document.open() + reportWindow.document.write( + createEngineeringSheetHtml({ + projectName: resolvedProjectName, + generatedAt: new Date().toLocaleString('zh-CN'), + overallDimensions: overall.dimensions, + snapshotDataUrl: snapshot, + bomRows, + notes, + }) + ) + reportWindow.document.close() + } + + const handleClose = async () => { + if (busy) return + setBusy(true) + try { + if (explodeOffsetsRef.current.length > 0) { + await resetExplodeTransforms() + await engineStreamZoomToFit({ + engineCommandManager: kclManager.engineCommandManager, + padding: 0.24, + }) + } + setOpen(false) + } finally { + setBusy(false) + } + } + + return ( + <> + + + {open && ( +
+
+
+
+

工程出图

+

+ 真实实体爆炸、尺寸、BOM 与可打印生产工艺图 +

+
+
+ + +
+
+ +
+ + + + + {bodies.length} 个独立实体 · {bomRows.length} 类零件 · 总尺寸{' '} + {formatNumber(overall.dimensions.x)} ×{' '} + {formatNumber(overall.dimensions.y)} ×{' '} + {formatNumber(overall.dimensions.z)} mm + +
+ +
+
+

装配 / 爆炸预览

+
+ {snapshot ? ( + {exploded + ) : ( + + {busy ? '正在读取工程数据…' : '暂无预览'} + + )} +
+

生产说明

+