From 75cecdfcadbbb7d3b6aabfa00ca63fab15bc89e1 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 22 Jul 2026 10:25:32 +0800 Subject: [PATCH] fix: validate AtlasCloud frontend model requests --- app/(chat)/api/files/upload/route.ts | 20 +- components/canvas/CanvasChatPanel.tsx | 206 +++++++++++++- .../canvas/chat-panel/canvas-chat-session.ts | 4 + lib/ai/atlascloud-models.ts | 49 ++++ lib/ai/atlascloud-request.ts | 209 ++++++++++++++ lib/ai/providers/atlascloud.ts | 268 ++++-------------- scripts/sync-atlascloud-models.ts | 92 ++++++ scripts/verify-atlascloud-frontend.ts | 204 +++++++++++++ tests/routes/model-classification.test.ts | 68 +++++ 9 files changed, 908 insertions(+), 212 deletions(-) create mode 100644 lib/ai/atlascloud-request.ts create mode 100644 scripts/sync-atlascloud-models.ts create mode 100644 scripts/verify-atlascloud-frontend.ts diff --git a/app/(chat)/api/files/upload/route.ts b/app/(chat)/api/files/upload/route.ts index 6c11efe0b..0a5297cb1 100644 --- a/app/(chat)/api/files/upload/route.ts +++ b/app/(chat)/api/files/upload/route.ts @@ -18,6 +18,11 @@ const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([ "image/heic", "image/heif", ]); +const ALLOWED_ATLASCLOUD_VIDEO_TYPES = new Set([ + "video/mp4", + "video/quicktime", + "video/webm", +]); const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]); @@ -197,6 +202,9 @@ export async function POST(request: Request) { let filename = originalFilename; let contentType = originalContentType; const uploadPurpose = String(formData.get("purpose") || ""); + const isAtlasCloudReferenceVideo = + uploadPurpose === "canvas-video-process" && + ALLOWED_ATLASCLOUD_VIDEO_TYPES.has(contentType); const clientUploadId = String(formData.get("clientUploadId") || "").trim() || null; const isSvgUpload = contentType === "image/svg+xml"; @@ -227,7 +235,10 @@ export async function POST(request: Request) { return NextResponse.json({ error: errorMessage }, { status: 400 }); } - if (!ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType)) { + if ( + !ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) && + !isAtlasCloudReferenceVideo + ) { const errorMessage = createUnsupportedImageTypeMessage(contentType, filename); logger.warn("[Upload API] Unsupported upload content type", { contentType, @@ -361,18 +372,19 @@ export async function POST(request: Request) { try { const asset = await upsertMediaAsset({ userId: session.user.id, - mediaType: "image", + mediaType: isAtlasCloudReferenceVideo ? "video" : "image", sourceType: "upload", status: "ready", originalUrl: data.url, - previewUrl: data.url, - thumbnailUrl: data.url, + previewUrl: isAtlasCloudReferenceVideo ? null : data.url, + thumbnailUrl: isAtlasCloudReferenceVideo ? null : data.url, mimeType: data.contentType || contentType || null, displayName: filename, sourceTable: "upload", metadata: { pathname: data.pathname, size: optimizedBuffer.length, + purpose: uploadPurpose || null, clientUploadId, convertedFrom, }, diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 5a8000c56..8c35ab4a2 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -2197,6 +2197,39 @@ function isLikelyVideoAttachment(url: string, contentType: string) { ); } +function collectMessageVideoAttachments(message: ChatMessage) { + const attachments: PickedObjectAttachment[] = []; + const seenUrls = new Set(); + const append = (value: any, fallbackName: string) => { + const url = String(value?.url || value?.data || "").trim(); + if (!isValidChatHttpUrl(url) || seenUrls.has(url)) return; + const contentType = normalizeChatMediaType( + value?.contentType || value?.mediaType || value?.mimeType + ); + if (!isLikelyVideoAttachment(url, contentType)) return; + const attachment = sanitizeChatAttachment({ + url, + name: normalizeChatAttachmentName(value?.name || fallbackName), + contentType, + assetId: + typeof value?.assetId === "string" && value.assetId.trim() + ? value.assetId.trim() + : undefined, + } as PickedObjectAttachment); + if (!attachment) return; + seenUrls.add(url); + attachments.push(attachment); + }; + + (message.parts || []).forEach((part: any, index: number) => { + if (part?.type === "file") append(part, `参考视频-${index + 1}.mp4`); + }); + ((message as any).attachments || []).forEach((attachment: any, index: number) => { + append(attachment, `参考视频-${index + 1}.mp4`); + }); + return attachments; +} + function buildCopiedMessageAttachment( value: any, fallbackName: string @@ -4206,6 +4239,7 @@ export function CanvasChatPanel({ videoToolCapability?: VideoToolCapabilityContract | null; imageGenConfig?: { maxImages?: number; + maxVideoReferences?: number; supportsSize?: boolean; supports1K?: boolean; supports2K?: boolean; @@ -4234,6 +4268,7 @@ export function CanvasChatPanel({ isMultimodal?: boolean; imageGenConfig?: { maxImages?: number; + maxVideoReferences?: number; supportsSize?: boolean; supports1K?: boolean; supports2K?: boolean; @@ -4731,6 +4766,25 @@ export function CanvasChatPanel({ [visualModels] ); const currentModel = allModels.find((model) => model.id === selectedModelId); + const atlasVideoReferenceLimit = + currentModel?.providerType === "atlascloud" + ? Math.max( + 0, + Math.min( + 4, + Number( + currentModel.capabilities?.maxVideoReferences || + currentModel.imageGenConfig?.maxVideoReferences || + 0 + ) + ) + ) + : 0; + const supportsAtlasVideoReference = atlasVideoReferenceLimit > 0; + const requiresAtlasVideoReference = Boolean( + supportsAtlasVideoReference && + currentModel?.capabilities?.requiresVideoReference === true + ); const isCanvasVideoModel = useCallback( (model?: (typeof allModels)[number] | null) => Boolean( @@ -11295,6 +11349,20 @@ export function CanvasChatPanel({ let effectiveAttachments = attachments .map((attachment) => sanitizeChatAttachment(attachment)) .filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment)); + if (supportsAtlasVideoReference) { + let keptVideoCount = 0; + effectiveAttachments = effectiveAttachments.filter((attachment) => { + if (!isLikelyVideoAttachment(attachment.url, attachment.contentType)) { + return true; + } + keptVideoCount += 1; + return keptVideoCount <= atlasVideoReferenceLimit; + }); + if (requiresAtlasVideoReference && keptVideoCount === 0) { + toast.error("当前模型需要先上传 1 个参考视频"); + return; + } + } if (hasAgentUploadSlots) { if (effectiveAttachments.length > agentRequiredUploadCount) { effectiveAttachments = effectiveAttachments.slice(0, agentRequiredUploadCount); @@ -11918,6 +11986,57 @@ export function CanvasChatPanel({ } }, [clearComposerUploadProgress, updateComposerUploadProgress]); + const uploadVideoFileForComposerAttachment = useCallback(async ( + file: File, + progressOptions?: CanvasUploadProgressOptions + ): Promise => { + if (!file.type.startsWith("video/")) { + toast.error("请选择视频文件"); + return null; + } + if (file.size > 15 * 1024 * 1024) { + toast.error("参考视频大小不能超过 15MB"); + return null; + } + + try { + const formData = new FormData(); + formData.append("file", file); + formData.append("purpose", "canvas-video-process"); + if (progressOptions) { + updateComposerUploadProgress(progressOptions); + } + const response = await fetchUploadWithRetry( + formData, + progressOptions + ? { + onProgress: (progress) => { + updateComposerUploadProgress(progressOptions, progress); + }, + } + : undefined + ); + if (!response.ok) { + throw new Error("上传失败"); + } + + const data = await response.json(); + clearComposerUploadProgress(250); + return { + imageUrl: String(data.url || ""), + name: file.name || `参考视频-${Date.now()}.mp4`, + contentType: file.type || "video/mp4", + assetId: + typeof data.assetId === "string" && data.assetId.trim() + ? data.assetId.trim() + : undefined, + }; + } catch (_error) { + toast.error("参考视频上传失败"); + return null; + } + }, [clearComposerUploadProgress, updateComposerUploadProgress]); + const handlePastedImages = useCallback(async (files: File[]) => { if (files.length === 0) return; @@ -12250,6 +12369,58 @@ export function CanvasChatPanel({ e.target.value = ""; return; } + + if (supportsAtlasVideoReference) { + const selectedVideoFiles = selectedFiles.filter((file) => + file.type.startsWith("video/") + ); + const existingVideoCount = attachments.filter((attachment) => + isLikelyVideoAttachment(attachment.url, attachment.contentType) + ).length; + const availableVideoSlots = Math.max( + 0, + atlasVideoReferenceLimit - existingVideoCount + ); + const acceptedVideoFiles = selectedVideoFiles.slice(0, availableVideoSlots); + + if (selectedVideoFiles.length > acceptedVideoFiles.length) { + toast.info(`当前模型最多支持 ${atlasVideoReferenceLimit} 个参考视频`); + } + if (acceptedVideoFiles.length > 0) { + let uploadedCount = 0; + try { + for (const [index, file] of acceptedVideoFiles.entries()) { + const uploaded = await uploadVideoFileForComposerAttachment(file, { + title: "正在上传参考视频", + fileName: file.name, + fileIndex: index, + fileTotal: acceptedVideoFiles.length, + target: "composer-reference", + }); + if (!uploaded?.imageUrl) continue; + addUploadedImageAttachmentToComposer(uploaded); + uploadedCount += 1; + } + } finally { + clearComposerUploadProgress(450); + } + if (uploadedCount > 0) { + toast.success( + uploadedCount === 1 + ? "参考视频已添加到对话" + : `已添加 ${uploadedCount} 个参考视频` + ); + } + } + + selectedFiles = selectedFiles.filter( + (file) => !file.type.startsWith("video/") + ); + if (selectedFiles.length === 0) { + e.target.value = ""; + return; + } + } let firstImageUrl: string | null = null; // 记录本次上传开始时的节点数量，用于计算新图片位置 @@ -13802,6 +13973,10 @@ export function CanvasChatPanel({ message.role === "user" ? collectMessageImageAttachments(message as any) : []; + const userVideoAttachments = + message.role === "user" + ? collectMessageVideoAttachments(message as any) + : []; const assistantTextParts = message.role === "assistant" ? (message.parts || []).filter((part: any) => part.type === "text" && part.text) : []; @@ -13954,6 +14129,25 @@ export function CanvasChatPanel({ ); }); + userVideoAttachments.forEach((attachment, idx) => { + renderedUserParts.push( + + + ); + }); + message.parts?.forEach((part: any, idx: number) => { if (part.type === 'text' && part.text) { const text = String(part.text) @@ -15619,8 +15813,8 @@ export function CanvasChatPanel({ className="relative flex h-8 w-8 shrink-0 items-center justify-center overflow-hidden rounded-full border border-dashed border-[#d9dde4] bg-[#f3f3f4] text-[#8f97a3] transition-colors hover:border-[#c6ccd6] hover:bg-[#ececee] hover:text-[#20242a] disabled:cursor-not-allowed disabled:opacity-60 dark:border-white/12 dark:bg-white/8 dark:text-zinc-400 dark:hover:border-white/20 dark:hover:bg-white/12 dark:hover:text-white" onClick={() => fileInputRef.current?.click()} disabled={isSubmitting} - title="添加图片" - aria-label="添加图片" + title={supportsAtlasVideoReference ? "添加图片或参考视频" : "添加图片"} + aria-label={supportsAtlasVideoReference ? "添加图片或参考视频" : "添加图片"} > {composerReferenceUploadProgress ? ( @@ -15926,7 +16120,13 @@ export function CanvasChatPanel({ ; imageInputKey?: string; imageInputMode?: "array" | "single"; + videoClipInput?: { + key: string; + maxItems: number; + required: boolean; + itemDefaults: Record; + }; resolutionKey?: string; aspectRatioKey?: string; outputCountKey?: string; aspectRatioValueMap: Record; maxReferenceImages: number; + maxVideoReferences: number; maxImages: number; defaults: Record; enums: Record>; @@ -196,6 +203,39 @@ function getImageInputs(properties: JsonRecord, requiredFields: string[]) { .sort((a, b) => Number(b.required) - Number(a.required)); } +function getVideoClipInput(properties: JsonRecord, requiredFields: string[]) { + const entry = Object.entries(properties).find(([key, property]) => { + const normalizedKey = key.toLowerCase(); + const record = asRecord(property); + const itemProperties = asRecord(asRecord(record.items).properties); + return ( + record.type === "array" && + normalizedKey.includes("video") && + ("url" in itemProperties || normalizedKey.includes("clip")) + ); + }); + if (!entry) return undefined; + + const [key, property] = entry; + const record = asRecord(property); + const itemProperties = asRecord(asRecord(record.items).properties); + const itemDefaults: Record = {}; + for (const [itemKey, itemProperty] of Object.entries(itemProperties)) { + if (itemKey === "url") continue; + const value = asRecord(itemProperty).default; + if (["string", "number", "boolean"].includes(typeof value)) { + itemDefaults[itemKey] = value; + } + } + + return { + key, + maxItems: Math.max(1, Math.min(4, Number(record.maxItems || 1))), + required: requiredFields.includes(key), + itemDefaults, + }; +} + function getEnumValues(property: unknown) { const record = asRecord(property); const values = Array.isArray(record.enum) ? record.enum : []; @@ -396,6 +436,7 @@ export function createAtlasCloudImagePreset( const properties = asRecord(input.properties); const requiredFields = uniqueStrings(Array.isArray(input.required) ? input.required : []); const imageInputs = getImageInputs(properties, requiredFields); + const videoClipInput = getVideoClipInput(properties, requiredFields); const imageInputKey = imageInputs[0]?.key; const resolutionKey = RESOLUTION_KEYS.find((key) => { if (!(key in properties)) return false; @@ -420,6 +461,7 @@ export function createAtlasCloudImagePreset( 14, imageInputs.reduce((total, input) => total + input.maxItems, 0) ); + const maxVideoReferences = videoClipInput?.maxItems || 0; const maxImages = deriveMaxImages(properties, outputCountKey); const categories = uniqueStrings(catalogModel.categories || []).map((value) => value.toLowerCase() @@ -436,6 +478,7 @@ export function createAtlasCloudImagePreset( categories.some((value) => value.includes("image-to-image")) ? ["image-edit", "reference-images"] : []), + ...(videoClipInput ? ["reference-video"] : []), ]; if (mediaKind === "image" && supportedModes.length === 0) { supportedModes.push("text-to-image"); @@ -449,11 +492,13 @@ export function createAtlasCloudImagePreset( imageInputs, imageInputKey, imageInputMode: imageProperty.type === "array" ? "array" : "single", + videoClipInput, resolutionKey, aspectRatioKey, outputCountKey, aspectRatioValueMap: aspectRatioConfig.map, maxReferenceImages, + maxVideoReferences, maxImages, defaults: buildScalarDefaults(properties), enums: buildEnums(properties), @@ -495,7 +540,10 @@ export function createAtlasCloudImagePreset( mediaType: mediaKind, supportedModes, maxReferenceImages, + maxVideoReferences, supportsMultiImageReference: maxReferenceImages > 1, + supportsVideoReference: maxVideoReferences > 0, + requiresVideoReference: videoClipInput?.required === true, provider: "atlascloud", schemaUrl: catalogModel.schema || null, }, @@ -509,6 +557,7 @@ export function createAtlasCloudImagePreset( apiFormat: "atlascloud-unified-image", maxImages, maxReferenceImages, + maxVideoReferences, supportsSize, supports1K: supportedImageSizes.includes("1K"), supports2K: supportedImageSizes.includes("2K"), diff --git a/lib/ai/atlascloud-request.ts b/lib/ai/atlascloud-request.ts new file mode 100644 index 000000000..4607c4853 --- /dev/null +++ b/lib/ai/atlascloud-request.ts @@ -0,0 +1,209 @@ +import type { AtlasCloudRuntimeConfig } from "@/lib/ai/atlascloud-models"; + +export interface AtlasCloudImageParams { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + negativePrompt?: string; + promptExtend?: boolean; + quality?: string; + seed?: number; + watermark?: boolean; +} + +export interface AtlasCloudModelConfig { + maxReferenceImages?: number; + maxVideoReferences?: number; + supportedAspectRatios?: string[]; + supportedImageSizes?: string[]; + atlasCloud?: AtlasCloudRuntimeConfig; +} + +function normalizeAspectRatio( + requested: string | undefined, + supported: string[] | undefined +) { + const values = (supported || []).map(String); + const normalized = String(requested || "auto").trim().toLowerCase(); + const exact = values.find((value) => value.toLowerCase() === normalized); + if (exact) return exact; + return ( + values.find((value) => value === "auto") || + values.find((value) => value === "1:1") || + values[0] || + "1:1" + ); +} + +function normalizeImageSize(requested: string | undefined, supported?: string[]) { + const values = (supported || []).map((value) => String(value).trim().toUpperCase()); + const normalized = String(requested || "2K").trim().toUpperCase(); + if (values.includes(normalized)) return normalized; + return values.includes("2K") ? "2K" : values[0] || normalized; +} + +function matchEnumValue(values: Array | undefined, value: unknown) { + if (!values?.length) return value; + const normalized = String(value || "").trim().toLowerCase(); + return values.find( + (item) => String(item).trim().toLowerCase() === normalized + ); +} + +function parseRatio(value: string) { + const match = value.match(/^(\d+):(\d+)$/); + if (!match) return 1; + return Number(match[1]) / Number(match[2]); +} + +function pickClosestSizeValue( + map: Record>, + imageSize: string, + aspectRatio: string +) { + const tiers = [imageSize, "2K", "1K", "4K"]; + const values = tiers.map((tier) => map[tier]).find(Boolean); + if (!values) return undefined; + if (values[aspectRatio]) return values[aspectRatio]; + const target = parseRatio(aspectRatio); + return Object.entries(values).sort( + ([a], [b]) => + Math.abs(Math.log(target / parseRatio(a))) - + Math.abs(Math.log(target / parseRatio(b))) + )[0]?.[1]; +} + +export function createAtlasCloudRequestBody(params: { + modelId: string; + prompt: string; + referenceImages: string[]; + referenceVideos?: string[]; + imageParams?: AtlasCloudImageParams; + modelConfig: AtlasCloudModelConfig; +}) { + const runtime = params.modelConfig.atlasCloud; + if (!runtime) { + throw new Error("Atlas Cloud 模型运行参数缺失，请重新同步模型"); + } + const body: Record = { ...runtime.defaults }; + if ( + runtime.propertyNames.includes("model") || + runtime.requiredFields.includes("model") + ) { + body.model = params.modelId; + } + if ( + runtime.propertyNames.includes("prompt") || + runtime.requiredFields.includes("prompt") + ) { + body.prompt = params.prompt; + } + + const aspectRatio = normalizeAspectRatio( + params.imageParams?.aspectRatio, + params.modelConfig.supportedAspectRatios + ); + const imageSize = normalizeImageSize( + params.imageParams?.imageSize, + params.modelConfig.supportedImageSizes + ); + + if (runtime.aspectRatioKey) { + body[runtime.aspectRatioKey] = + runtime.aspectRatioValueMap?.[aspectRatio] || + matchEnumValue(runtime.enums[runtime.aspectRatioKey], aspectRatio) || + aspectRatio; + } + if (runtime.resolutionKey) { + body[runtime.resolutionKey] = + matchEnumValue(runtime.enums[runtime.resolutionKey], imageSize.toLowerCase()) || + matchEnumValue(runtime.enums[runtime.resolutionKey], imageSize) || + imageSize.toLowerCase(); + } else { + const sizeValue = pickClosestSizeValue( + runtime.sizeValueMap, + imageSize, + aspectRatio === "auto" ? "1:1" : aspectRatio + ); + if (sizeValue) body.size = sizeValue; + } + + const imageInputs = runtime.imageInputs?.length + ? runtime.imageInputs + : runtime.imageInputKey + ? [ + { + key: runtime.imageInputKey, + mode: runtime.imageInputMode || ("single" as const), + maxItems: Math.max(1, runtime.maxReferenceImages || 1), + required: runtime.requiredFields.includes(runtime.imageInputKey), + }, + ] + : []; + let referenceCursor = 0; + for (const input of imageInputs) { + const values = params.referenceImages.slice( + referenceCursor, + referenceCursor + input.maxItems + ); + if (values.length > 0) { + body[input.key] = input.mode === "single" ? values[0] : values; + referenceCursor += values.length; + } else if (input.required) { + throw new Error( + `该 Atlas Cloud 模型至少需要 ${referenceCursor + 1} 张参考图` + ); + } + } + if (imageInputs.length === 0 && params.referenceImages.length > 0) { + throw new Error("当前 Atlas Cloud 模型不支持参考图，请选择图片编辑模型"); + } + + const referenceVideos = params.referenceVideos || []; + if (runtime.videoClipInput) { + const values = referenceVideos.slice(0, runtime.videoClipInput.maxItems); + if (values.length > 0) { + body[runtime.videoClipInput.key] = values.map((url) => ({ + ...runtime.videoClipInput?.itemDefaults, + url, + })); + } else if (runtime.videoClipInput.required) { + throw new Error("该 Atlas Cloud 模型至少需要 1 个参考视频"); + } + } else if (referenceVideos.length > 0) { + throw new Error("当前 Atlas Cloud 模型不支持参考视频"); + } + + if (runtime.outputCountKey) { + const requestedCount = Math.max(1, params.imageParams?.imageCount || 1); + body[runtime.outputCountKey] = Math.min(runtime.maxImages, requestedCount); + } + const optionalParams: Array<[string, unknown]> = [ + ["negative_prompt", params.imageParams?.negativePrompt], + ["prompt_extend", params.imageParams?.promptExtend], + ["quality", params.imageParams?.quality], + ["seed", params.imageParams?.seed], + ["watermark", params.imageParams?.watermark], + ]; + for (const [key, value] of optionalParams) { + if (value !== undefined && runtime.propertyNames.includes(key)) { + body[key] = matchEnumValue(runtime.enums[key], value) ?? value; + } + } + if (runtime.propertyNames.includes("enable_sync_mode")) { + body.enable_sync_mode = false; + } + if (runtime.propertyNames.includes("enable_base64_output")) { + body.enable_base64_output = false; + } + + const unsupportedRequiredFields = runtime.requiredFields.filter( + (key) => body[key] === undefined + ); + if (unsupportedRequiredFields.length > 0) { + throw new Error( + `当前界面暂不支持该 Atlas Cloud 模型的必填参数：${unsupportedRequiredFields.join(", ")}` + ); + } + return { body, imageSize }; +} diff --git a/lib/ai/providers/atlascloud.ts b/lib/ai/providers/atlascloud.ts index e55f82e09..81a0c79f2 100644 --- a/lib/ai/providers/atlascloud.ts +++ b/lib/ai/providers/atlascloud.ts @@ -1,10 +1,12 @@ import "server-only"; import sharp from "sharp"; +import { ATLASCLOUD_BASE_URL } from "@/lib/ai/atlascloud-models"; import { - ATLASCLOUD_BASE_URL, - type AtlasCloudRuntimeConfig, -} from "@/lib/ai/atlascloud-models"; + createAtlasCloudRequestBody, + type AtlasCloudImageParams, + type AtlasCloudModelConfig, +} from "@/lib/ai/atlascloud-request"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; @@ -13,24 +15,6 @@ interface AtlasCloudConfig { baseURL?: string; } -interface AtlasCloudImageParams { - aspectRatio?: string; - imageSize?: string; - imageCount?: number; - negativePrompt?: string; - promptExtend?: boolean; - quality?: string; - seed?: number; - watermark?: boolean; -} - -interface AtlasCloudModelConfig { - maxReferenceImages?: number; - supportedAspectRatios?: string[]; - supportedImageSizes?: string[]; - atlasCloud?: AtlasCloudRuntimeConfig; -} - interface UploadedImage { thumbnail: string; original: string; @@ -116,18 +100,19 @@ function extractOutputUrls(payload: any) { return urls; } -function extractPromptAndImages( +function extractPromptAndMedia( messages: Array<{ role: string; content?: any; parts?: any[] }> ) { const message = messages.filter((item) => item.role === "user").pop(); if (!message) throw new Error("未找到用户消息"); const texts: string[] = []; const imageUrls: string[] = []; + const videoUrls: string[] = []; const pushText = (value: unknown) => { if (typeof value === "string" && value.trim()) texts.push(value.trim()); }; - const pushImage = (value: unknown) => { + const pushMedia = (value: unknown, mediaType?: unknown) => { const url = String( typeof value === "string" ? value @@ -137,24 +122,31 @@ function extractPromptAndImages( (value as any)?.src || "" ).trim(); - if ( - url && - (/^https?:\/\//i.test(url) || url.startsWith("data:image/")) && - !imageUrls.includes(url) - ) { - imageUrls.push(url); - } + if (!url || !/^(?:https?:\/\/|data:)/i.test(url)) return; + const normalizedMediaType = String(mediaType || "").toLowerCase(); + const isVideo = + normalizedMediaType.startsWith("video/") || + url.startsWith("data:video/") || + /\.(?:mp4|mov|webm|m4v)(?:[?#].*)?$/i.test(url); + const target = isVideo ? videoUrls : imageUrls; + if (!target.includes(url)) target.push(url); }; const visitPart = (part: any) => { if (!part) return; if (typeof part === "string") return pushText(part); if (part.type === "text" || typeof part.text === "string") pushText(part.text); - if (part.type === "image_url") pushImage(part.image_url); - if (part.type === "image") pushImage(part.image || part.url); - if (part.type === "file") pushImage(part.url || part.file || part.data); + if (part.type === "image_url") pushMedia(part.image_url, "image/jpeg"); + if (part.type === "image") pushMedia(part.image || part.url, "image/jpeg"); + if (part.type === "file") { + pushMedia( + part.url || part.file || part.data, + part.mediaType || part.contentType || part.mimeType + ); + } if (part.inlineData?.data) { - pushImage( - `data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}` + pushMedia( + `data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}`, + part.inlineData.mimeType ); } }; @@ -165,31 +157,31 @@ function extractPromptAndImages( const prompt = texts.join("\n").trim(); if (!prompt) throw new Error("请输入图片生成或编辑要求"); - return { prompt, imageUrls }; + return { prompt, imageUrls, videoUrls }; } function parseDataUrl(value: string) { const match = value.match(/^data:([^;,]+);base64,(.+)$/s); - if (!match) throw new Error("Atlas Cloud 参考图 Base64 格式无效"); + if (!match) throw new Error("Atlas Cloud 参考素材 Base64 格式无效"); return { mimeType: match[1] || "image/png", buffer: Buffer.from(match[2] || "", "base64"), }; } -async function loadImageSource(imageUrl: string) { - if (imageUrl.startsWith("data:")) return parseDataUrl(imageUrl); - const response = await fetch(imageUrl, { +async function loadMediaSource(mediaUrl: string) { + if (mediaUrl.startsWith("data:")) return parseDataUrl(mediaUrl); + const response = await fetch(mediaUrl, { cache: "no-store", redirect: "follow", signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS), headers: { - Accept: "image/avif,image/webp,image/png,image/jpeg,image/*,*/*;q=0.8", + Accept: "image/*,video/*,*/*;q=0.8", "User-Agent": "yedir-ai-atlascloud/1.0", }, }); if (!response.ok) { - throw new Error(`Atlas Cloud 参考图下载失败（HTTP ${response.status}）`); + throw new Error(`Atlas Cloud 参考素材下载失败（HTTP ${response.status}）`); } return { mimeType: response.headers.get("content-type")?.split(";")[0] || "image/png", @@ -201,16 +193,19 @@ function detectExtension(mimeType: string) { if (/jpe?g/i.test(mimeType)) return "jpg"; if (/webp/i.test(mimeType)) return "webp"; if (/gif/i.test(mimeType)) return "gif"; + if (/quicktime/i.test(mimeType)) return "mov"; + if (/webm/i.test(mimeType)) return "webm"; + if (/mp4/i.test(mimeType)) return "mp4"; return "png"; } -async function uploadReferenceImage( +async function uploadReferenceMedia( baseURL: string | undefined, apiKey: string, - imageUrl: string + mediaUrl: string ) { - if (/^https:\/\//i.test(imageUrl)) return imageUrl; - const source = await loadImageSource(imageUrl); + if (/^https:\/\//i.test(mediaUrl)) return mediaUrl; + const source = await loadMediaSource(mediaUrl); const form = new FormData(); form.append( "file", @@ -226,171 +221,17 @@ async function uploadReferenceImage( const payload = await parseJsonResponse(response); if (!response.ok) { throw new Error( - `${extractErrorMessage(payload, "Atlas Cloud 参考图上传失败")}（HTTP ${response.status}）` + `${extractErrorMessage(payload, "Atlas Cloud 参考素材上传失败")}（HTTP ${response.status}）` ); } const data = payload?.data && typeof payload.data === "object" ? payload.data : payload; const url = String( data?.url || data?.file_url || data?.fileUrl || data?.media_url || data?.mediaUrl || "" ).trim(); - if (!url) throw new Error("Atlas Cloud 参考图上传成功但未返回文件地址"); + if (!url) throw new Error("Atlas Cloud 参考素材上传成功但未返回文件地址"); return url; } -function normalizeAspectRatio( - requested: string | undefined, - supported: string[] | undefined -) { - const values = (supported || []).map(String); - const normalized = String(requested || "auto").trim().toLowerCase(); - const exact = values.find((value) => value.toLowerCase() === normalized); - if (exact) return exact; - return values.find((value) => value === "auto") || values.find((value) => value === "1:1") || values[0] || "1:1"; -} - -function normalizeImageSize(requested: string | undefined, supported?: string[]) { - const values = (supported || []).map((value) => String(value).trim().toUpperCase()); - const normalized = String(requested || "2K").trim().toUpperCase(); - if (values.includes(normalized)) return normalized; - return values.includes("2K") ? "2K" : values[0] || normalized; -} - -function matchEnumValue(values: Array | undefined, value: unknown) { - if (!values?.length) return value; - const normalized = String(value || "").trim().toLowerCase(); - return values.find((item) => String(item).trim().toLowerCase() === normalized); -} - -function parseRatio(value: string) { - const match = value.match(/^(\d+):(\d+)$/); - if (!match) return 1; - return Number(match[1]) / Number(match[2]); -} - -function pickClosestSizeValue( - map: Record>, - imageSize: string, - aspectRatio: string -) { - const tiers = [imageSize, "2K", "1K", "4K"]; - const values = tiers.map((tier) => map[tier]).find(Boolean); - if (!values) return undefined; - if (values[aspectRatio]) return values[aspectRatio]; - const target = parseRatio(aspectRatio); - return Object.entries(values).sort( - ([a], [b]) => - Math.abs(Math.log(target / parseRatio(a))) - - Math.abs(Math.log(target / parseRatio(b))) - )[0]?.[1]; -} - -function createRequestBody(params: { - modelId: string; - prompt: string; - referenceImages: string[]; - imageParams?: AtlasCloudImageParams; - modelConfig: AtlasCloudModelConfig; -}) { - const runtime = params.modelConfig.atlasCloud; - if (!runtime) throw new Error("Atlas Cloud 模型运行参数缺失，请重新同步模型"); - const body: Record = { - ...runtime.defaults, - }; - if (runtime.propertyNames.includes("model")) body.model = params.modelId; - if (runtime.propertyNames.includes("prompt")) body.prompt = params.prompt; - const aspectRatio = normalizeAspectRatio( - params.imageParams?.aspectRatio, - params.modelConfig.supportedAspectRatios - ); - const imageSize = normalizeImageSize( - params.imageParams?.imageSize, - params.modelConfig.supportedImageSizes - ); - - if (runtime.aspectRatioKey) { - body[runtime.aspectRatioKey] = - runtime.aspectRatioValueMap?.[aspectRatio] || - matchEnumValue(runtime.enums[runtime.aspectRatioKey], aspectRatio) || aspectRatio; - } - if (runtime.resolutionKey) { - const resolutionValue = - matchEnumValue(runtime.enums[runtime.resolutionKey], imageSize.toLowerCase()) || - matchEnumValue(runtime.enums[runtime.resolutionKey], imageSize) || - imageSize.toLowerCase(); - body[runtime.resolutionKey] = resolutionValue; - } else { - const sizeValue = pickClosestSizeValue( - runtime.sizeValueMap, - imageSize, - aspectRatio === "auto" ? "1:1" : aspectRatio - ); - if (sizeValue) body.size = sizeValue; - } - - const imageInputs = runtime.imageInputs?.length - ? runtime.imageInputs - : runtime.imageInputKey - ? [ - { - key: runtime.imageInputKey, - mode: runtime.imageInputMode || "single", - maxItems: Math.max(1, runtime.maxReferenceImages || 1), - required: runtime.requiredFields.includes(runtime.imageInputKey), - }, - ] - : []; - let referenceCursor = 0; - for (const input of imageInputs) { - const values = params.referenceImages.slice( - referenceCursor, - referenceCursor + input.maxItems - ); - if (values.length > 0) { - body[input.key] = input.mode === "single" ? values[0] : values; - referenceCursor += values.length; - } else if (input.required) { - throw new Error( - `该 Atlas Cloud 模型至少需要 ${referenceCursor + 1} 张参考图` - ); - } - } - if (imageInputs.length === 0 && params.referenceImages.length > 0) { - throw new Error("当前 Atlas Cloud 模型不支持参考图，请选择图片编辑模型"); - } - - if (runtime.outputCountKey) { - const requestedCount = Math.max(1, params.imageParams?.imageCount || 1); - body[runtime.outputCountKey] = Math.min(runtime.maxImages, requestedCount); - } - const optionalParams: Array<[string, unknown]> = [ - ["negative_prompt", params.imageParams?.negativePrompt], - ["prompt_extend", params.imageParams?.promptExtend], - ["quality", params.imageParams?.quality], - ["seed", params.imageParams?.seed], - ["watermark", params.imageParams?.watermark], - ]; - for (const [key, value] of optionalParams) { - if (value !== undefined && runtime.propertyNames.includes(key)) { - body[key] = matchEnumValue(runtime.enums[key], value) ?? value; - } - } - if (runtime.propertyNames.includes("enable_sync_mode")) { - body.enable_sync_mode = false; - } - if (runtime.propertyNames.includes("enable_base64_output")) { - body.enable_base64_output = false; - } - const unsupportedRequiredFields = runtime.requiredFields.filter( - (key) => body[key] === undefined - ); - if (unsupportedRequiredFields.length > 0) { - throw new Error( - `当前界面暂不支持该 Atlas Cloud 模型的必填参数：${unsupportedRequiredFields.join(", ")}` - ); - } - return { body, imageSize }; -} - async function submitTask(params: { apiKey: string; baseURL?: string; @@ -540,7 +381,9 @@ export function createAtlasCloud({ apiKey, baseURL }: AtlasCloudConfig) { if (!modelConfig.atlasCloud) { throw new Error("Atlas Cloud 模型能力配置缺失，请在系统设置中重新同步模型"); } - const { prompt, imageUrls } = extractPromptAndImages(options.messages); + const { prompt, imageUrls, videoUrls } = extractPromptAndMedia( + options.messages + ); const maxReferenceImages = Math.max( 0, Math.min(14, modelConfig.maxReferenceImages ?? modelConfig.atlasCloud.maxReferenceImages) @@ -548,12 +391,27 @@ export function createAtlasCloud({ apiKey, baseURL }: AtlasCloudConfig) { const referenceImages = await Promise.all( imageUrls .slice(0, maxReferenceImages) - .map((url) => uploadReferenceImage(baseURL, apiKey, url)) + .map((url) => uploadReferenceMedia(baseURL, apiKey, url)) ); - const { body, imageSize } = createRequestBody({ + const maxVideoReferences = Math.max( + 0, + Math.min( + 4, + modelConfig.maxVideoReferences ?? + modelConfig.atlasCloud.maxVideoReferences ?? + 0 + ) + ); + const referenceVideos = await Promise.all( + videoUrls + .slice(0, maxVideoReferences) + .map((url) => uploadReferenceMedia(baseURL, apiKey, url)) + ); + const { body, imageSize } = createAtlasCloudRequestBody({ modelId: runtimeModelId, prompt, referenceImages, + referenceVideos, imageParams: options.imageParams, modelConfig, }); diff --git a/scripts/sync-atlascloud-models.ts b/scripts/sync-atlascloud-models.ts new file mode 100644 index 000000000..0455116e6 --- /dev/null +++ b/scripts/sync-atlascloud-models.ts @@ -0,0 +1,92 @@ +import { and, eq } from "drizzle-orm"; +import { fetchAtlasCloudImagePresets } from "../lib/ai/atlascloud-models"; +import { db } from "../lib/db"; +import { aiModel, aiProvider } from "../lib/db/schema"; + +async function main() { + const providers = await db + .select() + .from(aiProvider) + .where(and(eq(aiProvider.type, "atlascloud"), eq(aiProvider.isEnabled, true))); + if (providers.length === 0) { + throw new Error("没有找到已启用的 AtlasCloud 服务商"); + } + + const catalog = await fetchAtlasCloudImagePresets(); + if (catalog.errors.length > 0) { + throw new Error( + `AtlasCloud 目录有 ${catalog.errors.length} 个 schema 加载失败：${catalog.errors + .map((item) => item.modelId) + .join(", ")}` + ); + } + + const summaries = []; + for (const provider of providers) { + let added = 0; + let updated = 0; + for (const preset of catalog.presets) { + const existing = await db.query.aiModel.findFirst({ + where: and( + eq(aiModel.providerId, provider.id), + eq(aiModel.modelId, preset.modelId) + ), + }); + const values = { + displayName: preset.displayName, + avatarUrl: preset.avatarUrl || null, + description: preset.description, + supportsVision: preset.supportsVision, + supportsImageGen: preset.supportsImageGen, + isEnabled: true, + isHidden: false, + sortOrder: preset.sortOrder, + capabilities: preset.capabilities, + defaultParams: preset.defaultParams, + imageGenConfig: preset.imageGenConfig, + updatedAt: new Date(), + }; + + if (existing) { + await db + .update(aiModel) + .set(values) + .where(eq(aiModel.id, existing.id)); + updated += 1; + } else { + await db.insert(aiModel).values({ + providerId: provider.id, + modelId: preset.modelId, + ...values, + isDefault: false, + isSubModel: false, + parentModelId: null, + }); + added += 1; + } + } + summaries.push({ + providerId: provider.id, + providerName: provider.name, + added, + updated, + }); + } + + console.log( + JSON.stringify( + { + catalogModels: catalog.catalogCount, + availableModels: catalog.presets.length, + providers: summaries, + }, + null, + 2 + ) + ); +} + +void main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/scripts/verify-atlascloud-frontend.ts b/scripts/verify-atlascloud-frontend.ts new file mode 100644 index 000000000..2bfe269bd --- /dev/null +++ b/scripts/verify-atlascloud-frontend.ts @@ -0,0 +1,204 @@ +import { fetchAtlasCloudImagePresets } from "../lib/ai/atlascloud-models"; +import { createAtlasCloudRequestBody } from "../lib/ai/atlascloud-request"; +import { + is3DModelDescriptor, + isImageGenerationModelDescriptor, + isVideoModelDescriptor, +} from "../lib/ai/model-classification"; + +const FAKE_IMAGES = Array.from( + { length: 14 }, + (_, index) => `https://example.com/reference-${index + 1}.png` +); +const FAKE_VIDEOS = Array.from( + { length: 4 }, + (_, index) => `https://example.com/reference-${index + 1}.mp4` +); + +type Failure = { + modelId: string; + scenario: string; + error: string; +}; + +function validateBody( + body: Record, + runtime: any +) { + const missing = runtime.requiredFields.filter( + (key: string) => body[key] === undefined + ); + if (missing.length > 0) { + throw new Error(`缺少必填参数：${missing.join(", ")}`); + } + for (const [key, values] of Object.entries( + runtime.enums as Record> + )) { + if (body[key] === undefined || !values.length) continue; + if (!values.some((value) => String(value) === String(body[key]))) { + throw new Error(`${key}=${String(body[key])} 不在官方枚举中`); + } + } + if (runtime.videoClipInput && body[runtime.videoClipInput.key] !== undefined) { + const clips = body[runtime.videoClipInput.key]; + if (!Array.isArray(clips) || clips.length === 0) { + throw new Error("video_clips 请求格式无效"); + } + for (const clip of clips) { + if (!clip || typeof clip !== "object" || !("url" in clip)) { + throw new Error("video_clips 缺少 url"); + } + } + } +} + +async function main() { + const catalog = await fetchAtlasCloudImagePresets(); + const failures: Failure[] = catalog.errors.map((item) => ({ + modelId: item.modelId, + scenario: "catalog-schema", + error: item.error, + })); + let simulatedCases = 0; + let imageModels = 0; + let model3dModels = 0; + let videoReferenceModels = 0; + + for (const preset of catalog.presets) { + const capabilities = preset.capabilities as Record; + const descriptor = { + providerType: "atlascloud", + modelId: preset.modelId, + capabilities, + supportsImageGen: preset.supportsImageGen, + }; + const mediaType = String(capabilities.mediaType || ""); + + if (mediaType === "3d") { + model3dModels += 1; + simulatedCases += 1; + if ( + !is3DModelDescriptor(descriptor) || + isImageGenerationModelDescriptor(descriptor) || + isVideoModelDescriptor(descriptor) + ) { + failures.push({ + modelId: preset.modelId, + scenario: "3d-classification", + error: "3D 模型被错误归入图片或视频模型", + }); + } + continue; + } + + imageModels += 1; + if ( + !isImageGenerationModelDescriptor(descriptor) || + is3DModelDescriptor(descriptor) || + isVideoModelDescriptor(descriptor) + ) { + failures.push({ + modelId: preset.modelId, + scenario: "image-classification", + error: "图片模型分类不正确", + }); + continue; + } + + const imageGenConfig = preset.imageGenConfig as Record; + const runtime = imageGenConfig.atlasCloud; + if (!runtime) { + failures.push({ + modelId: preset.modelId, + scenario: "runtime-config", + error: "缺少 AtlasCloud 运行配置", + }); + continue; + } + + const ratios = imageGenConfig.supportedAspectRatios?.length + ? imageGenConfig.supportedAspectRatios + : [String(preset.defaultParams.aspectRatio || "1:1")]; + const sizes = imageGenConfig.supportedImageSizes?.length + ? imageGenConfig.supportedImageSizes + : [String(preset.defaultParams.imageSize || "1K")]; + const counts = runtime.outputCountKey && runtime.maxImages > 1 + ? [1, runtime.maxImages] + : [1]; + const requiredImageCount = runtime.imageInputs + .filter((input: any) => input.required) + .reduce((total: number, input: any) => total + input.maxItems, 0); + const imageReferenceCounts = new Set([ + requiredImageCount, + ...(runtime.maxReferenceImages > requiredImageCount + ? [runtime.maxReferenceImages] + : []), + ]); + const requiredVideoCount = runtime.videoClipInput?.required + ? runtime.videoClipInput.maxItems + : 0; + const videoReferenceCounts = new Set([ + requiredVideoCount, + ...(runtime.maxVideoReferences > requiredVideoCount + ? [runtime.maxVideoReferences] + : []), + ]); + if (runtime.maxVideoReferences > 0) videoReferenceModels += 1; + + for (const ratio of ratios) { + for (const size of sizes) { + for (const imageCount of counts) { + for (const referenceImageCount of imageReferenceCounts) { + for (const referenceVideoCount of videoReferenceCounts) { + const scenario = [ + `ratio=${ratio}`, + `size=${size}`, + `count=${imageCount}`, + `images=${referenceImageCount}`, + `videos=${referenceVideoCount}`, + ].join(","); + simulatedCases += 1; + try { + const { body } = createAtlasCloudRequestBody({ + modelId: preset.modelId, + prompt: "AtlasCloud 前端请求模拟测试", + referenceImages: FAKE_IMAGES.slice(0, referenceImageCount), + referenceVideos: FAKE_VIDEOS.slice(0, referenceVideoCount), + imageParams: { + aspectRatio: ratio, + imageSize: size, + imageCount, + }, + modelConfig: imageGenConfig, + }); + validateBody(body, runtime); + } catch (error) { + failures.push({ + modelId: preset.modelId, + scenario, + error: error instanceof Error ? error.message : String(error), + }); + } + } + } + } + } + } + } + + const summary = { + catalogModels: catalog.catalogCount, + availableModels: catalog.presets.length, + imageModels, + model3dModels, + videoReferenceModels, + simulatedCases, + passedCases: simulatedCases - failures.length, + failedCases: failures.length, + failures, + }; + console.log(JSON.stringify(summary, null, 2)); + if (failures.length > 0) process.exitCode = 1; +} + +void main(); diff --git a/tests/routes/model-classification.test.ts b/tests/routes/model-classification.test.ts index c0b50796b..9bbf17e91 100644 --- a/tests/routes/model-classification.test.ts +++ b/tests/routes/model-classification.test.ts @@ -3,6 +3,7 @@ import { createAtlasCloudImagePreset, inferAtlasCloudMediaKind, } from "@/lib/ai/atlascloud-models"; +import { createAtlasCloudRequestBody } from "@/lib/ai/atlascloud-request"; import { is3DModelDescriptor, isImageGenerationModelDescriptor, @@ -120,4 +121,71 @@ test.describe("AI model media classification", () => { expect(isVideoModelDescriptor(videoDescriptor)).toBe(true); expect(isImageGenerationModelDescriptor(videoDescriptor)).toBe(false); }); + + test("maps Atlas reference-video image models to a valid frontend request", () => { + const preset = createAtlasCloudImagePreset( + { + model: "google/nano-banana-2/reference-to-image", + type: "Image", + displayName: "Nano Banana 2 Reference to Image", + categories: ["IMAGE-TO-IMAGE"], + }, + { + paths: { + "/api/v1/model/generateImage": { post: {} }, + "/api/v1/model/prediction/{request_id}": { get: {} }, + }, + components: { + schemas: { + Input: { + type: "object", + properties: { + model: { type: "string" }, + prompt: { type: "string" }, + resolution: { type: "string", enum: ["1k", "2k", "4k"] }, + video_clips: { + type: "array", + minItems: 1, + maxItems: 1, + items: { + type: "object", + properties: { + url: { type: "string" }, + start: { type: "number", default: 0 }, + ends: { type: "number", default: 0 }, + fps: { type: "number", default: 1 }, + }, + }, + }, + }, + required: ["model", "prompt", "video_clips"], + }, + }, + }, + } + ); + + expect(preset.capabilities).toMatchObject({ + mediaType: "image", + supportsVideoReference: true, + requiresVideoReference: true, + maxVideoReferences: 1, + }); + const { body } = createAtlasCloudRequestBody({ + modelId: preset.modelId, + prompt: "参考视频生成商品图", + referenceImages: [], + referenceVideos: ["https://example.com/reference.mp4"], + imageParams: { imageSize: "2K", aspectRatio: "1:1" }, + modelConfig: preset.imageGenConfig, + }); + expect(body.video_clips).toEqual([ + { + url: "https://example.com/reference.mp4", + start: 0, + ends: 0, + fps: 1, + }, + ]); + }); });