import "server-only"; import { randomUUID } from "node:crypto"; import sharp from "sharp"; import { getVisionaryModelPresetById, VISIONARY_BASE_URL, } from "@/lib/ai/visionary-models"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface VisionaryConfig { apiKey: string; baseURL?: string; } interface VisionaryImageParams { aspectRatio?: string; imageSize?: string; imageCount?: number; promptExtend?: boolean; quality?: "auto" | "low" | "medium" | "high" | string; } interface VisionaryModelConfig { maxReferenceImages?: number; supportedAspectRatios?: string[]; supportedImageSizes?: string[]; supportsPromptExtend?: boolean; supportsQuality?: boolean; } interface ReferenceImage { apiValue: string; height?: number; width?: number; } interface UploadedImage { thumbnail: string; original: string; } const POLL_INTERVAL_MS = 3000; const MAX_POLL_INTERVAL_MS = 15_000; const POLL_TIMEOUT_MS = 18 * 60 * 1000; const MAX_TRANSIENT_POLL_FAILURES = 8; const IMAGE_DOWNLOAD_TIMEOUT_MS = 120_000; function joinApiPath(baseURL: string, path: string) { const cleanBase = String(baseURL || VISIONARY_BASE_URL).replace(/\/+$/, ""); const cleanPath = path.startsWith("/") ? path : `/${path}`; return `${cleanBase}${cleanPath}`; } function delay(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } function normalizeModelId(modelId?: string) { const normalized = String(modelId || "").trim(); if (!normalized) throw new Error("Visionary 模型 ID 未配置"); return normalized; } function normalizeAspectRatio(value?: string) { const normalized = String(value || "auto").trim().toLowerCase(); if (normalized === "auto") return "auto"; const match = normalized.match(/^(\d+)\s*:\s*(\d+)$/); if (!match) return "auto"; return `${Number(match[1])}:${Number(match[2])}`; } function getSupportedAspectRatios( modelId: string, modelConfig?: VisionaryModelConfig | null ) { const preset = getVisionaryModelPresetById(modelId); const configured = modelConfig?.supportedAspectRatios; const values = Array.isArray(configured) ? configured : (preset?.imageGenConfig?.supportedAspectRatios as string[] | undefined); return Array.from( new Set((values || ["auto", "1:1"]).map(normalizeAspectRatio)) ); } function resolveAspectRatio( requested: string | undefined, modelId: string, modelConfig?: VisionaryModelConfig | null ) { const supported = getSupportedAspectRatios(modelId, modelConfig); const normalized = normalizeAspectRatio(requested); if (supported.includes(normalized)) return normalized; return supported.includes("auto") ? "auto" : supported[0] || "1:1"; } function normalizeImageSize( requested: string | undefined, modelId: string, modelConfig?: VisionaryModelConfig | null ) { const preset = getVisionaryModelPresetById(modelId); const configured = modelConfig?.supportedImageSizes; const supported = (Array.isArray(configured) ? configured : (preset?.imageGenConfig?.supportedImageSizes as string[] | undefined) || ["1K"] ) .map((size) => String(size).trim().toUpperCase()) .filter((size) => ["1K", "2K", "4K"].includes(size)); const defaultSize = String(preset?.defaultParams?.imageSize || supported[0] || "1K") .trim() .toUpperCase(); const normalized = String(requested || defaultSize).trim().toUpperCase(); if (supported.includes(normalized)) return normalized; return supported.includes(defaultSize) ? defaultSize : supported[0] || "1K"; } function normalizeQuality(value?: string) { const normalized = String(value || "auto").trim().toLowerCase(); return ["auto", "low", "medium", "high"].includes(normalized) ? normalized : "auto"; } function extractPromptAndImages( messages: Array<{ role: string; content?: any; parts?: any[] }> ) { const lastUserMessage = messages.filter((message) => message.role === "user").pop(); if (!lastUserMessage) throw new Error("未找到用户消息"); const texts: string[] = []; const imageUrls: string[] = []; const seenImages = new Set(); const pushText = (value: unknown) => { if (typeof value === "string" && value.trim()) texts.push(value.trim()); }; const pushImage = (value: unknown) => { let url = ""; if (typeof value === "string") url = value; else if (value instanceof URL) url = value.toString(); else if (value && typeof value === "object") { url = String( (value as any).url || (value as any).imageUrl || (value as any).fileUrl || (value as any).src || "" ); } url = url.trim(); if (!url || seenImages.has(url)) return; if (!url.startsWith("data:image/") && !/^https?:\/\//i.test(url)) return; seenImages.add(url); imageUrls.push(url); }; const visitPart = (part: any) => { if (!part) return; if (typeof part === "string") { pushText(part); return; } if (part.type === "text" || typeof part.text === "string") pushText(part.text); if (part.type === "image_url") pushImage(part.image_url); if (part.type === "image") pushImage(part.image || part.url); if (part.type === "file") pushImage(part.url || part.file || part.data); if (part.inlineData?.data) { pushImage( `data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}` ); } }; const content = lastUserMessage.content; if (typeof content === "string") pushText(content); else if (Array.isArray(content)) content.forEach(visitPart); if (Array.isArray(lastUserMessage.parts)) lastUserMessage.parts.forEach(visitPart); const prompt = texts.join("\n").trim(); if (!prompt) throw new Error("请输入图片生成或编辑要求"); return { prompt, imageUrls }; } function parseDataUrl(dataUrl: string) { const match = dataUrl.match(/^data:(image\/(?:jpeg|png|webp));base64,(.+)$/s); if (!match) throw new Error("Visionary 参考图仅支持 JPEG、PNG 或 WebP Data URI"); const mimeType = match[1] || "image/png"; const buffer = Buffer.from(match[2] || "", "base64"); return { mimeType, buffer, dataUrl }; } async function loadReferenceImage(imageUrl: string): Promise { if (imageUrl.startsWith("data:")) { const parsed = parseDataUrl(imageUrl); const metadata = await sharp(parsed.buffer, { failOn: "none" }) .metadata() .catch(() => null); return { apiValue: parsed.dataUrl, width: metadata?.width, height: metadata?.height, }; } const response = await fetch(imageUrl, { cache: "no-store", redirect: "follow", signal: AbortSignal.timeout(IMAGE_DOWNLOAD_TIMEOUT_MS), headers: { Accept: "image/avif,image/webp,image/png,image/jpeg,image/*,*/*;q=0.8", "User-Agent": "yedir-ai-visionary/1.0", }, }); if (!response.ok) { throw new Error(`Visionary 参考图下载失败(HTTP ${response.status})`); } const mimeType = response.headers.get("content-type")?.split(";")[0] || ""; if (!/^image\/(?:jpeg|png|webp)$/i.test(mimeType)) { throw new Error(`Visionary 不支持此参考图格式:${mimeType || "unknown"}`); } const buffer = Buffer.from(await response.arrayBuffer()); const metadata = await sharp(buffer, { failOn: "none" }) .metadata() .catch(() => null); return { apiValue: /^https:\/\//i.test(imageUrl) ? imageUrl : `data:${mimeType};base64,${buffer.toString("base64")}`, width: metadata?.width, height: metadata?.height, }; } function detectImageExtension(mimeType?: string | null) { const normalized = String(mimeType || "").toLowerCase(); if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg"; if (normalized.includes("webp")) return "webp"; if (normalized.includes("gif")) return "gif"; return "png"; } function generateFilename(prefix: string, extension: string) { return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${extension}`; } function toArrayBuffer(buffer: Buffer) { return buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; } async function uploadImageBuffer( buffer: Buffer, mimeType?: string | null ): Promise { const extension = detectImageExtension(mimeType); const thumbnailBuffer = await sharp(buffer, { failOn: "none" }) .resize(400, 400, { fit: "inside", withoutEnlargement: true }) .jpeg({ quality: 82 }) .toBuffer() .catch(() => buffer); const [original, thumbnail] = await Promise.all([ uploadFile(generateFilename("visionary-orig", extension), toArrayBuffer(buffer)), uploadFile(generateFilename("visionary-thumb", "jpg"), toArrayBuffer(thumbnailBuffer)), ]); return { original: original.url, thumbnail: thumbnail.url }; } async function archiveOutputImage(imageUrl: string): Promise { if (imageUrl.startsWith("data:")) { const parsed = parseDataUrl(imageUrl); return uploadImageBuffer(parsed.buffer, parsed.mimeType); } const deferred = await createDeferredUploadedImage(imageUrl, { providerType: "visionary", }); if (deferred) return deferred; const response = await fetch(imageUrl, { cache: "no-store", redirect: "follow", signal: AbortSignal.timeout(IMAGE_DOWNLOAD_TIMEOUT_MS), headers: { "User-Agent": "yedir-ai-visionary/1.0" }, }); if (!response.ok) { throw new Error(`Visionary 结果图片下载失败(HTTP ${response.status})`); } return uploadImageBuffer( Buffer.from(await response.arrayBuffer()), response.headers.get("content-type") ); } async function parseJsonResponse(response: Response) { const rawText = await response.text(); if (!rawText.trim()) return null; try { return JSON.parse(rawText); } catch { return rawText; } } function extractErrorMessage(payload: any, fallback = "Visionary 请求失败") { if (typeof payload === "string" && payload.trim()) return payload.trim(); const value = payload?.error?.message || payload?.data?.error?.message || payload?.data?.error || payload?.error || payload?.message || fallback; return typeof value === "string" ? value : fallback; } function extractSubmittedTask(payload: any) { const task = Array.isArray(payload?.data) ? payload.data[0] : payload?.data || payload; const taskId = String(task?.task_id || task?.taskId || task?.id || "").trim(); const retryAfter = Number(task?.retry_after || task?.retryAfter || 3); return { taskId, retryAfter }; } function extractTaskImages(payload: any) { const task = payload?.data || payload; const images = task?.result?.images; if (!Array.isArray(images)) return [] as string[]; return Array.from( new Set( images.flatMap((image: any) => { const value = image?.url; if (Array.isArray(value)) return value; return value ? [value] : []; }) ) ) .map((value) => String(value || "").trim()) .filter((value) => /^https?:\/\//i.test(value) || value.startsWith("data:image/")); } async function submitTask(params: { apiKey: string; aspectRatio: string; baseURL: string; imageSize: string; modelId: string; prompt: string; promptExtend?: boolean; quality?: string; referenceImages: ReferenceImage[]; }) { const clientRequestId = `yedir-${randomUUID()}`; const body: Record = { model: params.modelId, prompt: params.prompt, images: params.referenceImages.map((image) => image.apiValue), size: params.aspectRatio, resolution: params.imageSize, client_request_id: clientRequestId, }; if (params.modelId === "gpt-image-2") { body.quality = normalizeQuality(params.quality); } if (params.modelId === "nano-banana-pro") { body.optimizeChineseText = Boolean(params.promptExtend); } const response = await fetch(joinApiPath(params.baseURL, "/v1/images/generations"), { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", "Idempotency-Key": clientRequestId, "User-Agent": "yedir-ai-visionary/1.0", }, body: JSON.stringify(body), signal: AbortSignal.timeout(30_000), }); const payload = await parseJsonResponse(response); if (!response.ok) { throw new Error(`${extractErrorMessage(payload)}(HTTP ${response.status})`); } const task = extractSubmittedTask(payload); if (!task.taskId) throw new Error("Visionary 未返回 task_id"); return task; } async function waitForTask(params: { apiKey: string; baseURL: string; initialRetryAfter: number; taskId: string; }) { const startedAt = Date.now(); let retryAfter = params.initialRetryAfter; let transientFailures = 0; while (Date.now() - startedAt < POLL_TIMEOUT_MS) { const waitMs = Math.min( MAX_POLL_INTERVAL_MS, Math.max(POLL_INTERVAL_MS, Math.round(retryAfter * 1000)) ); await delay(waitMs); try { const response = await fetch( joinApiPath(params.baseURL, `/v1/tasks/${encodeURIComponent(params.taskId)}`), { cache: "no-store", headers: { Authorization: `Bearer ${params.apiKey}`, "User-Agent": "yedir-ai-visionary/1.0", }, signal: AbortSignal.timeout(30_000), } ); const payload = await parseJsonResponse(response); if (!response.ok) { if (response.status >= 500 || response.status === 429) { throw new Error(`HTTP ${response.status}`); } throw new Error(extractErrorMessage(payload)); } transientFailures = 0; const task = payload?.data || payload; const status = String(task?.status || "").trim().toLowerCase(); retryAfter = Number(task?.retry_after || retryAfter || 3); if (status === "completed") { const images = extractTaskImages(payload); if (images.length === 0) throw new Error("Visionary 任务完成但未返回图片"); return images; } if (["failed", "error", "cancelled", "canceled"].includes(status)) { throw new Error(extractErrorMessage(payload, "Visionary 图片生成失败")); } } catch (error) { const message = error instanceof Error ? error.message : String(error); if (!/^HTTP (?:429|5\d\d)$/.test(message) && !/fetch|network|abort/i.test(message)) { throw error; } transientFailures += 1; if (transientFailures >= MAX_TRANSIENT_POLL_FAILURES) throw error; } } throw new Error("Visionary 生成超时,任务可能仍在处理,请稍后重试"); } function formatMarkdown(images: UploadedImage[], imageSize: string) { return images .map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`) .join("\n\n"); } export function createVisionary({ apiKey, baseURL = VISIONARY_BASE_URL }: VisionaryConfig) { return { provider: "visionary", chat: (modelId: string) => ({ provider: "visionary", doGenerate: async (options: { messages: Array<{ role: string; content?: any; parts?: any[] }>; imageParams?: VisionaryImageParams; imageGenConfig?: VisionaryModelConfig | null; overrideModelId?: string; }) => { const requestedModelId = normalizeModelId(options.overrideModelId || modelId); const preset = getVisionaryModelPresetById(requestedModelId); if (!preset) throw new Error(`Visionary 暂不支持模型:${requestedModelId}`); const { prompt, imageUrls } = extractPromptAndImages(options.messages); const maxReferenceImages = Math.max( 0, Math.min(9, options.imageGenConfig?.maxReferenceImages || 9) ); const referenceImages = await Promise.all( imageUrls.slice(0, maxReferenceImages).map(loadReferenceImage) ); const aspectRatio = resolveAspectRatio( options.imageParams?.aspectRatio, requestedModelId, options.imageGenConfig ); const imageSize = normalizeImageSize( options.imageParams?.imageSize, requestedModelId, options.imageGenConfig ); const submitted = await submitTask({ apiKey, aspectRatio, baseURL, imageSize, modelId: requestedModelId, prompt, promptExtend: options.imageParams?.promptExtend, quality: options.imageParams?.quality, referenceImages, }); const outputUrls = await waitForTask({ apiKey, baseURL, initialRetryAfter: submitted.retryAfter, taskId: submitted.taskId, }); const uploadedImages = await Promise.all( outputUrls .slice(0, Math.max(1, options.imageParams?.imageCount || 1)) .map(archiveOutputImage) ); const text = formatMarkdown(uploadedImages, imageSize); return { text, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: text.length }, }; }, doStream: async function* (options: { messages: Array<{ role: string; content?: any; parts?: any[] }>; imageParams?: VisionaryImageParams; imageGenConfig?: VisionaryModelConfig | null; overrideModelId?: string; }) { const result = await this.doGenerate(options); yield { type: "text-delta", textDelta: result.text }; }, }), }; }