import "server-only"; interface VolcanoMultimodalConfig { apiKey: string; baseURL: string; timeout?: number; } function normalizeResponsesEndpoint(baseURL: string) { const cleaned = baseURL.trim().replace(/\/+$/, ""); if (!cleaned) return "https://ark.cn-beijing.volces.com/api/v3/responses"; if (/\/responses$/i.test(cleaned)) return cleaned; if (/\/api\/v3$/i.test(cleaned)) return `${cleaned}/responses`; if (/ark\.cn-beijing\.volces\.com/i.test(cleaned)) { return `${cleaned}/api/v3/responses`.replace(/\/api\/v3\/api\/v3\//, "/api/v3/"); } return `${cleaned}/responses`; } function parseMessageParts(content: unknown) { const textParts: string[] = []; const imageUrls: string[] = []; const pushImageUrl = (value: unknown) => { const raw = typeof value === "string" ? value : value instanceof URL ? value.toString() : ""; if (!raw) return; const url = raw.trim(); if (!url) return; imageUrls.push(url); }; const readPart = (part: any) => { if (!part || typeof part !== "object") return; if (part.type === "text" || part.type === "input_text") { if (typeof part.text === "string" && part.text.trim()) textParts.push(part.text.trim()); return; } if (part.type === "image" || part.type === "input_image") { pushImageUrl(part.image ?? part.image_url); return; } if (part.type === "image_url") { if (typeof part.image_url === "string") { pushImageUrl(part.image_url); } else { pushImageUrl(part.image_url?.url); } return; } if (part.type === "file") { pushImageUrl(part.url ?? part.data); } }; if (typeof content === "string") { if (content.trim()) textParts.push(content.trim()); return { text: textParts.join("\n"), imageUrls }; } if (Array.isArray(content)) { for (const part of content) readPart(part); return { text: textParts.join("\n"), imageUrls }; } if (content && typeof content === "object") { readPart(content); } return { text: textParts.join("\n"), imageUrls }; } function extractPromptAndImages(options: any) { if (typeof options?.prompt === "string") { return { text: options.prompt.trim(), imageUrls: [] as string[] }; } const messages = Array.isArray(options?.messages) ? options.messages : Array.isArray(options?.prompt) ? options.prompt : []; for (let i = messages.length - 1; i >= 0; i -= 1) { const message = messages[i]; if (message?.role !== "user") continue; const parsed = parseMessageParts(message.content); if (parsed.text || parsed.imageUrls.length > 0) { return parsed; } } const lastMessage = messages[messages.length - 1]; if (lastMessage) { return parseMessageParts(lastMessage.content); } return { text: "", imageUrls: [] as string[] }; } function extractOutputText(payload: any): string { if (typeof payload?.output_text === "string" && payload.output_text.trim()) { return payload.output_text.trim(); } const chunks: string[] = []; const output = Array.isArray(payload?.output) ? payload.output : []; for (const item of output) { const content = Array.isArray(item?.content) ? item.content : []; for (const part of content) { if (typeof part?.text === "string" && part.text.trim()) { chunks.push(part.text.trim()); } else if (typeof part?.output_text === "string" && part.output_text.trim()) { chunks.push(part.output_text.trim()); } } } return chunks.join("\n").trim(); } async function callResponsesApi(params: { endpoint: string; apiKey: string; modelId: string; text: string; imageUrls: string[]; timeout: number; }) { const remoteImageUrls = params.imageUrls .map((url) => String(url || "").trim()) .filter((url) => /^https?:\/\//i.test(url)); const content = [ ...remoteImageUrls.map((url) => ({ type: "input_image", image_url: url })), { type: "input_text", text: params.text || "请根据图片内容进行分析。" }, ]; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), params.timeout); try { const response = await fetch(params.endpoint, { method: "POST", signal: controller.signal, headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: params.modelId, input: [ { role: "user", content, }, ], }), }); const payload = await response.json().catch(() => ({})); if (!response.ok || payload?.error) { const message = payload?.error?.message || payload?.message || `HTTP ${response.status}`; throw new Error(message); } const text = extractOutputText(payload); if (!text) { throw new Error("火山多模态未返回文本结果"); } const usage = payload?.usage || {}; return { text, usage: { promptTokens: Number(usage?.input_tokens) || Number(usage?.prompt_tokens) || 0, completionTokens: Number(usage?.output_tokens) || Number(usage?.completion_tokens) || 0, }, }; } finally { clearTimeout(timeoutId); } } export function createVolcanoMultimodal(config: VolcanoMultimodalConfig) { const endpoint = normalizeResponsesEndpoint(config.baseURL); const timeout = config.timeout ?? 120000; const toV2Usage = (usage: { promptTokens?: number; completionTokens?: number }) => { const inputTokens = usage.promptTokens ?? 0; const outputTokens = usage.completionTokens ?? 0; return { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens, }; }; const createV2TextStream = ( run: Promise<{ text: string; finishReason: "stop" | "error"; usage: { promptTokens?: number; completionTokens?: number } }> ) => new ReadableStream({ start(controller) { controller.enqueue({ type: "stream-start", warnings: [] }); void run .then((result) => { if (result.text) { const textId = `txt-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; controller.enqueue({ type: "text-start", id: textId }); controller.enqueue({ type: "text-delta", id: textId, delta: result.text, }); controller.enqueue({ type: "text-end", id: textId }); } controller.enqueue({ type: "finish", finishReason: result.finishReason, usage: toV2Usage(result.usage), }); }) .catch((error) => { controller.enqueue({ type: "error", error }); controller.enqueue({ type: "finish", finishReason: "error", usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, }, }); }) .finally(() => controller.close()); }, }); return { provider: "volcano-multimodal", chat: (modelId: string) => ({ provider: "volcano-multimodal", modelId, specificationVersion: "v2" as const, supportedUrls: {}, defaultObjectGenerationMode: "json" as const, doGenerate: async (options: any) => { const { text, imageUrls } = extractPromptAndImages(options); const result = await callResponsesApi({ endpoint, apiKey: config.apiKey, modelId, text, imageUrls, timeout, }); const legacyResult = { text: result.text, finishReason: "stop" as const, usage: result.usage, }; const isV2Call = Array.isArray(options?.prompt); if (isV2Call) { return { content: [{ type: "text", text: result.text }], finishReason: "stop" as const, usage: toV2Usage(result.usage), warnings: [], }; } return legacyResult; }, doStream: function (this: any, options: any) { const isV2Call = Array.isArray(options?.prompt); if (!isV2Call) { const legacyStream = async function* (this: any) { const result = await this.doGenerate(options); yield { type: "text-delta" as const, textDelta: result.text, }; yield { type: "finish" as const, finishReason: "stop" as const, usage: result.usage, }; }; return legacyStream.call(this); } const run = (async () => { try { const result = await this.doGenerate(options); const content = Array.isArray(result?.content) ? result.content : []; const text = content .filter((part: any) => part?.type === "text" && typeof part.text === "string") .map((part: any) => part.text) .join(""); return { text, finishReason: "stop" as const, usage: { promptTokens: result?.usage?.inputTokens ?? 0, completionTokens: result?.usage?.outputTokens ?? 0, }, }; } catch (error) { return { text: error instanceof Error ? error.message : "请求失败", finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } })(); return Promise.resolve({ stream: createV2TextStream(run), }); }, }), }; }