import "server-only"; interface VolcanoMultimodalConfig { apiKey: string; baseURL: string; timeout?: number; } function normalizeResponsesEndpoint(baseURL: string) { const cleaned = baseURL.trim().replace(/\/+$/, ""); if (!cleaned) return "https://ark.cn-beijing.volces.com/api/v3/responses"; if (/\/responses$/i.test(cleaned)) return cleaned; if (/\/api\/v3$/i.test(cleaned)) return `${cleaned}/responses`; if (/ark\.cn-beijing\.volces\.com/i.test(cleaned)) { return `${cleaned}/api/v3/responses`.replace(/\/api\/v3\/api\/v3\//, "/api/v3/"); } return `${cleaned}/responses`; } function parseMessageParts(content: unknown) { const textParts: string[] = []; const imageUrls: string[] = []; const pushImageUrl = (value: unknown) => { if (typeof value !== "string") return; const url = value.trim(); if (!url) return; imageUrls.push(url); }; const readPart = (part: any) => { if (!part || typeof part !== "object") return; if (part.type === "text" || part.type === "input_text") { if (typeof part.text === "string" && part.text.trim()) textParts.push(part.text.trim()); return; } if (part.type === "image" || part.type === "input_image") { pushImageUrl(part.image ?? part.image_url); return; } if (part.type === "image_url") { if (typeof part.image_url === "string") { pushImageUrl(part.image_url); } else { pushImageUrl(part.image_url?.url); } return; } if (part.type === "file") { pushImageUrl(part.url ?? part.data); } }; if (typeof content === "string") { if (content.trim()) textParts.push(content.trim()); return { text: textParts.join("\n"), imageUrls }; } if (Array.isArray(content)) { for (const part of content) readPart(part); return { text: textParts.join("\n"), imageUrls }; } if (content && typeof content === "object") { readPart(content); } return { text: textParts.join("\n"), imageUrls }; } function extractPromptAndImages(options: any) { if (typeof options?.prompt === "string") { return { text: options.prompt.trim(), imageUrls: [] as string[] }; } const messages = Array.isArray(options?.messages) ? options.messages : []; for (let i = messages.length - 1; i >= 0; i -= 1) { const message = messages[i]; if (message?.role !== "user") continue; const parsed = parseMessageParts(message.content); if (parsed.text || parsed.imageUrls.length > 0) { return parsed; } } const lastMessage = messages[messages.length - 1]; if (lastMessage) { return parseMessageParts(lastMessage.content); } return { text: "", imageUrls: [] as string[] }; } function extractOutputText(payload: any): string { if (typeof payload?.output_text === "string" && payload.output_text.trim()) { return payload.output_text.trim(); } const chunks: string[] = []; const output = Array.isArray(payload?.output) ? payload.output : []; for (const item of output) { const content = Array.isArray(item?.content) ? item.content : []; for (const part of content) { if (typeof part?.text === "string" && part.text.trim()) { chunks.push(part.text.trim()); } else if (typeof part?.output_text === "string" && part.output_text.trim()) { chunks.push(part.output_text.trim()); } } } return chunks.join("\n").trim(); } async function callResponsesApi(params: { endpoint: string; apiKey: string; modelId: string; text: string; imageUrls: string[]; timeout: number; }) { const content = [ ...params.imageUrls.map((url) => ({ type: "input_image", image_url: url })), { type: "input_text", text: params.text || "请根据图片内容进行分析。" }, ]; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), params.timeout); try { const response = await fetch(params.endpoint, { method: "POST", signal: controller.signal, headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: params.modelId, input: [ { role: "user", content, }, ], }), }); const payload = await response.json().catch(() => ({})); if (!response.ok || payload?.error) { const message = payload?.error?.message || payload?.message || `HTTP ${response.status}`; throw new Error(message); } const text = extractOutputText(payload); if (!text) { throw new Error("火山多模态未返回文本结果"); } const usage = payload?.usage || {}; return { text, usage: { promptTokens: Number(usage?.input_tokens) || Number(usage?.prompt_tokens) || 0, completionTokens: Number(usage?.output_tokens) || Number(usage?.completion_tokens) || 0, }, }; } finally { clearTimeout(timeoutId); } } export function createVolcanoMultimodal(config: VolcanoMultimodalConfig) { const endpoint = normalizeResponsesEndpoint(config.baseURL); const timeout = config.timeout ?? 60000; return { provider: "volcano-multimodal", chat: (modelId: string) => ({ provider: "volcano-multimodal", modelId, specificationVersion: "v1" as const, defaultObjectGenerationMode: "json" as const, doGenerate: async (options: any) => { const { text, imageUrls } = extractPromptAndImages(options); const result = await callResponsesApi({ endpoint, apiKey: config.apiKey, modelId, text, imageUrls, timeout, }); return { text: result.text, finishReason: "stop" as const, usage: result.usage, }; }, doStream: async function* (options: any) { const result = await this.doGenerate(options); yield { type: "text-delta" as const, textDelta: result.text, }; yield { type: "finish" as const, finishReason: "stop" as const, usage: result.usage, }; }, }), }; }