ai-chatbot/lib/ai/providers/volcano-multimodal.ts

import "server-only";

interface VolcanoMultimodalConfig {
  apiKey: string;
  baseURL: string;
  timeout?: number;
}

function normalizeResponsesEndpoint(baseURL: string) {
  const cleaned = baseURL.trim().replace(/\/+$/, "");
  if (!cleaned) return "https://ark.cn-beijing.volces.com/api/v3/responses";
  if (/\/responses$/i.test(cleaned)) return cleaned;
  if (/\/api\/v3$/i.test(cleaned)) return `${cleaned}/responses`;
  if (/ark\.cn-beijing\.volces\.com/i.test(cleaned)) {
    return `${cleaned}/api/v3/responses`.replace(/\/api\/v3\/api\/v3\//, "/api/v3/");
  }
  return `${cleaned}/responses`;
}

function parseMessageParts(content: unknown) {
  const textParts: string[] = [];
  const imageUrls: string[] = [];

  const pushImageUrl = (value: unknown) => {
    if (typeof value !== "string") return;
    const url = value.trim();
    if (!url) return;
    imageUrls.push(url);
  };

  const readPart = (part: any) => {
    if (!part || typeof part !== "object") return;
    if (part.type === "text" || part.type === "input_text") {
      if (typeof part.text === "string" && part.text.trim()) textParts.push(part.text.trim());
      return;
    }
    if (part.type === "image" || part.type === "input_image") {
      pushImageUrl(part.image ?? part.image_url);
      return;
    }
    if (part.type === "image_url") {
      if (typeof part.image_url === "string") {
        pushImageUrl(part.image_url);
      } else {
        pushImageUrl(part.image_url?.url);
      }
      return;
    }
    if (part.type === "file") {
      pushImageUrl(part.url ?? part.data);
    }
  };

  if (typeof content === "string") {
    if (content.trim()) textParts.push(content.trim());
    return { text: textParts.join("\n"), imageUrls };
  }

  if (Array.isArray(content)) {
    for (const part of content) readPart(part);
    return { text: textParts.join("\n"), imageUrls };
  }

  if (content && typeof content === "object") {
    readPart(content);
  }

  return { text: textParts.join("\n"), imageUrls };
}

function extractPromptAndImages(options: any) {
  if (typeof options?.prompt === "string") {
    return { text: options.prompt.trim(), imageUrls: [] as string[] };
  }

  const messages = Array.isArray(options?.messages) ? options.messages : [];
  for (let i = messages.length - 1; i >= 0; i -= 1) {
    const message = messages[i];
    if (message?.role !== "user") continue;
    const parsed = parseMessageParts(message.content);
    if (parsed.text || parsed.imageUrls.length > 0) {
      return parsed;
    }
  }

  const lastMessage = messages[messages.length - 1];
  if (lastMessage) {
    return parseMessageParts(lastMessage.content);
  }

  return { text: "", imageUrls: [] as string[] };
}

function extractOutputText(payload: any): string {
  if (typeof payload?.output_text === "string" && payload.output_text.trim()) {
    return payload.output_text.trim();
  }

  const chunks: string[] = [];
  const output = Array.isArray(payload?.output) ? payload.output : [];
  for (const item of output) {
    const content = Array.isArray(item?.content) ? item.content : [];
    for (const part of content) {
      if (typeof part?.text === "string" && part.text.trim()) {
        chunks.push(part.text.trim());
      } else if (typeof part?.output_text === "string" && part.output_text.trim()) {
        chunks.push(part.output_text.trim());
      }
    }
  }

  return chunks.join("\n").trim();
}

async function callResponsesApi(params: {
  endpoint: string;
  apiKey: string;
  modelId: string;
  text: string;
  imageUrls: string[];
  timeout: number;
}) {
  const content = [
    ...params.imageUrls.map((url) => ({ type: "input_image", image_url: url })),
    { type: "input_text", text: params.text || "请根据图片内容进行分析。" },
  ];

  const controller = new AbortController();
  const timeoutId = setTimeout(() => controller.abort(), params.timeout);

  try {
    const response = await fetch(params.endpoint, {
      method: "POST",
      signal: controller.signal,
      headers: {
        Authorization: `Bearer ${params.apiKey}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify({
        model: params.modelId,
        input: [
          {
            role: "user",
            content,
          },
        ],
      }),
    });

    const payload = await response.json().catch(() => ({}));
    if (!response.ok || payload?.error) {
      const message =
        payload?.error?.message || payload?.message || `HTTP ${response.status}`;
      throw new Error(message);
    }

    const text = extractOutputText(payload);
    if (!text) {
      throw new Error("火山多模态未返回文本结果");
    }

    const usage = payload?.usage || {};
    return {
      text,
      usage: {
        promptTokens:
          Number(usage?.input_tokens) ||
          Number(usage?.prompt_tokens) ||
          0,
        completionTokens:
          Number(usage?.output_tokens) ||
          Number(usage?.completion_tokens) ||
          0,
      },
    };
  } finally {
    clearTimeout(timeoutId);
  }
}

export function createVolcanoMultimodal(config: VolcanoMultimodalConfig) {
  const endpoint = normalizeResponsesEndpoint(config.baseURL);
  const timeout = config.timeout ?? 60000;

  return {
    provider: "volcano-multimodal",
    chat: (modelId: string) => ({
      provider: "volcano-multimodal",
      modelId,
      specificationVersion: "v1" as const,
      defaultObjectGenerationMode: "json" as const,
      doGenerate: async (options: any) => {
        const { text, imageUrls } = extractPromptAndImages(options);
        const result = await callResponsesApi({
          endpoint,
          apiKey: config.apiKey,
          modelId,
          text,
          imageUrls,
          timeout,
        });
        return {
          text: result.text,
          finishReason: "stop" as const,
          usage: result.usage,
        };
      },
      doStream: async function* (options: any) {
        const result = await this.doGenerate(options);
        yield {
          type: "text-delta" as const,
          textDelta: result.text,
        };
        yield {
          type: "finish" as const,
          finishReason: "stop" as const,
          usage: result.usage,
        };
      },
    }),
  };
}