337 lines
9.7 KiB
TypeScript
337 lines
9.7 KiB
TypeScript
import "server-only";
|
|
|
|
interface VolcanoMultimodalConfig {
|
|
apiKey: string;
|
|
baseURL: string;
|
|
timeout?: number;
|
|
}
|
|
|
|
function normalizeResponsesEndpoint(baseURL: string) {
|
|
const cleaned = baseURL.trim().replace(/\/+$/, "");
|
|
if (!cleaned) return "https://ark.cn-beijing.volces.com/api/v3/responses";
|
|
if (/\/responses$/i.test(cleaned)) return cleaned;
|
|
if (/\/api\/v3$/i.test(cleaned)) return `${cleaned}/responses`;
|
|
if (/ark\.cn-beijing\.volces\.com/i.test(cleaned)) {
|
|
return `${cleaned}/api/v3/responses`.replace(/\/api\/v3\/api\/v3\//, "/api/v3/");
|
|
}
|
|
return `${cleaned}/responses`;
|
|
}
|
|
|
|
function parseMessageParts(content: unknown) {
|
|
const textParts: string[] = [];
|
|
const imageUrls: string[] = [];
|
|
|
|
const pushImageUrl = (value: unknown) => {
|
|
const raw =
|
|
typeof value === "string"
|
|
? value
|
|
: value instanceof URL
|
|
? value.toString()
|
|
: "";
|
|
if (!raw) return;
|
|
const url = raw.trim();
|
|
if (!url) return;
|
|
imageUrls.push(url);
|
|
};
|
|
|
|
const readPart = (part: any) => {
|
|
if (!part || typeof part !== "object") return;
|
|
if (part.type === "text" || part.type === "input_text") {
|
|
if (typeof part.text === "string" && part.text.trim()) textParts.push(part.text.trim());
|
|
return;
|
|
}
|
|
if (part.type === "image" || part.type === "input_image") {
|
|
pushImageUrl(part.image ?? part.image_url);
|
|
return;
|
|
}
|
|
if (part.type === "image_url") {
|
|
if (typeof part.image_url === "string") {
|
|
pushImageUrl(part.image_url);
|
|
} else {
|
|
pushImageUrl(part.image_url?.url);
|
|
}
|
|
return;
|
|
}
|
|
if (part.type === "file") {
|
|
pushImageUrl(part.url ?? part.data);
|
|
}
|
|
};
|
|
|
|
if (typeof content === "string") {
|
|
if (content.trim()) textParts.push(content.trim());
|
|
return { text: textParts.join("\n"), imageUrls };
|
|
}
|
|
|
|
if (Array.isArray(content)) {
|
|
for (const part of content) readPart(part);
|
|
return { text: textParts.join("\n"), imageUrls };
|
|
}
|
|
|
|
if (content && typeof content === "object") {
|
|
readPart(content);
|
|
}
|
|
|
|
return { text: textParts.join("\n"), imageUrls };
|
|
}
|
|
|
|
function extractPromptAndImages(options: any) {
|
|
if (typeof options?.prompt === "string") {
|
|
return { text: options.prompt.trim(), imageUrls: [] as string[] };
|
|
}
|
|
|
|
const messages = Array.isArray(options?.messages)
|
|
? options.messages
|
|
: Array.isArray(options?.prompt)
|
|
? options.prompt
|
|
: [];
|
|
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
|
const message = messages[i];
|
|
if (message?.role !== "user") continue;
|
|
const parsed = parseMessageParts(message.content);
|
|
if (parsed.text || parsed.imageUrls.length > 0) {
|
|
return parsed;
|
|
}
|
|
}
|
|
|
|
const lastMessage = messages[messages.length - 1];
|
|
if (lastMessage) {
|
|
return parseMessageParts(lastMessage.content);
|
|
}
|
|
|
|
return { text: "", imageUrls: [] as string[] };
|
|
}
|
|
|
|
function extractOutputText(payload: any): string {
|
|
if (typeof payload?.output_text === "string" && payload.output_text.trim()) {
|
|
return payload.output_text.trim();
|
|
}
|
|
|
|
const chunks: string[] = [];
|
|
const output = Array.isArray(payload?.output) ? payload.output : [];
|
|
for (const item of output) {
|
|
const content = Array.isArray(item?.content) ? item.content : [];
|
|
for (const part of content) {
|
|
if (typeof part?.text === "string" && part.text.trim()) {
|
|
chunks.push(part.text.trim());
|
|
} else if (typeof part?.output_text === "string" && part.output_text.trim()) {
|
|
chunks.push(part.output_text.trim());
|
|
}
|
|
}
|
|
}
|
|
|
|
return chunks.join("\n").trim();
|
|
}
|
|
|
|
async function callResponsesApi(params: {
|
|
endpoint: string;
|
|
apiKey: string;
|
|
modelId: string;
|
|
text: string;
|
|
imageUrls: string[];
|
|
timeout: number;
|
|
}) {
|
|
const remoteImageUrls = params.imageUrls
|
|
.map((url) => String(url || "").trim())
|
|
.filter((url) => /^https?:\/\//i.test(url));
|
|
|
|
const content = [
|
|
...remoteImageUrls.map((url) => ({ type: "input_image", image_url: url })),
|
|
{ type: "input_text", text: params.text || "请根据图片内容进行分析。" },
|
|
];
|
|
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => controller.abort(), params.timeout);
|
|
|
|
try {
|
|
const response = await fetch(params.endpoint, {
|
|
method: "POST",
|
|
signal: controller.signal,
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({
|
|
model: params.modelId,
|
|
input: [
|
|
{
|
|
role: "user",
|
|
content,
|
|
},
|
|
],
|
|
}),
|
|
});
|
|
|
|
const payload = await response.json().catch(() => ({}));
|
|
if (!response.ok || payload?.error) {
|
|
const message =
|
|
payload?.error?.message || payload?.message || `HTTP ${response.status}`;
|
|
throw new Error(message);
|
|
}
|
|
|
|
const text = extractOutputText(payload);
|
|
if (!text) {
|
|
throw new Error("火山多模态未返回文本结果");
|
|
}
|
|
|
|
const usage = payload?.usage || {};
|
|
return {
|
|
text,
|
|
usage: {
|
|
promptTokens:
|
|
Number(usage?.input_tokens) ||
|
|
Number(usage?.prompt_tokens) ||
|
|
0,
|
|
completionTokens:
|
|
Number(usage?.output_tokens) ||
|
|
Number(usage?.completion_tokens) ||
|
|
0,
|
|
},
|
|
};
|
|
} finally {
|
|
clearTimeout(timeoutId);
|
|
}
|
|
}
|
|
|
|
export function createVolcanoMultimodal(config: VolcanoMultimodalConfig) {
|
|
const endpoint = normalizeResponsesEndpoint(config.baseURL);
|
|
const timeout = config.timeout ?? 120000;
|
|
const toV2Usage = (usage: { promptTokens?: number; completionTokens?: number }) => {
|
|
const inputTokens = usage.promptTokens ?? 0;
|
|
const outputTokens = usage.completionTokens ?? 0;
|
|
return {
|
|
inputTokens,
|
|
outputTokens,
|
|
totalTokens: inputTokens + outputTokens,
|
|
};
|
|
};
|
|
|
|
const createV2TextStream = (
|
|
run: Promise<{ text: string; finishReason: "stop" | "error"; usage: { promptTokens?: number; completionTokens?: number } }>
|
|
) =>
|
|
new ReadableStream({
|
|
start(controller) {
|
|
controller.enqueue({ type: "stream-start", warnings: [] });
|
|
|
|
void run
|
|
.then((result) => {
|
|
if (result.text) {
|
|
const textId = `txt-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
controller.enqueue({ type: "text-start", id: textId });
|
|
controller.enqueue({
|
|
type: "text-delta",
|
|
id: textId,
|
|
delta: result.text,
|
|
});
|
|
controller.enqueue({ type: "text-end", id: textId });
|
|
}
|
|
|
|
controller.enqueue({
|
|
type: "finish",
|
|
finishReason: result.finishReason,
|
|
usage: toV2Usage(result.usage),
|
|
});
|
|
})
|
|
.catch((error) => {
|
|
controller.enqueue({ type: "error", error });
|
|
controller.enqueue({
|
|
type: "finish",
|
|
finishReason: "error",
|
|
usage: {
|
|
inputTokens: 0,
|
|
outputTokens: 0,
|
|
totalTokens: 0,
|
|
},
|
|
});
|
|
})
|
|
.finally(() => controller.close());
|
|
},
|
|
});
|
|
|
|
return {
|
|
provider: "volcano-multimodal",
|
|
chat: (modelId: string) => ({
|
|
provider: "volcano-multimodal",
|
|
modelId,
|
|
specificationVersion: "v2" as const,
|
|
supportedUrls: {},
|
|
defaultObjectGenerationMode: "json" as const,
|
|
doGenerate: async (options: any) => {
|
|
const { text, imageUrls } = extractPromptAndImages(options);
|
|
const result = await callResponsesApi({
|
|
endpoint,
|
|
apiKey: config.apiKey,
|
|
modelId,
|
|
text,
|
|
imageUrls,
|
|
timeout,
|
|
});
|
|
const legacyResult = {
|
|
text: result.text,
|
|
finishReason: "stop" as const,
|
|
usage: result.usage,
|
|
};
|
|
|
|
const isV2Call = Array.isArray(options?.prompt);
|
|
if (isV2Call) {
|
|
return {
|
|
content: [{ type: "text", text: result.text }],
|
|
finishReason: "stop" as const,
|
|
usage: toV2Usage(result.usage),
|
|
warnings: [],
|
|
};
|
|
}
|
|
|
|
return legacyResult;
|
|
},
|
|
doStream: function (this: any, options: any) {
|
|
const isV2Call = Array.isArray(options?.prompt);
|
|
|
|
if (!isV2Call) {
|
|
const legacyStream = async function* (this: any) {
|
|
const result = await this.doGenerate(options);
|
|
yield {
|
|
type: "text-delta" as const,
|
|
textDelta: result.text,
|
|
};
|
|
yield {
|
|
type: "finish" as const,
|
|
finishReason: "stop" as const,
|
|
usage: result.usage,
|
|
};
|
|
};
|
|
return legacyStream.call(this);
|
|
}
|
|
|
|
const run = (async () => {
|
|
try {
|
|
const result = await this.doGenerate(options);
|
|
const content = Array.isArray(result?.content) ? result.content : [];
|
|
const text = content
|
|
.filter((part: any) => part?.type === "text" && typeof part.text === "string")
|
|
.map((part: any) => part.text)
|
|
.join("");
|
|
return {
|
|
text,
|
|
finishReason: "stop" as const,
|
|
usage: {
|
|
promptTokens: result?.usage?.inputTokens ?? 0,
|
|
completionTokens: result?.usage?.outputTokens ?? 0,
|
|
},
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
text: error instanceof Error ? error.message : "请求失败",
|
|
finishReason: "error" as const,
|
|
usage: { promptTokens: 0, completionTokens: 0 },
|
|
};
|
|
}
|
|
})();
|
|
|
|
return Promise.resolve({
|
|
stream: createV2TextStream(run),
|
|
});
|
|
},
|
|
}),
|
|
};
|
|
}
|