1103 lines
37 KiB
TypeScript
1103 lines
37 KiB
TypeScript
import "server-only";
|
||
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import { getDefaultStorageConfig } from "@/lib/db/queries";
|
||
import { createLogger } from "@/lib/observability/logger";
|
||
|
||
const logger = createLogger("volcano-vision");
|
||
interface VolcanoVisionConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
type VolcanoImageSizeBucket = "1K" | "2K" | "4K";
|
||
|
||
type VolcanoUrlRewriteConfig = {
|
||
publicOrigin: string;
|
||
endpointBase: string;
|
||
bucket: string;
|
||
};
|
||
|
||
let cachedRewriteConfig: VolcanoUrlRewriteConfig | null | undefined;
|
||
const VOLCANO_IMAGE_FETCH_TIMEOUT_MS = 20_000;
|
||
const VOLCANO_INLINE_MAX_SIDE = 1024;
|
||
const DEFAULT_VOLCANO_IMAGE_SIZE = "2K";
|
||
const SEEDREAM_MODEL_ID_PATTERN = /doubao-seedream-4-5/i;
|
||
|
||
const VOLCANO_ASPECT_SIZE_OPTIONS: Record<
|
||
VolcanoImageSizeBucket,
|
||
Array<{ width: number; height: number }>
|
||
> = {
|
||
"1K": [
|
||
{ width: 1024, height: 1024 },
|
||
{ width: 1152, height: 864 },
|
||
{ width: 864, height: 1152 },
|
||
{ width: 1280, height: 720 },
|
||
{ width: 720, height: 1280 },
|
||
{ width: 1248, height: 832 },
|
||
{ width: 832, height: 1248 },
|
||
{ width: 1512, height: 648 },
|
||
],
|
||
"2K": [
|
||
{ width: 2048, height: 2048 },
|
||
{ width: 2304, height: 1728 },
|
||
{ width: 1728, height: 2304 },
|
||
{ width: 2560, height: 1440 },
|
||
{ width: 1440, height: 2560 },
|
||
{ width: 2496, height: 1664 },
|
||
{ width: 1664, height: 2496 },
|
||
{ width: 3024, height: 1296 },
|
||
],
|
||
"4K": [
|
||
{ width: 4096, height: 4096 },
|
||
{ width: 4704, height: 3520 },
|
||
{ width: 3520, height: 4704 },
|
||
{ width: 5504, height: 3040 },
|
||
{ width: 3040, height: 5504 },
|
||
{ width: 4992, height: 3328 },
|
||
{ width: 3328, height: 4992 },
|
||
{ width: 6240, height: 2656 },
|
||
],
|
||
};
|
||
|
||
function normalizeVolcanoImageSize(imageSize?: string): VolcanoImageSizeBucket {
|
||
const normalized = String(imageSize || "").trim().toUpperCase();
|
||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
|
||
return normalized;
|
||
}
|
||
return DEFAULT_VOLCANO_IMAGE_SIZE;
|
||
}
|
||
|
||
function resolveVolcanoImageSizeForModel(
|
||
modelId: string,
|
||
imageSize?: string
|
||
): VolcanoImageSizeBucket {
|
||
const exactSize = normalizeVolcanoExactSize(imageSize);
|
||
const normalized = exactSize
|
||
? inferVolcanoImageSizeBucketFromExactSize(exactSize)
|
||
: normalizeVolcanoImageSize(imageSize);
|
||
if (SEEDREAM_MODEL_ID_PATTERN.test(modelId)) {
|
||
// doubao-seedream-4-5 在当前链路下 1K 会返回 InvalidParameter，强制回落到 2K
|
||
if (normalized === "1K") {
|
||
logger.debug(
|
||
`[Volcano Vision] Size normalized for ${modelId}: 1K -> 2K (model constraint)`
|
||
);
|
||
return "2K";
|
||
}
|
||
}
|
||
return normalized;
|
||
}
|
||
|
||
function inferVolcanoImageSizeBucketFromExactSize(exactSize: string): VolcanoImageSizeBucket {
|
||
const [widthRaw, heightRaw] = exactSize.split("x");
|
||
const width = Number(widthRaw);
|
||
const height = Number(heightRaw);
|
||
const area = width * height;
|
||
if (area >= 7_000_000) return "4K";
|
||
if (area >= 2_000_000) return "2K";
|
||
return "1K";
|
||
}
|
||
|
||
function normalizeVolcanoExactSize(imageSize?: string): string | null {
|
||
const normalized = String(imageSize || "")
|
||
.trim()
|
||
.toLowerCase()
|
||
.replace(/[×*]/g, "x");
|
||
const match = normalized.match(/^(\d{2,5})x(\d{2,5})$/);
|
||
if (!match) return null;
|
||
|
||
const width = Number(match[1]);
|
||
const height = Number(match[2]);
|
||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
|
||
return null;
|
||
}
|
||
|
||
return `${width}x${height}`;
|
||
}
|
||
|
||
function parseVolcanoAspectRatio(aspectRatio?: string): number | null {
|
||
const normalized = String(aspectRatio || "").trim().toLowerCase();
|
||
if (!normalized || normalized === "auto") return null;
|
||
|
||
const match = normalized.match(/^(\d+(?:\.\d+)?)\s*:\s*(\d+(?:\.\d+)?)$/);
|
||
if (!match) return null;
|
||
|
||
const width = Number(match[1]);
|
||
const height = Number(match[2]);
|
||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
|
||
return null;
|
||
}
|
||
|
||
return width / height;
|
||
}
|
||
|
||
function resolveVolcanoAspectAwareSize(
|
||
aspectRatio: string | undefined,
|
||
imageSize: VolcanoImageSizeBucket
|
||
): string | null {
|
||
const targetRatio = parseVolcanoAspectRatio(aspectRatio);
|
||
if (!targetRatio) return null;
|
||
|
||
const candidates = VOLCANO_ASPECT_SIZE_OPTIONS[imageSize];
|
||
const best = candidates.reduce((currentBest, option) => {
|
||
const currentDelta = Math.abs(currentBest.width / currentBest.height - targetRatio);
|
||
const optionDelta = Math.abs(option.width / option.height - targetRatio);
|
||
if (optionDelta < currentDelta) {
|
||
return option;
|
||
}
|
||
if (optionDelta === currentDelta) {
|
||
return option.width * option.height > currentBest.width * currentBest.height
|
||
? option
|
||
: currentBest;
|
||
}
|
||
return currentBest;
|
||
}, candidates[0]);
|
||
|
||
return `${best.width}x${best.height}`;
|
||
}
|
||
|
||
function getFirstImageSource(image: string | string[] | undefined): string | null {
|
||
if (!image) return null;
|
||
if (Array.isArray(image)) {
|
||
return image.find((item) => typeof item === "string" && item.length > 0) || null;
|
||
}
|
||
return typeof image === "string" && image.length > 0 ? image : null;
|
||
}
|
||
|
||
async function inferImageAspectRatioFromSource(
|
||
image: string | string[] | undefined
|
||
): Promise<string | null> {
|
||
const source = getFirstImageSource(image);
|
||
if (!source) return null;
|
||
|
||
try {
|
||
let buffer: Buffer | null = null;
|
||
|
||
if (source.startsWith("data:image")) {
|
||
const match = source.match(/^data:[^;]+;base64,(.+)$/);
|
||
if (match?.[1]) {
|
||
buffer = Buffer.from(match[1], "base64");
|
||
}
|
||
} else if (/^https?:\/\//i.test(source)) {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), VOLCANO_IMAGE_FETCH_TIMEOUT_MS);
|
||
try {
|
||
const response = await fetch(source, { signal: controller.signal });
|
||
if (!response.ok) {
|
||
logger.warn("[Volcano Vision] Failed to fetch reference image for aspect inference:", response.status);
|
||
return null;
|
||
}
|
||
buffer = Buffer.from(await response.arrayBuffer());
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
if (!buffer || buffer.length === 0) return null;
|
||
|
||
const sharp = await import("sharp");
|
||
const metadata = await sharp.default(buffer).metadata();
|
||
if (!metadata.width || !metadata.height) return null;
|
||
|
||
return `${metadata.width}:${metadata.height}`;
|
||
} catch (error) {
|
||
logger.warn("[Volcano Vision] Failed to infer reference aspect ratio:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function resolveVolcanoRequestSize(params: {
|
||
modelId: string;
|
||
imageSize?: string;
|
||
aspectRatio?: string;
|
||
image?: string | string[];
|
||
}): Promise<{
|
||
size: string;
|
||
fallbackSize: VolcanoImageSizeBucket;
|
||
inferredAspectRatio: string | null;
|
||
}> {
|
||
const fallbackSize = resolveVolcanoImageSizeForModel(params.modelId, params.imageSize);
|
||
const explicitSize = normalizeVolcanoExactSize(params.imageSize);
|
||
if (explicitSize) {
|
||
return { size: explicitSize, fallbackSize, inferredAspectRatio: null };
|
||
}
|
||
|
||
const requestedAspectRatio = String(params.aspectRatio || "").trim();
|
||
const inferredAspectRatio =
|
||
requestedAspectRatio === "auto" && params.image
|
||
? await inferImageAspectRatioFromSource(params.image)
|
||
: null;
|
||
const resolvedAspectRatio = inferredAspectRatio || requestedAspectRatio;
|
||
const aspectAwareSize = resolveVolcanoAspectAwareSize(resolvedAspectRatio, fallbackSize);
|
||
|
||
if (inferredAspectRatio) {
|
||
logger.debug(`[Volcano Vision] Inferred reference aspect ratio: ${inferredAspectRatio}`);
|
||
}
|
||
|
||
if (aspectAwareSize) {
|
||
logger.debug(
|
||
`[Volcano Vision] Resolved size: aspectRatio=${resolvedAspectRatio}, imageSize=${fallbackSize} => ${aspectAwareSize}`
|
||
);
|
||
return { size: aspectAwareSize, fallbackSize, inferredAspectRatio };
|
||
}
|
||
|
||
logger.debug(
|
||
`[Volcano Vision] Resolved size: aspectRatio=${resolvedAspectRatio || "default"}, imageSize=${fallbackSize} => ${fallbackSize}`
|
||
);
|
||
return { size: fallbackSize, fallbackSize, inferredAspectRatio };
|
||
}
|
||
|
||
function guessImageMimeTypeFromUrl(url: string): string {
|
||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||
if (cleanUrl.endsWith(".png")) return "image/png";
|
||
if (cleanUrl.endsWith(".webp")) return "image/webp";
|
||
if (cleanUrl.endsWith(".gif")) return "image/gif";
|
||
if (cleanUrl.endsWith(".bmp")) return "image/bmp";
|
||
return "image/jpeg";
|
||
}
|
||
|
||
function shouldRetryVolcanoWithInlineImage(errorText: string): boolean {
|
||
if (!errorText) return false;
|
||
return /timeout while downloading url|download multimodal file timed out|invalidparameter/i.test(
|
||
errorText
|
||
);
|
||
}
|
||
|
||
function shouldRetryVolcanoWithBucketSize(errorText: string): boolean {
|
||
if (!errorText) return false;
|
||
return /invalidparameter|invalid parameter|size|resolution|dimension/i.test(errorText);
|
||
}
|
||
|
||
async function fetchImageAsDataUri(url: string): Promise<string | null> {
|
||
if (!/^https?:\/\//i.test(url)) return null;
|
||
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), VOLCANO_IMAGE_FETCH_TIMEOUT_MS);
|
||
|
||
try {
|
||
const response = await fetch(url, { signal: controller.signal });
|
||
if (!response.ok) {
|
||
logger.warn("[Volcano Vision] Failed to fetch source image for inline retry:", response.status, url);
|
||
return null;
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
if (buffer.length === 0) return null;
|
||
|
||
const inputMime = response.headers.get("content-type") || guessImageMimeTypeFromUrl(url);
|
||
|
||
// 优先保留原图，避免 base64 回退路径带来明显画质损失
|
||
if (buffer.length <= 6 * 1024 * 1024) {
|
||
return `data:${inputMime};base64,${buffer.toString("base64")}`;
|
||
}
|
||
|
||
try {
|
||
const sharp = await import("sharp");
|
||
const optimized = await sharp.default(buffer)
|
||
.rotate()
|
||
.resize(VOLCANO_INLINE_MAX_SIDE, VOLCANO_INLINE_MAX_SIDE, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 90, mozjpeg: true })
|
||
.toBuffer();
|
||
return `data:image/jpeg;base64,${optimized.toString("base64")}`;
|
||
} catch {
|
||
return `data:${inputMime};base64,${buffer.toString("base64")}`;
|
||
}
|
||
} catch (error) {
|
||
logger.warn("[Volcano Vision] Inline image fetch failed:", error);
|
||
return null;
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
async function convertImageFieldToInlineDataUri(
|
||
image: string | string[] | undefined
|
||
): Promise<string | string[] | undefined> {
|
||
if (!image) return image;
|
||
const list = Array.isArray(image) ? image : [image];
|
||
const converted = await Promise.all(
|
||
list.map(async (item) => {
|
||
if (typeof item !== "string") return null;
|
||
if (item.startsWith("data:image")) return item;
|
||
return fetchImageAsDataUri(item);
|
||
})
|
||
);
|
||
|
||
const ready = converted.filter((item): item is string => typeof item === "string" && item.length > 0);
|
||
if (ready.length === 0) return undefined;
|
||
return Array.isArray(image) ? ready : ready[0];
|
||
}
|
||
|
||
async function getVolcanoUrlRewriteConfig(): Promise<VolcanoUrlRewriteConfig | null> {
|
||
if (cachedRewriteConfig !== undefined) {
|
||
return cachedRewriteConfig;
|
||
}
|
||
|
||
try {
|
||
const storageConfig = await getDefaultStorageConfig();
|
||
if (
|
||
!storageConfig ||
|
||
storageConfig.type !== "s3" ||
|
||
!storageConfig.endpoint ||
|
||
!storageConfig.bucket ||
|
||
!storageConfig.publicUrl
|
||
) {
|
||
cachedRewriteConfig = null;
|
||
return null;
|
||
}
|
||
|
||
const publicOrigin = new URL(storageConfig.publicUrl).origin;
|
||
const endpointBase = storageConfig.endpoint.replace(/\/+$/, "");
|
||
|
||
cachedRewriteConfig = {
|
||
publicOrigin,
|
||
endpointBase,
|
||
bucket: storageConfig.bucket,
|
||
};
|
||
return cachedRewriteConfig;
|
||
} catch {
|
||
cachedRewriteConfig = null;
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function rewriteUploadPublicUrlForVolcano(url: string): Promise<string> {
|
||
if (!/^https?:\/\//i.test(url)) return url;
|
||
|
||
const config = await getVolcanoUrlRewriteConfig();
|
||
if (!config) return url;
|
||
|
||
try {
|
||
const parsed = new URL(url);
|
||
if (parsed.origin !== config.publicOrigin) return url;
|
||
if (!parsed.pathname.startsWith("/uploads/")) return url;
|
||
|
||
const rewritten = `${config.endpointBase}/${config.bucket}${parsed.pathname}${parsed.search}`;
|
||
logger.debug("[Volcano Vision] Rewrote upload URL for provider fetch:", rewritten);
|
||
return rewritten;
|
||
} catch {
|
||
return url;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 压缩图片生成缩略图（使用 sharp）
|
||
*/
|
||
async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer> {
|
||
try {
|
||
const sharp = await import("sharp");
|
||
|
||
// 创建缩略图（宽度 500px，高度按比例，质量 80%）
|
||
const thumbnailBuffer = await sharp.default(buffer)
|
||
.resize(500, null, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
|
||
logger.debug(`[Volcano Vision] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`);
|
||
return thumbnailBuffer;
|
||
} catch (error) {
|
||
logger.error("[Volcano Vision] Failed to create thumbnail:", error);
|
||
// 如果失败，返回原图
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 将图片 URL 或 base64 上传到 S3，返回缩略图和原图的 URL
|
||
*/
|
||
async function uploadImageToS3(imageData: string): Promise<{ thumbnail: string; original: string }> {
|
||
try {
|
||
let buffer: Buffer;
|
||
let mimeType = "image/jpeg";
|
||
|
||
// 检查是否是 base64
|
||
if (imageData.startsWith("data:")) {
|
||
const matches = imageData.match(/^data:([^;]+);base64,(.+)$/);
|
||
if (matches) {
|
||
mimeType = matches[1];
|
||
buffer = Buffer.from(matches[2], "base64");
|
||
} else {
|
||
throw new Error("Invalid base64 format");
|
||
}
|
||
} else {
|
||
// 是 URL，下载图片
|
||
logger.debug(`[Volcano Vision] Downloading image from URL...`);
|
||
const response = await fetch(imageData);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to download image: ${response.statusText}`);
|
||
}
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
buffer = Buffer.from(arrayBuffer);
|
||
mimeType = response.headers.get("content-type") || "image/jpeg";
|
||
}
|
||
|
||
// 生成文件名
|
||
const ext = mimeType.split("/")[1] || "jpg";
|
||
const timestamp = Date.now();
|
||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||
const thumbnailFilename = `volcano-thumb-${timestamp}-${randomStr}.${ext}`;
|
||
const originalFilename = `volcano-orig-${timestamp}-${randomStr}.${ext}`;
|
||
|
||
logger.debug(`[Volcano Vision] Original size: ${buffer.length} bytes`);
|
||
|
||
// 生成缩略图
|
||
const thumbnailBuffer = await createThumbnail(buffer, mimeType);
|
||
|
||
// 转换为 ArrayBuffer
|
||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||
thumbnailBuffer.byteOffset,
|
||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||
) as ArrayBuffer;
|
||
const originalArrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
|
||
// 并行上传缩略图和原图到 S3
|
||
logger.debug(`[Volcano Vision] Uploading thumbnail and original to S3...`);
|
||
const [thumbnailResult, originalResult] = await Promise.all([
|
||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||
uploadFile(originalFilename, originalArrayBuffer),
|
||
]);
|
||
|
||
logger.debug(`[Volcano Vision] Uploaded to S3:`);
|
||
logger.debug(` - Thumbnail: ${thumbnailResult.url}`);
|
||
logger.debug(` - Original: ${originalResult.url}`);
|
||
|
||
return {
|
||
thumbnail: thumbnailResult.url,
|
||
original: originalResult.url,
|
||
};
|
||
} catch (error) {
|
||
logger.error("[Volcano Vision] Failed to upload to S3:", error);
|
||
throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 火山引擎 API 响应格式
|
||
*/
|
||
interface VolcanoVisionResponse {
|
||
model: string;
|
||
created: number;
|
||
data: Array<{
|
||
url?: string;
|
||
b64_json?: string;
|
||
size: string;
|
||
}>;
|
||
usage: {
|
||
generated_images: number;
|
||
output_tokens: number;
|
||
total_tokens: number;
|
||
};
|
||
error?: {
|
||
code: string;
|
||
message: string;
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 火山引擎流式响应事件
|
||
*/
|
||
interface VolcanoVisionStreamEvent {
|
||
type: "image_generation.partial_succeeded" | "image_generation.completed";
|
||
model: string;
|
||
created: number;
|
||
image_index?: number;
|
||
url?: string;
|
||
size?: string;
|
||
usage?: {
|
||
generated_images: number;
|
||
output_tokens: number;
|
||
total_tokens: number;
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 从 API 响应中提取图片并上传到 S3
|
||
*/
|
||
async function extractAndUploadImages(response: VolcanoVisionResponse): Promise<Array<{ thumbnail: string; original: string }>> {
|
||
const images: Array<{ thumbnail: string; original: string }> = [];
|
||
const uploadPromises: Promise<void>[] = [];
|
||
|
||
if (response.data && response.data.length > 0) {
|
||
for (const item of response.data) {
|
||
if (item.url) {
|
||
// URL 格式
|
||
const uploadPromise = uploadImageToS3(item.url)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
} else if (item.b64_json) {
|
||
// Base64 格式
|
||
const base64Data = `data:image/jpeg;base64,${item.b64_json}`;
|
||
const uploadPromise = uploadImageToS3(base64Data)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 等待所有上传完成
|
||
await Promise.all(uploadPromises);
|
||
|
||
logger.debug("[Volcano Vision] Extracted and uploaded", images.length, "images (thumbnail + original)");
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* 将图片数据转换为 Markdown 格式
|
||
* 使用特殊格式传递缩略图和原图 URL：
|
||
*/
|
||
function formatImageResponse(
|
||
images: Array<{ thumbnail: string; original: string }>,
|
||
imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE
|
||
): string {
|
||
if (images.length === 0) {
|
||
return "生成图片失败，未找到图片数据";
|
||
}
|
||
|
||
// 🆕 统一使用 ||| 分隔符格式（与 Gemini 保持一致）
|
||
// 格式: 
|
||
const normalizedSize = normalizeVolcanoExactSize(imageSize) || normalizeVolcanoImageSize(imageSize);
|
||
return images
|
||
.map((img) => ``)
|
||
.join("\n\n");
|
||
}
|
||
|
||
/**
|
||
* 将消息转换为火山引擎 API 格式
|
||
*/
|
||
async function convertMessagesToVolcanoFormat(messages: any[]): Promise<{ prompt: string; image?: string | string[] }> {
|
||
const lastMessage = messages[messages.length - 1];
|
||
let prompt = "";
|
||
const images: string[] = [];
|
||
|
||
logger.debug("[Volcano Vision] Converting messages, last message:", JSON.stringify(lastMessage).substring(0, 500));
|
||
|
||
// 提取文本和图片
|
||
if (lastMessage.content) {
|
||
if (typeof lastMessage.content === "string") {
|
||
prompt = lastMessage.content;
|
||
} else if (Array.isArray(lastMessage.content)) {
|
||
for (const part of lastMessage.content) {
|
||
logger.debug("[Volcano Vision] Processing part:", JSON.stringify(part).substring(0, 150));
|
||
|
||
if (part.type === "text") {
|
||
prompt += part.text;
|
||
} else if (part.type === "image_url") {
|
||
// OpenAI 格式：{ type: "image_url", image_url: { url: "..." } }
|
||
if (part.image_url && part.image_url.url) {
|
||
images.push(part.image_url.url);
|
||
logger.debug("[Volcano Vision] ✅ Found image_url:", part.image_url.url.substring(0, 100) + "...");
|
||
}
|
||
} else if (part.type === "file") {
|
||
// 文件格式：{ type: "file", mediaType: "image/...", data: "url" }
|
||
logger.debug("[Volcano Vision] File part - mediaType:", part.mediaType, "data exists:", !!part.data);
|
||
if (part.data && part.mediaType && part.mediaType.startsWith("image/")) {
|
||
images.push(part.data);
|
||
logger.debug("[Volcano Vision] ✅ Found file (image):", part.data.substring(0, 100) + "...");
|
||
} else {
|
||
logger.debug("[Volcano Vision] ❌ File part skipped - mediaType:", part.mediaType, "data:", !!part.data);
|
||
}
|
||
} else if (part.type === "image") {
|
||
// 其他格式：{ type: "image", image: "..." } 或 { type: "image", url: "..." }
|
||
if (part.image) {
|
||
images.push(part.image);
|
||
logger.debug("[Volcano Vision] ✅ Found image:", part.image.substring(0, 100) + "...");
|
||
} else if (part.url) {
|
||
images.push(part.url);
|
||
logger.debug("[Volcano Vision] ✅ Found image url:", part.url.substring(0, 100) + "...");
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
logger.debug("[Volcano Vision] Extracted prompt:", prompt.substring(0, 100));
|
||
logger.debug("[Volcano Vision] Extracted images count:", images.length);
|
||
|
||
// 🆕 直接使用图片URL，不再转换Base64（既梦API支持URL格式）
|
||
// 只有当图片是Base64格式时才保持原样
|
||
const processedImages: string[] = [];
|
||
for (const imageUrl of images) {
|
||
if (imageUrl.startsWith('data:image')) {
|
||
// Base64格式：需要先上传到S3获取URL
|
||
logger.debug("[Volcano Vision] Found Base64 image, uploading to S3...");
|
||
const uploaded = await uploadImageToS3(imageUrl);
|
||
const rewritten = await rewriteUploadPublicUrlForVolcano(uploaded.original);
|
||
processedImages.push(rewritten);
|
||
} else {
|
||
// URL格式：直接使用
|
||
const rewritten = await rewriteUploadPublicUrlForVolcano(imageUrl);
|
||
processedImages.push(rewritten);
|
||
logger.debug("[Volcano Vision] Using image URL directly:", rewritten.substring(0, 80) + "...");
|
||
}
|
||
}
|
||
|
||
// 不在这里添加"生成多个变体"的提示，由调用方根据maxImages决定
|
||
// 这样可以避免在场景裂变等只需要1张图片的场景中生成多张图片
|
||
const enhancedPrompt = prompt;
|
||
logger.debug("[Volcano Vision] Prompt:", enhancedPrompt.substring(0, 150));
|
||
|
||
// 构建请求体
|
||
const result: { prompt: string; image?: string | string[] } = { prompt: enhancedPrompt };
|
||
|
||
if (processedImages.length === 1) {
|
||
result.image = processedImages[0];
|
||
} else if (processedImages.length > 1) {
|
||
result.image = processedImages; // 多图直接传数组
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* 发送火山引擎图像生成请求（非流式）
|
||
*/
|
||
async function sendVolcanoRequest(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
messages: any[],
|
||
modelId: string,
|
||
maxImages: number = 3, // 默认 3 张，故障转移时传 1
|
||
imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE,
|
||
aspectRatio?: string
|
||
): Promise<VolcanoVisionResponse> {
|
||
const url = `${baseURL}/api/v3/images/generations`;
|
||
|
||
logger.debug("[Volcano Vision] Sending non-stream request to:", url);
|
||
logger.debug("[Volcano Vision] Using model:", modelId);
|
||
|
||
// 转换消息格式（直接使用URL）
|
||
let { prompt, image } = await convertMessagesToVolcanoFormat(messages);
|
||
|
||
// 根据 maxImages 参数优化提示词
|
||
if (maxImages === 1) {
|
||
// 只生成1张图片时，明确指示只生成1张，避免生成多张
|
||
if (!prompt.includes("只生成") && !prompt.includes("生成1张") && !prompt.includes("生成一张")) {
|
||
prompt = prompt.trim();
|
||
// 移除末尾的句号（如果有），然后添加明确指令
|
||
if (prompt.endsWith("。") || prompt.endsWith(".")) {
|
||
prompt = prompt.slice(0, -1);
|
||
}
|
||
prompt += "。只生成1张图片。";
|
||
}
|
||
} else if (maxImages > 1) {
|
||
// 生成多张图片时，明确要求生成多个变体
|
||
if (!prompt.includes("生成") || (!prompt.includes("变体") && !prompt.includes("不同"))) {
|
||
prompt = prompt.trim();
|
||
if (prompt.endsWith("。") || prompt.endsWith(".")) {
|
||
prompt = prompt.slice(0, -1);
|
||
}
|
||
prompt += `。请生成 ${maxImages} 个不同的变体。`;
|
||
}
|
||
}
|
||
|
||
logger.debug("[Volcano Vision] Prompt:", prompt.substring(0, 100) + "...");
|
||
logger.debug("[Volcano Vision] Image count:", image ? (Array.isArray(image) ? image.length : 1) : 0);
|
||
|
||
// 构建请求体
|
||
const resolvedSize = await resolveVolcanoRequestSize({
|
||
modelId,
|
||
imageSize,
|
||
aspectRatio,
|
||
image,
|
||
});
|
||
const requestBody: any = {
|
||
model: modelId,
|
||
prompt,
|
||
size: resolvedSize.size,
|
||
watermark: false,
|
||
response_format: "url",
|
||
stream: false,
|
||
};
|
||
|
||
// 添加图片（如果有）
|
||
if (image) {
|
||
requestBody.image = image;
|
||
}
|
||
|
||
// 根据 maxImages 参数配置组图功能
|
||
if (maxImages > 1) {
|
||
requestBody.sequential_image_generation = "auto";
|
||
requestBody.sequential_image_generation_options = {
|
||
max_images: maxImages,
|
||
};
|
||
logger.debug(`[Volcano Vision] Generating up to ${maxImages} images (mode: ${image ? "image-to-image" : "text-to-image"})`);
|
||
} else {
|
||
requestBody.sequential_image_generation = "disabled";
|
||
logger.debug(`[Volcano Vision] Generating 1 image (mode: ${image ? "image-to-image" : "text-to-image"})`);
|
||
}
|
||
|
||
logger.debug("[Volcano Vision] Request body:", JSON.stringify(requestBody).substring(0, 500) + "...");
|
||
logger.debug("[Volcano Vision] Full request body (prompt + image info):");
|
||
logger.debug(" - Prompt:", requestBody.prompt);
|
||
logger.debug(" - Image field exists:", !!requestBody.image);
|
||
logger.debug(" - Image type:", typeof requestBody.image);
|
||
logger.debug(" - Image is array:", Array.isArray(requestBody.image));
|
||
logger.debug(" - Image count:", Array.isArray(requestBody.image) ? requestBody.image.length : (requestBody.image ? 1 : 0));
|
||
if (requestBody.image) {
|
||
if (Array.isArray(requestBody.image)) {
|
||
requestBody.image.forEach((img: string, idx: number) => {
|
||
logger.debug(` - Image ${idx}: ${img.substring(0, 100)}...`);
|
||
});
|
||
} else {
|
||
logger.debug(` - Image: ${requestBody.image.substring(0, 100)}...`);
|
||
}
|
||
}
|
||
|
||
const executeRequest = async (payload: any, tag: string): Promise<VolcanoVisionResponse> => {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 600000);
|
||
|
||
try {
|
||
const response = await fetch(url, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(payload),
|
||
signal: controller.signal,
|
||
});
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text();
|
||
logger.error(`[Volcano Vision] Request failed (${tag}):`, response.status, errorText);
|
||
const err = new Error(
|
||
`Volcano Vision API error: ${response.status} ${response.statusText} - ${errorText}`
|
||
) as Error & { status?: number; body?: string };
|
||
err.status = response.status;
|
||
err.body = errorText;
|
||
throw err;
|
||
}
|
||
|
||
const data = await response.json();
|
||
logger.debug(
|
||
`[Volcano Vision] Response received (${tag}):`,
|
||
JSON.stringify(data).substring(0, 200) + "..."
|
||
);
|
||
|
||
if (data.error) {
|
||
throw new Error(`API Error: ${data.error.message || data.error.code}`);
|
||
}
|
||
|
||
return data as VolcanoVisionResponse;
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
};
|
||
|
||
try {
|
||
return await executeRequest(requestBody, "primary");
|
||
} catch (error) {
|
||
if (error instanceof Error && error.name === "AbortError") {
|
||
logger.error("[Volcano Vision] Request timeout after 10 minutes");
|
||
throw new Error("请求超时，图片处理时间过长，请稍后重试");
|
||
}
|
||
|
||
const errorBody = (error as any)?.body || String(error || "");
|
||
let inlineRetryBody = requestBody;
|
||
let latestError: unknown = error;
|
||
let latestErrorBody = errorBody;
|
||
|
||
if (
|
||
requestBody.size !== resolvedSize.fallbackSize &&
|
||
shouldRetryVolcanoWithBucketSize(errorBody)
|
||
) {
|
||
logger.warn(
|
||
`[Volcano Vision] Retrying with bucket size fallback: ${requestBody.size} -> ${resolvedSize.fallbackSize}`
|
||
);
|
||
const sizeFallbackBody = { ...requestBody, size: resolvedSize.fallbackSize };
|
||
try {
|
||
return await executeRequest(sizeFallbackBody, "size-fallback");
|
||
} catch (sizeError) {
|
||
latestError = sizeError;
|
||
latestErrorBody = (sizeError as any)?.body || String(sizeError || "");
|
||
inlineRetryBody = sizeFallbackBody;
|
||
}
|
||
}
|
||
|
||
if (inlineRetryBody.image && shouldRetryVolcanoWithInlineImage(latestErrorBody)) {
|
||
logger.warn("[Volcano Vision] Retrying with inline base64 image payload...");
|
||
const inlineImage = await convertImageFieldToInlineDataUri(inlineRetryBody.image);
|
||
if (inlineImage) {
|
||
const retryBody = { ...inlineRetryBody, image: inlineImage };
|
||
return await executeRequest(retryBody, "inline-retry");
|
||
}
|
||
logger.warn("[Volcano Vision] Inline retry skipped: failed to convert source image(s).");
|
||
}
|
||
throw latestError;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 发送火山引擎图像生成请求（流式）
|
||
*/
|
||
async function* sendVolcanoStreamRequest(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
messages: any[],
|
||
modelId: string,
|
||
imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE,
|
||
aspectRatio?: string
|
||
): AsyncGenerator<VolcanoVisionStreamEvent> {
|
||
const url = `${baseURL}/api/v3/images/generations`;
|
||
|
||
logger.debug("[Volcano Vision] Sending stream request to:", url);
|
||
logger.debug("[Volcano Vision] Using model:", modelId);
|
||
|
||
// 转换消息格式（直接使用URL）
|
||
const { prompt, image } = await convertMessagesToVolcanoFormat(messages);
|
||
|
||
logger.debug("[Volcano Vision] Prompt:", prompt.substring(0, 100) + "...");
|
||
logger.debug("[Volcano Vision] Image count:", image ? (Array.isArray(image) ? image.length : 1) : 0);
|
||
|
||
// 构建请求体
|
||
const resolvedSize = await resolveVolcanoRequestSize({
|
||
modelId,
|
||
imageSize,
|
||
aspectRatio,
|
||
image,
|
||
});
|
||
const requestBody: any = {
|
||
model: modelId,
|
||
prompt,
|
||
size: resolvedSize.size,
|
||
watermark: false,
|
||
response_format: "url",
|
||
stream: true,
|
||
};
|
||
|
||
// 添加图片（如果有）
|
||
if (image) {
|
||
requestBody.image = image;
|
||
}
|
||
|
||
logger.debug("[Volcano Vision] Stream request body:", JSON.stringify(requestBody).substring(0, 500) + "...");
|
||
logger.debug("[Volcano Vision] Stream - Full request info:");
|
||
logger.debug(" - Prompt:", requestBody.prompt);
|
||
logger.debug(" - Image field exists:", !!requestBody.image);
|
||
logger.debug(" - Image type:", typeof requestBody.image);
|
||
logger.debug(" - Image is array:", Array.isArray(requestBody.image));
|
||
logger.debug(" - Image count:", Array.isArray(requestBody.image) ? requestBody.image.length : (requestBody.image ? 1 : 0));
|
||
|
||
// 设置 10 分钟超时
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 600000);
|
||
|
||
try {
|
||
const response = await fetch(url, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
signal: controller.signal,
|
||
});
|
||
|
||
clearTimeout(timeoutId);
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text();
|
||
logger.error("[Volcano Vision] Stream request failed:", response.status, errorText);
|
||
throw new Error(
|
||
`Volcano Vision API error: ${response.status} ${response.statusText}`
|
||
);
|
||
}
|
||
|
||
// 解析 SSE 流
|
||
const reader = response.body?.getReader();
|
||
if (!reader) {
|
||
throw new Error("Failed to get response reader");
|
||
}
|
||
|
||
const decoder = new TextDecoder();
|
||
let buffer = "";
|
||
|
||
while (true) {
|
||
const { done, value } = await reader.read();
|
||
if (done) break;
|
||
|
||
buffer += decoder.decode(value, { stream: true });
|
||
const lines = buffer.split("\n");
|
||
buffer = lines.pop() || "";
|
||
|
||
for (const line of lines) {
|
||
const trimmedLine = line.trim();
|
||
if (!trimmedLine || trimmedLine === "data: [DONE]") continue;
|
||
|
||
if (trimmedLine.startsWith("data: ")) {
|
||
const jsonStr = trimmedLine.slice(6);
|
||
try {
|
||
const event = JSON.parse(jsonStr) as VolcanoVisionStreamEvent;
|
||
logger.debug("[Volcano Vision] Stream event:", event.type, event.image_index);
|
||
yield event;
|
||
} catch (error) {
|
||
logger.error("[Volcano Vision] Failed to parse stream event:", jsonStr);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch (error) {
|
||
clearTimeout(timeoutId);
|
||
if (error instanceof Error && error.name === "AbortError") {
|
||
logger.error("[Volcano Vision] Stream request timeout after 10 minutes");
|
||
throw new Error("请求超时，图片处理时间过长，请稍后重试");
|
||
}
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 创建火山引擎视觉 Provider
|
||
*/
|
||
export function createVolcanoVision({ apiKey, baseURL }: VolcanoVisionConfig) {
|
||
return {
|
||
chat: (modelId: string) => ({
|
||
doGenerate: async (options: any) => {
|
||
logger.debug("[Volcano Vision] doGenerate called!");
|
||
logger.debug("[Volcano Vision] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "...");
|
||
const activeModelId = options.overrideModelId || modelId;
|
||
logger.debug("[Volcano Vision] Using modelId:", activeModelId);
|
||
logger.debug("[Volcano Vision] Image params from options:", options.imageParams);
|
||
const imageSize = resolveVolcanoImageSizeForModel(
|
||
activeModelId,
|
||
options.imageParams?.imageSize
|
||
);
|
||
|
||
// 🆕 优先使用用户选择的图片数量，其次是 maxImages 参数，默认 1 张
|
||
const imageCount = options.imageParams?.imageCount || options.maxImages || 1;
|
||
const maxImages = Math.min(Math.max(imageCount, 1), 3); // 限制 1-3 张
|
||
logger.debug("[Volcano Vision] Max images:", maxImages);
|
||
|
||
try {
|
||
// 发送请求
|
||
const response = await sendVolcanoRequest(
|
||
baseURL,
|
||
apiKey,
|
||
options.messages,
|
||
activeModelId,
|
||
maxImages,
|
||
imageSize,
|
||
options.imageParams?.aspectRatio
|
||
);
|
||
|
||
// 提取图片并上传到 S3
|
||
const imageUrls = await extractAndUploadImages(response);
|
||
const responseImageSize = response.data?.find((item) => item.size)?.size || imageSize;
|
||
const markdownText = formatImageResponse(imageUrls, responseImageSize);
|
||
|
||
logger.debug(
|
||
`[Volcano Vision] Generated ${imageUrls.length} images (uploaded to S3)`
|
||
);
|
||
|
||
// 返回 AI SDK v1 格式
|
||
return {
|
||
text: markdownText,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
logger.error("[Volcano Vision] Generation failed:", error);
|
||
|
||
// 返回错误信息
|
||
const errorMessage =
|
||
error instanceof Error
|
||
? `图片生成失败: ${error.message}`
|
||
: "图片生成失败，请稍后重试";
|
||
|
||
return {
|
||
text: errorMessage,
|
||
finishReason: "error" as const,
|
||
usage: { promptTokens: 0, completionTokens: 0 },
|
||
};
|
||
}
|
||
},
|
||
|
||
doStream: async function* (options: any) {
|
||
logger.debug("[Volcano Vision] ===== doStream ENTERED! =====");
|
||
logger.debug("[Volcano Vision] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "...");
|
||
logger.debug("[Volcano Vision] Using modelId:", modelId);
|
||
|
||
try {
|
||
const allImages: Array<{ thumbnail: string; original: string }> = [];
|
||
|
||
// 使用流式请求
|
||
const imageSize = resolveVolcanoImageSizeForModel(
|
||
modelId,
|
||
options.imageParams?.imageSize
|
||
);
|
||
for await (const event of sendVolcanoStreamRequest(
|
||
baseURL,
|
||
apiKey,
|
||
options.messages,
|
||
modelId,
|
||
imageSize,
|
||
options.imageParams?.aspectRatio
|
||
)) {
|
||
if (event.type === "image_generation.partial_succeeded" && event.url) {
|
||
// 每张图片生成完成后立即上传并发送
|
||
try {
|
||
const imageUrls = await uploadImageToS3(event.url);
|
||
allImages.push(imageUrls);
|
||
|
||
const markdown = ``;
|
||
logger.debug(`[Volcano Vision Stream] Image ${event.image_index} uploaded, sending...`);
|
||
|
||
yield { type: "text-delta" as const, textDelta: markdown + "\n\n" };
|
||
} catch (error) {
|
||
logger.error(`[Volcano Vision Stream] Failed to upload image ${event.image_index}:`, error);
|
||
}
|
||
} else if (event.type === "image_generation.completed") {
|
||
logger.debug(`[Volcano Vision Stream] Generation completed, total images: ${allImages.length}`);
|
||
|
||
// 发送完成信号
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: allImages.length * 50, // 估算
|
||
},
|
||
};
|
||
}
|
||
}
|
||
} catch (error) {
|
||
logger.error("[Volcano Vision Stream] Generation failed:", error);
|
||
|
||
const errorMessage =
|
||
error instanceof Error
|
||
? `图片生成失败: ${error.message}`
|
||
: "图片生成失败，请稍后重试";
|
||
|
||
// 发送错误信息
|
||
yield { type: "text-delta" as const, textDelta: errorMessage };
|
||
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "error" as const,
|
||
usage: { promptTokens: 0, completionTokens: 0 },
|
||
};
|
||
}
|
||
},
|
||
|
||
modelId,
|
||
provider: "volcano-vision",
|
||
specificationVersion: "v1" as const,
|
||
defaultObjectGenerationMode: "json" as const,
|
||
}),
|
||
};
|
||
}
|