531 lines
17 KiB
TypeScript
531 lines
17 KiB
TypeScript
import "server-only";
|
||
|
||
import { randomUUID } from "node:crypto";
|
||
import sharp from "sharp";
|
||
import {
|
||
getVisionaryModelPresetById,
|
||
VISIONARY_BASE_URL,
|
||
} from "@/lib/ai/visionary-models";
|
||
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
interface VisionaryConfig {
|
||
apiKey: string;
|
||
baseURL?: string;
|
||
}
|
||
|
||
interface VisionaryImageParams {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
promptExtend?: boolean;
|
||
quality?: "auto" | "low" | "medium" | "high" | string;
|
||
}
|
||
|
||
interface VisionaryModelConfig {
|
||
maxReferenceImages?: number;
|
||
supportedAspectRatios?: string[];
|
||
supportedImageSizes?: string[];
|
||
supportsPromptExtend?: boolean;
|
||
supportsQuality?: boolean;
|
||
}
|
||
|
||
interface ReferenceImage {
|
||
apiValue: string;
|
||
height?: number;
|
||
width?: number;
|
||
}
|
||
|
||
interface UploadedImage {
|
||
thumbnail: string;
|
||
original: string;
|
||
}
|
||
|
||
const POLL_INTERVAL_MS = 3000;
|
||
const MAX_POLL_INTERVAL_MS = 15_000;
|
||
const POLL_TIMEOUT_MS = 18 * 60 * 1000;
|
||
const MAX_TRANSIENT_POLL_FAILURES = 8;
|
||
const IMAGE_DOWNLOAD_TIMEOUT_MS = 120_000;
|
||
|
||
function joinApiPath(baseURL: string, path: string) {
|
||
const cleanBase = String(baseURL || VISIONARY_BASE_URL).replace(/\/+$/, "");
|
||
const cleanPath = path.startsWith("/") ? path : `/${path}`;
|
||
return `${cleanBase}${cleanPath}`;
|
||
}
|
||
|
||
function delay(ms: number) {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
function normalizeModelId(modelId?: string) {
|
||
const normalized = String(modelId || "").trim();
|
||
if (!normalized) throw new Error("Visionary 模型 ID 未配置");
|
||
return normalized;
|
||
}
|
||
|
||
function normalizeAspectRatio(value?: string) {
|
||
const normalized = String(value || "auto").trim().toLowerCase();
|
||
if (normalized === "auto") return "auto";
|
||
const match = normalized.match(/^(\d+)\s*:\s*(\d+)$/);
|
||
if (!match) return "auto";
|
||
return `${Number(match[1])}:${Number(match[2])}`;
|
||
}
|
||
|
||
function getSupportedAspectRatios(
|
||
modelId: string,
|
||
modelConfig?: VisionaryModelConfig | null
|
||
) {
|
||
const preset = getVisionaryModelPresetById(modelId);
|
||
const configured = modelConfig?.supportedAspectRatios;
|
||
const values = Array.isArray(configured)
|
||
? configured
|
||
: (preset?.imageGenConfig?.supportedAspectRatios as string[] | undefined);
|
||
return Array.from(
|
||
new Set((values || ["auto", "1:1"]).map(normalizeAspectRatio))
|
||
);
|
||
}
|
||
|
||
function resolveAspectRatio(
|
||
requested: string | undefined,
|
||
modelId: string,
|
||
modelConfig?: VisionaryModelConfig | null
|
||
) {
|
||
const supported = getSupportedAspectRatios(modelId, modelConfig);
|
||
const normalized = normalizeAspectRatio(requested);
|
||
if (supported.includes(normalized)) return normalized;
|
||
return supported.includes("auto") ? "auto" : supported[0] || "1:1";
|
||
}
|
||
|
||
function normalizeImageSize(
|
||
requested: string | undefined,
|
||
modelId: string,
|
||
modelConfig?: VisionaryModelConfig | null
|
||
) {
|
||
const preset = getVisionaryModelPresetById(modelId);
|
||
const configured = modelConfig?.supportedImageSizes;
|
||
const supported = (Array.isArray(configured)
|
||
? configured
|
||
: (preset?.imageGenConfig?.supportedImageSizes as string[] | undefined) || ["1K"]
|
||
)
|
||
.map((size) => String(size).trim().toUpperCase())
|
||
.filter((size) => ["1K", "2K", "4K"].includes(size));
|
||
const defaultSize = String(preset?.defaultParams?.imageSize || supported[0] || "1K")
|
||
.trim()
|
||
.toUpperCase();
|
||
const normalized = String(requested || defaultSize).trim().toUpperCase();
|
||
if (supported.includes(normalized)) return normalized;
|
||
return supported.includes(defaultSize) ? defaultSize : supported[0] || "1K";
|
||
}
|
||
|
||
function normalizeQuality(value?: string) {
|
||
const normalized = String(value || "auto").trim().toLowerCase();
|
||
return ["auto", "low", "medium", "high"].includes(normalized)
|
||
? normalized
|
||
: "auto";
|
||
}
|
||
|
||
function extractPromptAndImages(
|
||
messages: Array<{ role: string; content?: any; parts?: any[] }>
|
||
) {
|
||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||
if (!lastUserMessage) throw new Error("未找到用户消息");
|
||
|
||
const texts: string[] = [];
|
||
const imageUrls: string[] = [];
|
||
const seenImages = new Set<string>();
|
||
const pushText = (value: unknown) => {
|
||
if (typeof value === "string" && value.trim()) texts.push(value.trim());
|
||
};
|
||
const pushImage = (value: unknown) => {
|
||
let url = "";
|
||
if (typeof value === "string") url = value;
|
||
else if (value instanceof URL) url = value.toString();
|
||
else if (value && typeof value === "object") {
|
||
url = String(
|
||
(value as any).url ||
|
||
(value as any).imageUrl ||
|
||
(value as any).fileUrl ||
|
||
(value as any).src ||
|
||
""
|
||
);
|
||
}
|
||
url = url.trim();
|
||
if (!url || seenImages.has(url)) return;
|
||
if (!url.startsWith("data:image/") && !/^https?:\/\//i.test(url)) return;
|
||
seenImages.add(url);
|
||
imageUrls.push(url);
|
||
};
|
||
const visitPart = (part: any) => {
|
||
if (!part) return;
|
||
if (typeof part === "string") {
|
||
pushText(part);
|
||
return;
|
||
}
|
||
if (part.type === "text" || typeof part.text === "string") pushText(part.text);
|
||
if (part.type === "image_url") pushImage(part.image_url);
|
||
if (part.type === "image") pushImage(part.image || part.url);
|
||
if (part.type === "file") pushImage(part.url || part.file || part.data);
|
||
if (part.inlineData?.data) {
|
||
pushImage(
|
||
`data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}`
|
||
);
|
||
}
|
||
};
|
||
|
||
const content = lastUserMessage.content;
|
||
if (typeof content === "string") pushText(content);
|
||
else if (Array.isArray(content)) content.forEach(visitPart);
|
||
if (Array.isArray(lastUserMessage.parts)) lastUserMessage.parts.forEach(visitPart);
|
||
|
||
const prompt = texts.join("\n").trim();
|
||
if (!prompt) throw new Error("请输入图片生成或编辑要求");
|
||
return { prompt, imageUrls };
|
||
}
|
||
|
||
function parseDataUrl(dataUrl: string) {
|
||
const match = dataUrl.match(/^data:(image\/(?:jpeg|png|webp));base64,(.+)$/s);
|
||
if (!match) throw new Error("Visionary 参考图仅支持 JPEG、PNG 或 WebP Data URI");
|
||
const mimeType = match[1] || "image/png";
|
||
const buffer = Buffer.from(match[2] || "", "base64");
|
||
return { mimeType, buffer, dataUrl };
|
||
}
|
||
|
||
async function loadReferenceImage(imageUrl: string): Promise<ReferenceImage> {
|
||
if (imageUrl.startsWith("data:")) {
|
||
const parsed = parseDataUrl(imageUrl);
|
||
const metadata = await sharp(parsed.buffer, { failOn: "none" })
|
||
.metadata()
|
||
.catch(() => null);
|
||
return {
|
||
apiValue: parsed.dataUrl,
|
||
width: metadata?.width,
|
||
height: metadata?.height,
|
||
};
|
||
}
|
||
|
||
const response = await fetch(imageUrl, {
|
||
cache: "no-store",
|
||
redirect: "follow",
|
||
signal: AbortSignal.timeout(IMAGE_DOWNLOAD_TIMEOUT_MS),
|
||
headers: {
|
||
Accept: "image/avif,image/webp,image/png,image/jpeg,image/*,*/*;q=0.8",
|
||
"User-Agent": "yedir-ai-visionary/1.0",
|
||
},
|
||
});
|
||
if (!response.ok) {
|
||
throw new Error(`Visionary 参考图下载失败（HTTP ${response.status}）`);
|
||
}
|
||
const mimeType = response.headers.get("content-type")?.split(";")[0] || "";
|
||
if (!/^image\/(?:jpeg|png|webp)$/i.test(mimeType)) {
|
||
throw new Error(`Visionary 不支持此参考图格式：${mimeType || "unknown"}`);
|
||
}
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
const metadata = await sharp(buffer, { failOn: "none" })
|
||
.metadata()
|
||
.catch(() => null);
|
||
return {
|
||
apiValue: /^https:\/\//i.test(imageUrl)
|
||
? imageUrl
|
||
: `data:${mimeType};base64,${buffer.toString("base64")}`,
|
||
width: metadata?.width,
|
||
height: metadata?.height,
|
||
};
|
||
}
|
||
|
||
function detectImageExtension(mimeType?: string | null) {
|
||
const normalized = String(mimeType || "").toLowerCase();
|
||
if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg";
|
||
if (normalized.includes("webp")) return "webp";
|
||
if (normalized.includes("gif")) return "gif";
|
||
return "png";
|
||
}
|
||
|
||
function generateFilename(prefix: string, extension: string) {
|
||
return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${extension}`;
|
||
}
|
||
|
||
function toArrayBuffer(buffer: Buffer) {
|
||
return buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
}
|
||
|
||
async function uploadImageBuffer(
|
||
buffer: Buffer,
|
||
mimeType?: string | null
|
||
): Promise<UploadedImage> {
|
||
const extension = detectImageExtension(mimeType);
|
||
const thumbnailBuffer = await sharp(buffer, { failOn: "none" })
|
||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||
.jpeg({ quality: 82 })
|
||
.toBuffer()
|
||
.catch(() => buffer);
|
||
const [original, thumbnail] = await Promise.all([
|
||
uploadFile(generateFilename("visionary-orig", extension), toArrayBuffer(buffer)),
|
||
uploadFile(generateFilename("visionary-thumb", "jpg"), toArrayBuffer(thumbnailBuffer)),
|
||
]);
|
||
return { original: original.url, thumbnail: thumbnail.url };
|
||
}
|
||
|
||
async function archiveOutputImage(imageUrl: string): Promise<UploadedImage> {
|
||
const immediatePreview = await createDeferredUploadedImage(imageUrl, { providerType: "visionary" });
|
||
if (immediatePreview) return immediatePreview;
|
||
if (imageUrl.startsWith("data:")) {
|
||
const parsed = parseDataUrl(imageUrl);
|
||
return uploadImageBuffer(parsed.buffer, parsed.mimeType);
|
||
}
|
||
|
||
|
||
const response = await fetch(imageUrl, {
|
||
cache: "no-store",
|
||
redirect: "follow",
|
||
signal: AbortSignal.timeout(IMAGE_DOWNLOAD_TIMEOUT_MS),
|
||
headers: { "User-Agent": "yedir-ai-visionary/1.0" },
|
||
});
|
||
if (!response.ok) {
|
||
throw new Error(`Visionary 结果图片下载失败（HTTP ${response.status}）`);
|
||
}
|
||
return uploadImageBuffer(
|
||
Buffer.from(await response.arrayBuffer()),
|
||
response.headers.get("content-type")
|
||
);
|
||
}
|
||
|
||
async function parseJsonResponse(response: Response) {
|
||
const rawText = await response.text();
|
||
if (!rawText.trim()) return null;
|
||
try {
|
||
return JSON.parse(rawText);
|
||
} catch {
|
||
return rawText;
|
||
}
|
||
}
|
||
|
||
function extractErrorMessage(payload: any, fallback = "Visionary 请求失败") {
|
||
if (typeof payload === "string" && payload.trim()) return payload.trim();
|
||
const value =
|
||
payload?.error?.message ||
|
||
payload?.data?.error?.message ||
|
||
payload?.data?.error ||
|
||
payload?.error ||
|
||
payload?.message ||
|
||
fallback;
|
||
return typeof value === "string" ? value : fallback;
|
||
}
|
||
|
||
function extractSubmittedTask(payload: any) {
|
||
const task = Array.isArray(payload?.data) ? payload.data[0] : payload?.data || payload;
|
||
const taskId = String(task?.task_id || task?.taskId || task?.id || "").trim();
|
||
const retryAfter = Number(task?.retry_after || task?.retryAfter || 3);
|
||
return { taskId, retryAfter };
|
||
}
|
||
|
||
function extractTaskImages(payload: any) {
|
||
const task = payload?.data || payload;
|
||
const images = task?.result?.images;
|
||
if (!Array.isArray(images)) return [] as string[];
|
||
return Array.from(
|
||
new Set(
|
||
images.flatMap((image: any) => {
|
||
const value = image?.url;
|
||
if (Array.isArray(value)) return value;
|
||
return value ? [value] : [];
|
||
})
|
||
)
|
||
)
|
||
.map((value) => String(value || "").trim())
|
||
.filter((value) => /^https?:\/\//i.test(value) || value.startsWith("data:image/"));
|
||
}
|
||
|
||
async function submitTask(params: {
|
||
apiKey: string;
|
||
aspectRatio: string;
|
||
baseURL: string;
|
||
imageSize: string;
|
||
modelId: string;
|
||
prompt: string;
|
||
promptExtend?: boolean;
|
||
quality?: string;
|
||
referenceImages: ReferenceImage[];
|
||
}) {
|
||
const clientRequestId = `yedir-${randomUUID()}`;
|
||
const body: Record<string, unknown> = {
|
||
model: params.modelId,
|
||
prompt: params.prompt,
|
||
images: params.referenceImages.map((image) => image.apiValue),
|
||
size: params.aspectRatio,
|
||
resolution: params.imageSize,
|
||
client_request_id: clientRequestId,
|
||
};
|
||
if (params.modelId === "gpt-image-2") {
|
||
body.quality = normalizeQuality(params.quality);
|
||
}
|
||
if (params.modelId === "nano-banana-pro") {
|
||
body.optimizeChineseText = Boolean(params.promptExtend);
|
||
}
|
||
|
||
const response = await fetch(joinApiPath(params.baseURL, "/v1/images/generations"), {
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
"Content-Type": "application/json",
|
||
"Idempotency-Key": clientRequestId,
|
||
"User-Agent": "yedir-ai-visionary/1.0",
|
||
},
|
||
body: JSON.stringify(body),
|
||
signal: AbortSignal.timeout(30_000),
|
||
});
|
||
const payload = await parseJsonResponse(response);
|
||
if (!response.ok) {
|
||
throw new Error(`${extractErrorMessage(payload)}（HTTP ${response.status}）`);
|
||
}
|
||
const task = extractSubmittedTask(payload);
|
||
if (!task.taskId) throw new Error("Visionary 未返回 task_id");
|
||
return task;
|
||
}
|
||
|
||
async function waitForTask(params: {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
initialRetryAfter: number;
|
||
taskId: string;
|
||
}) {
|
||
const startedAt = Date.now();
|
||
let retryAfter = params.initialRetryAfter;
|
||
let transientFailures = 0;
|
||
|
||
while (Date.now() - startedAt < POLL_TIMEOUT_MS) {
|
||
const waitMs = Math.min(
|
||
MAX_POLL_INTERVAL_MS,
|
||
Math.max(POLL_INTERVAL_MS, Math.round(retryAfter * 1000))
|
||
);
|
||
await delay(waitMs);
|
||
|
||
try {
|
||
const response = await fetch(
|
||
joinApiPath(params.baseURL, `/v1/tasks/${encodeURIComponent(params.taskId)}`),
|
||
{
|
||
cache: "no-store",
|
||
headers: {
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
"User-Agent": "yedir-ai-visionary/1.0",
|
||
},
|
||
signal: AbortSignal.timeout(30_000),
|
||
}
|
||
);
|
||
const payload = await parseJsonResponse(response);
|
||
if (!response.ok) {
|
||
if (response.status >= 500 || response.status === 429) {
|
||
throw new Error(`HTTP ${response.status}`);
|
||
}
|
||
throw new Error(extractErrorMessage(payload));
|
||
}
|
||
|
||
transientFailures = 0;
|
||
const task = payload?.data || payload;
|
||
const status = String(task?.status || "").trim().toLowerCase();
|
||
retryAfter = Number(task?.retry_after || retryAfter || 3);
|
||
if (status === "completed") {
|
||
const images = extractTaskImages(payload);
|
||
if (images.length === 0) throw new Error("Visionary 任务完成但未返回图片");
|
||
return images;
|
||
}
|
||
if (["failed", "error", "cancelled", "canceled"].includes(status)) {
|
||
throw new Error(extractErrorMessage(payload, "Visionary 图片生成失败"));
|
||
}
|
||
} catch (error) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
if (!/^HTTP (?:429|5\d\d)$/.test(message) && !/fetch|network|abort/i.test(message)) {
|
||
throw error;
|
||
}
|
||
transientFailures += 1;
|
||
if (transientFailures >= MAX_TRANSIENT_POLL_FAILURES) throw error;
|
||
}
|
||
}
|
||
|
||
throw new Error("Visionary 生成超时，任务可能仍在处理，请稍后重试");
|
||
}
|
||
|
||
function formatMarkdown(images: UploadedImage[], imageSize: string) {
|
||
return images
|
||
.map((image) => ``)
|
||
.join("\n\n");
|
||
}
|
||
|
||
export function createVisionary({ apiKey, baseURL = VISIONARY_BASE_URL }: VisionaryConfig) {
|
||
return {
|
||
provider: "visionary",
|
||
chat: (modelId: string) => ({
|
||
provider: "visionary",
|
||
doGenerate: async (options: {
|
||
messages: Array<{ role: string; content?: any; parts?: any[] }>;
|
||
imageParams?: VisionaryImageParams;
|
||
imageGenConfig?: VisionaryModelConfig | null;
|
||
overrideModelId?: string;
|
||
}) => {
|
||
const requestedModelId = normalizeModelId(options.overrideModelId || modelId);
|
||
const preset = getVisionaryModelPresetById(requestedModelId);
|
||
if (!preset) throw new Error(`Visionary 暂不支持模型：${requestedModelId}`);
|
||
|
||
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
|
||
const maxReferenceImages = Math.max(
|
||
0,
|
||
Math.min(9, options.imageGenConfig?.maxReferenceImages || 9)
|
||
);
|
||
const referenceImages = await Promise.all(
|
||
imageUrls.slice(0, maxReferenceImages).map(loadReferenceImage)
|
||
);
|
||
const aspectRatio = resolveAspectRatio(
|
||
options.imageParams?.aspectRatio,
|
||
requestedModelId,
|
||
options.imageGenConfig
|
||
);
|
||
const imageSize = normalizeImageSize(
|
||
options.imageParams?.imageSize,
|
||
requestedModelId,
|
||
options.imageGenConfig
|
||
);
|
||
|
||
const submitted = await submitTask({
|
||
apiKey,
|
||
aspectRatio,
|
||
baseURL,
|
||
imageSize,
|
||
modelId: requestedModelId,
|
||
prompt,
|
||
promptExtend: options.imageParams?.promptExtend,
|
||
quality: options.imageParams?.quality,
|
||
referenceImages,
|
||
});
|
||
const outputUrls = await waitForTask({
|
||
apiKey,
|
||
baseURL,
|
||
initialRetryAfter: submitted.retryAfter,
|
||
taskId: submitted.taskId,
|
||
});
|
||
const uploadedImages = await Promise.all(
|
||
outputUrls
|
||
.slice(0, Math.max(1, options.imageParams?.imageCount || 1))
|
||
.map(archiveOutputImage)
|
||
);
|
||
const text = formatMarkdown(uploadedImages, imageSize);
|
||
return {
|
||
text,
|
||
finishReason: "stop" as const,
|
||
usage: { promptTokens: 0, completionTokens: text.length },
|
||
};
|
||
},
|
||
doStream: async function* (options: {
|
||
messages: Array<{ role: string; content?: any; parts?: any[] }>;
|
||
imageParams?: VisionaryImageParams;
|
||
imageGenConfig?: VisionaryModelConfig | null;
|
||
overrideModelId?: string;
|
||
}) {
|
||
const result = await this.doGenerate(options);
|
||
yield { type: "text-delta", textDelta: result.text };
|
||
},
|
||
}),
|
||
};
|
||
}
|