/** * 阿里云千问图片生成/编辑 Provider * * 支持模型: * - qwen-image-edit-plus 系列:支持 1-6 张输出,自定义分辨率,提示词优化 * - qwen-image-edit:仅支持 1 张输出 * * API 文档:https://help.aliyun.com/zh/model-studio/developer-reference/qwen-vl-plus */ import sharp from "sharp"; import { uploadFile } from "@/lib/storage/service"; interface AlibabaImageConfig { apiKey: string; baseURL: string; } // 阿里生图请求参数 export interface AlibabaImageParams { n?: number; // 输出图片数量,1-6(qwen-image-edit-plus)或 1(qwen-image-edit) size?: string; // 输出分辨率,如 "1024*1024",宽高 512-2048 negativePrompt?: string; // 反向提示词 promptExtend?: boolean; // 是否开启提示词智能优化 watermark?: boolean; // 是否添加水印 seed?: number; // 随机种子 } // 阿里生图 API 响应 interface AlibabaImageResponse { output?: { choices?: Array<{ finish_reason: string; message: { role: string; content: Array<{ image?: string }>; }; }>; }; usage?: { width: number; height: number; image_count: number; }; request_id?: string; code?: string; message?: string; } const ALIBABA_IMAGE_FETCH_TIMEOUT_MS = 20000; const ALIBABA_MIN_SIZE = 512; const ALIBABA_MAX_SIZE = 2048; function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" { const normalized = String(imageSize || "").trim().toUpperCase(); if (normalized === "1K" || normalized === "2K" || normalized === "4K") return normalized; return "2K"; } function calculateAlibabaSizeByAspect(aspectValue: number, baseSize: number): string { const safeAspect = Number.isFinite(aspectValue) && aspectValue > 0 ? aspectValue : 1; const totalPixels = baseSize * baseSize; let height = Math.round(Math.sqrt(totalPixels / safeAspect)); let width = Math.round(height * safeAspect); if (width > ALIBABA_MAX_SIZE) { const scale = ALIBABA_MAX_SIZE / width; width = ALIBABA_MAX_SIZE; height = Math.round(height * scale); } if (height > ALIBABA_MAX_SIZE) { const scale = ALIBABA_MAX_SIZE / height; height = ALIBABA_MAX_SIZE; width = Math.round(width * scale); } if (width < ALIBABA_MIN_SIZE) { const scale = ALIBABA_MIN_SIZE / width; width = ALIBABA_MIN_SIZE; height = Math.round(height * scale); } if (height < ALIBABA_MIN_SIZE) { const scale = ALIBABA_MIN_SIZE / height; height = ALIBABA_MIN_SIZE; width = Math.round(width * scale); } width = Math.max(ALIBABA_MIN_SIZE, Math.min(ALIBABA_MAX_SIZE, width)); height = Math.max(ALIBABA_MIN_SIZE, Math.min(ALIBABA_MAX_SIZE, height)); width = Math.round(width / 2) * 2; height = Math.round(height / 2) * 2; return `${width}*${height}`; } async function inferAspectRatioFromImageUrl(imageUrl: string): Promise { if (!/^https?:\/\//i.test(imageUrl)) return null; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), ALIBABA_IMAGE_FETCH_TIMEOUT_MS); try { const response = await fetch(imageUrl, { signal: controller.signal }); if (!response.ok) { console.warn("[Alibaba Image] Failed to fetch reference image for auto size:", response.status); return null; } const buffer = Buffer.from(await response.arrayBuffer()); if (!buffer.length) return null; const metadata = await sharp(buffer).metadata(); if (!metadata.width || !metadata.height || metadata.height <= 0) return null; return metadata.width / metadata.height; } catch (error) { console.warn("[Alibaba Image] Failed to infer aspect ratio from reference image:", error); return null; } finally { clearTimeout(timeoutId); } } async function inferImageDimensionsFromImageUrl( imageUrl: string, ): Promise<{ width: number; height: number; aspectRatio: number } | null> { if (!/^https?:\/\//i.test(imageUrl)) return null; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), ALIBABA_IMAGE_FETCH_TIMEOUT_MS); try { const response = await fetch(imageUrl, { signal: controller.signal }); if (!response.ok) { console.warn("[Alibaba Image] Failed to fetch reference image dimensions:", response.status); return null; } const buffer = Buffer.from(await response.arrayBuffer()); if (!buffer.length) return null; const metadata = await sharp(buffer).metadata(); if (!metadata.width || !metadata.height || metadata.height <= 0) return null; return { width: metadata.width, height: metadata.height, aspectRatio: metadata.width / metadata.height, }; } catch (error) { console.warn("[Alibaba Image] Failed to infer image dimensions from reference image:", error); return null; } finally { clearTimeout(timeoutId); } } function calculateAlibabaSizeFromDimensions( width: number, height: number, baseSize: number, ): string { const safeWidth = Math.max(1, Math.round(width)); const safeHeight = Math.max(1, Math.round(height)); const longestSide = Math.max(safeWidth, safeHeight); const scale = baseSize / longestSide; let scaledWidth = Math.round(safeWidth * scale); let scaledHeight = Math.round(safeHeight * scale); if (scaledWidth > ALIBABA_MAX_SIZE || scaledHeight > ALIBABA_MAX_SIZE) { const clampScale = Math.min( ALIBABA_MAX_SIZE / scaledWidth, ALIBABA_MAX_SIZE / scaledHeight, ); scaledWidth = Math.round(scaledWidth * clampScale); scaledHeight = Math.round(scaledHeight * clampScale); } if (scaledWidth < ALIBABA_MIN_SIZE || scaledHeight < ALIBABA_MIN_SIZE) { const boostScale = Math.max( ALIBABA_MIN_SIZE / scaledWidth, ALIBABA_MIN_SIZE / scaledHeight, ); scaledWidth = Math.round(scaledWidth * boostScale); scaledHeight = Math.round(scaledHeight * boostScale); } scaledWidth = Math.max(ALIBABA_MIN_SIZE, Math.min(ALIBABA_MAX_SIZE, scaledWidth)); scaledHeight = Math.max(ALIBABA_MIN_SIZE, Math.min(ALIBABA_MAX_SIZE, scaledHeight)); scaledWidth = Math.round(scaledWidth / 2) * 2; scaledHeight = Math.round(scaledHeight / 2) * 2; return `${scaledWidth}*${scaledHeight}`; } /** * 创建缩略图 */ async function createThumbnail(buffer: Buffer): Promise { try { const thumbnailBuffer = await sharp(buffer) .resize(400, 400, { fit: "inside", withoutEnlargement: true }) .jpeg({ quality: 80 }) .toBuffer(); console.log(`[Alibaba Image] Thumbnail created, size: ${thumbnailBuffer.length} bytes`); return thumbnailBuffer; } catch (error) { console.error("[Alibaba Image] Failed to create thumbnail:", error); return buffer; } } /** * 生成唯一文件名 */ function generateFilename(prefix: string, extension: string): string { const timestamp = Date.now(); const random = Math.random().toString(36).substring(2, 8); return `${prefix}-${timestamp}-${random}.${extension}`; } /** * 下载图片并上传到 S3 */ async function downloadAndUploadToS3( imageUrl: string, prefix: string = "alibaba-image" ): Promise<{ thumbnail: string; original: string }> { try { console.log("[Alibaba Image] Downloading image from:", imageUrl.substring(0, 100) + "..."); const response = await fetch(imageUrl); if (!response.ok) { throw new Error(`Failed to download image: ${response.status}`); } const buffer = Buffer.from(await response.arrayBuffer()); console.log(`[Alibaba Image] Downloaded image, size: ${buffer.length} bytes`); // 创建缩略图 const thumbnailBuffer = await createThumbnail(buffer); // 上传原图(Buffer 转 ArrayBuffer) const originalFilename = generateFilename(`${prefix}-orig`, "png"); const originalResult = await uploadFile(originalFilename, buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength)); console.log(`[Alibaba Image] Uploaded original to: ${originalResult.url}`); // 上传缩略图 const thumbnailFilename = generateFilename(`${prefix}-thumb`, "jpg"); const thumbnailArrayBuffer = new Uint8Array(thumbnailBuffer).buffer; const thumbnailResult = await uploadFile(thumbnailFilename, thumbnailArrayBuffer); console.log(`[Alibaba Image] Uploaded thumbnail to: ${thumbnailResult.url}`); return { original: originalResult.url, thumbnail: thumbnailResult.url, }; } catch (error) { console.error("[Alibaba Image] Failed to download/upload image:", error); throw error; } } /** * 将 Base64 图片上传到 S3 获取 URL */ async function uploadBase64ToS3(base64Data: string): Promise { try { // 移除 data URL 前缀 const base64Clean = base64Data.replace(/^data:image\/\w+;base64,/, ""); const buffer = Buffer.from(base64Clean, "base64"); const filename = generateFilename("alibaba-ref", "png"); const result = await uploadFile(filename, buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength)); console.log(`[Alibaba Image] Uploaded reference image to: ${result.url}`); return result.url; } catch (error) { console.error("[Alibaba Image] Failed to upload reference image:", error); throw error; } } /** * 发送阿里生图 API 请求 */ async function sendAlibabaImageRequest( baseURL: string, apiKey: string, modelId: string, prompt: string, imageUrls: string[], params: AlibabaImageParams ): Promise { // 构建消息内容 const content: Array<{ image?: string; text?: string }> = []; // 添加输入图片(最多 3 张) for (const url of imageUrls.slice(0, 3)) { content.push({ image: url }); } // 添加编辑指令 content.push({ text: prompt }); const requestBody = { model: modelId, input: { messages: [ { role: "user", content, }, ], }, parameters: { n: params.n || 1, ...(params.negativePrompt && { negative_prompt: params.negativePrompt }), ...(params.size && { size: params.size }), prompt_extend: params.promptExtend ?? true, watermark: params.watermark ?? false, ...(params.seed !== undefined && { seed: params.seed }), }, }; console.log("[Alibaba Image] Sending request to:", baseURL); console.log("[Alibaba Image] Model:", modelId); console.log("[Alibaba Image] Parameters:", JSON.stringify(requestBody.parameters)); const response = await fetch(baseURL, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify(requestBody), }); const data = await response.json(); if (!response.ok || data.code) { console.error("[Alibaba Image] API Error:", data); throw new Error(data.message || `API request failed: ${response.status}`); } console.log("[Alibaba Image] Response received, request_id:", data.request_id); return data; } /** * 处理 API 响应,提取并上传图片 */ async function processResponse( response: AlibabaImageResponse ): Promise> { const images: Array<{ thumbnail: string; original: string }> = []; const choices = response.output?.choices; if (!choices || choices.length === 0) { throw new Error("No images in response"); } for (const choice of choices) { const content = choice.message?.content; if (!content) continue; for (const item of content) { if (item.image) { try { const uploaded = await downloadAndUploadToS3(item.image); images.push(uploaded); } catch (error) { console.error("[Alibaba Image] Failed to process image:", error); } } } } if (images.length === 0) { throw new Error("Failed to process any images"); } console.log(`[Alibaba Image] Processed ${images.length} images`); return images; } /** * 格式化图片响应为 Markdown * 使用与其他生图模型一致的格式:thumb|||originalUrl|||size * 这样 saveGeneratedImages 函数可以正确解析并保存到数据库 */ function formatImageResponse( images: Array<{ thumbnail: string; original: string }>, size?: string ): string { if (images.length === 0) { return "图片生成失败,请重试。"; } // 将分辨率转换为简短格式(如 "1024*1024" -> "1K") const sizeLabel = size ? getSizeLabel(size) : "2K"; return images .map((img) => { // 使用与 Gemini/火山引擎一致的格式:thumb|||originalUrl|||size const altText = `thumb|||${img.original}|||${sizeLabel}`; return `![${altText}](${img.thumbnail})`; }) .join("\n\n"); } /** * 将分辨率字符串转换为简短标签 */ function getSizeLabel(size: string): string { const sizeMap: Record = { "1024*1024": "1K", "1280*1280": "1K", "1536*1536": "2K", "2048*2048": "4K", }; if (sizeMap[size]) return sizeMap[size]; const matched = size.match(/^(\d+)\*(\d+)$/); if (matched) { const width = Number.parseInt(matched[1], 10); const height = Number.parseInt(matched[2], 10); const maxSide = Math.max(width, height); if (maxSide >= 1900) return "4K"; if (maxSide >= 1450) return "2K"; if (maxSide >= 950) return "1K"; } return "2K"; } /** * 创建阿里云千问图片生成 Provider */ export function createAlibabaImage({ apiKey, baseURL }: AlibabaImageConfig) { return { provider: "alibaba-image", chat: (modelId: string) => ({ provider: "alibaba-image", // 🆕 让 chat() 返回的对象也包含 provider 属性,用于 route.ts 中的类型判断 doGenerate: async (options: { messages: Array<{ role: string; content: string | Array<{ type: string; text?: string; image?: string }> }>; imageParams?: AlibabaImageParams & { aspectRatio?: string; imageSize?: string; imageCount?: number }; modelDisplayName?: string; // 🆕 模型显示名称,用于错误提示 overrideModelId?: string; // 🆕 备用模型 ID }) => { console.log("[Alibaba Image] doGenerate called"); const activeModelId = options.overrideModelId || modelId; console.log("[Alibaba Image] Model:", activeModelId); console.log("[Alibaba Image] Image params:", options.imageParams); console.log("[Alibaba Image] Messages count:", options.messages.length); console.log("[Alibaba Image] Messages:", JSON.stringify(options.messages, null, 2)); try { // 提取最后一条用户消息 const lastUserMessage = options.messages.filter((m) => m.role === "user").pop(); if (!lastUserMessage) { throw new Error("No user message found"); } console.log("[Alibaba Image] Last user message:", JSON.stringify(lastUserMessage, null, 2)); // 解析消息内容 let prompt = ""; const imageUrls: string[] = []; if (typeof lastUserMessage.content === "string") { prompt = lastUserMessage.content; } else if (Array.isArray(lastUserMessage.content)) { for (const part of lastUserMessage.content) { if (part.type === "text" && part.text) { prompt = part.text; } // 支持多种图片格式 else if (part.type === "image" && part.image) { // 格式1: { type: "image", image: string } if (part.image.startsWith("data:")) { const url = await uploadBase64ToS3(part.image); imageUrls.push(url); } else { imageUrls.push(part.image); } } else if (part.type === "image_url" && (part as any).image_url?.url) { // 格式2: { type: "image_url", image_url: { url: string } } const imageUrl = (part as any).image_url.url; if (imageUrl.startsWith("data:")) { const url = await uploadBase64ToS3(imageUrl); imageUrls.push(url); } else { imageUrls.push(imageUrl); } } else if (part.type === "file") { // 格式3: { type: "file", url/data: string } - 来自 ChatMessage 的原始格式 // AI SDK convertToModelMessages 会将 url 转换为 data 字段 const fileUrl = (part as any).url || (part as any).data; if (fileUrl) { if (fileUrl.startsWith("data:")) { const url = await uploadBase64ToS3(fileUrl); imageUrls.push(url); } else { imageUrls.push(fileUrl); } } } } } if (!prompt) { throw new Error("No prompt found in message"); } // 验证:qwen-image-edit 系列模型必须有输入图片 if (imageUrls.length === 0) { const modelName = options.modelDisplayName || modelId; throw new Error(`「${modelName}」是图片编辑模型,需要至少上传一张图片才能进行编辑。如需纯文生图,请使用其他生图模型。`); } // 构建参数 const params: AlibabaImageParams = { n: options.imageParams?.imageCount || options.imageParams?.n || 1, negativePrompt: options.imageParams?.negativePrompt, promptExtend: options.imageParams?.promptExtend, watermark: options.imageParams?.watermark, seed: options.imageParams?.seed, }; // 处理分辨率:结合 aspectRatio(宽高比)和 imageSize(分辨率等级) const aspectRatio = options.imageParams?.aspectRatio || "auto"; const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize); // 根据分辨率等级确定基准像素 const baseSizeMap: Record = { "1K": 1024, "2K": 1536, "4K": 2048, }; const baseSize = baseSizeMap[imageSize] || 1536; // 根据宽高比计算实际分辨率 // 阿里云 API 要求宽高在 512-2048 之间 const calculateSize = (ratio: string, base: number): string | null => { // 解析宽高比 const ratioMap: Record = { "1:1": [1, 1], "16:9": [16, 9], "9:16": [9, 16], "4:3": [4, 3], "3:4": [3, 4], "3:2": [3, 2], "2:3": [2, 3], "5:4": [5, 4], "4:5": [4, 5], "21:9": [21, 9], }; if (!ratioMap[ratio]) return null; const [w, h] = ratioMap[ratio]; const aspectValue = w / h; return calculateAlibabaSizeByAspect(aspectValue, base); }; // 处理 size 参数 // 1. 如果用户直接指定了 size,使用用户指定的 // 2. 如果是 auto 模式,按参考图宽高比计算 size,确保清晰度档位生效 // 3. 如果指定了具体比例,根据 aspectRatio 和 imageSize 计算 if (options.imageParams?.size) { params.size = options.imageParams.size; console.log(`[Alibaba Image] Using user-specified size: ${params.size}`); } else { if (aspectRatio === "auto") { // 阿里文档在 auto 下默认接近 1024*1024,这会让 4K 选择失效。 // 这里改为根据参考图比例计算 size,确保 1K/2K/4K 档位真正生效。 const referenceImageUrl = imageUrls[0]; const inferredDimensions = await inferImageDimensionsFromImageUrl(referenceImageUrl); if (inferredDimensions) { params.size = calculateAlibabaSizeFromDimensions( inferredDimensions.width, inferredDimensions.height, baseSize, ); console.log( `[Alibaba Image] Auto mode resolved size: imageSize=${imageSize}, reference=${inferredDimensions.width}x${inferredDimensions.height}, inferredAspect=${inferredDimensions.aspectRatio} => size=${params.size}` ); } else { const inferredAspectRatio = await inferAspectRatioFromImageUrl(referenceImageUrl); if (inferredAspectRatio) { params.size = calculateAlibabaSizeByAspect(inferredAspectRatio, baseSize); console.log( `[Alibaba Image] Auto mode resolved size via aspect fallback: imageSize=${imageSize}, inferredAspect=${inferredAspectRatio} => size=${params.size}` ); } else { console.log( `[Alibaba Image] Auto mode keeps provider default size: imageSize=${imageSize}, inferredAspect=unavailable` ); } } } else { const calculatedSize = calculateSize(aspectRatio, baseSize); if (calculatedSize) { params.size = calculatedSize; console.log(`[Alibaba Image] Calculated size: aspectRatio=${aspectRatio}, imageSize=${imageSize} => size=${params.size}`); } else { params.size = `${baseSize}*${baseSize}`; console.log( `[Alibaba Image] Unknown aspect ratio "${aspectRatio}", fallback size=${params.size}` ); } } } // 根据图片数量确定编辑模式 const editMode = imageUrls.length === 1 ? "单图编辑" : `多图融合 (${imageUrls.length}张)`; console.log("[Alibaba Image] 编辑模式:", editMode); console.log("[Alibaba Image] Prompt:", prompt.substring(0, 100) + (prompt.length > 100 ? "..." : "")); console.log("[Alibaba Image] Input images:", imageUrls.length); console.log("[Alibaba Image] Final params:", params); // 发送请求 const response = await sendAlibabaImageRequest( baseURL, apiKey, activeModelId, prompt, imageUrls, params ); // 处理响应 const images = await processResponse(response); const markdownText = formatImageResponse(images, params.size); console.log(`[Alibaba Image] Generated ${images.length} images successfully`); return { text: markdownText, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: markdownText.length, }, }; } catch (error) { console.error("[Alibaba Image] Generation failed:", error); throw error; } }, doStream: async function* (options: { messages: Array<{ role: string; content: string | Array<{ type: string; text?: string; image?: string }> }>; imageParams?: AlibabaImageParams & { aspectRatio?: string; imageSize?: string; imageCount?: number }; modelDisplayName?: string; }) { console.log("[Alibaba Image] doStream called (using doGenerate internally)"); // 阿里生图 API 不支持流式,直接调用 doGenerate const result = await this.doGenerate(options); // 模拟流式输出 yield { type: "text-delta" as const, textDelta: result.text, }; yield { type: "finish" as const, finishReason: "stop" as const, usage: result.usage, }; }, }), }; } export default createAlibabaImage;