524 lines
18 KiB
TypeScript
524 lines
18 KiB
TypeScript
/**
|
||
* 阿里云千问图片生成/编辑 Provider
|
||
*
|
||
* 支持模型：
|
||
* - qwen-image-edit-plus 系列：支持 1-6 张输出，自定义分辨率，提示词优化
|
||
* - qwen-image-edit：仅支持 1 张输出
|
||
*
|
||
* API 文档：https://help.aliyun.com/zh/model-studio/developer-reference/qwen-vl-plus
|
||
*/
|
||
|
||
import sharp from "sharp";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
interface AlibabaImageConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
// 阿里生图请求参数
|
||
export interface AlibabaImageParams {
|
||
n?: number; // 输出图片数量，1-6（qwen-image-edit-plus）或 1（qwen-image-edit）
|
||
size?: string; // 输出分辨率，如 "1024*1024"，宽高 512-2048
|
||
negativePrompt?: string; // 反向提示词
|
||
promptExtend?: boolean; // 是否开启提示词智能优化
|
||
watermark?: boolean; // 是否添加水印
|
||
seed?: number; // 随机种子
|
||
}
|
||
|
||
// 阿里生图 API 响应
|
||
interface AlibabaImageResponse {
|
||
output?: {
|
||
choices?: Array<{
|
||
finish_reason: string;
|
||
message: {
|
||
role: string;
|
||
content: Array<{ image?: string }>;
|
||
};
|
||
}>;
|
||
};
|
||
usage?: {
|
||
width: number;
|
||
height: number;
|
||
image_count: number;
|
||
};
|
||
request_id?: string;
|
||
code?: string;
|
||
message?: string;
|
||
}
|
||
|
||
/**
|
||
* 创建缩略图
|
||
*/
|
||
async function createThumbnail(buffer: Buffer): Promise<Buffer> {
|
||
try {
|
||
const thumbnailBuffer = await sharp(buffer)
|
||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
console.log(`[Alibaba Image] Thumbnail created, size: ${thumbnailBuffer.length} bytes`);
|
||
return thumbnailBuffer;
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Failed to create thumbnail:", error);
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 生成唯一文件名
|
||
*/
|
||
function generateFilename(prefix: string, extension: string): string {
|
||
const timestamp = Date.now();
|
||
const random = Math.random().toString(36).substring(2, 8);
|
||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||
}
|
||
|
||
/**
|
||
* 下载图片并上传到 S3
|
||
*/
|
||
async function downloadAndUploadToS3(
|
||
imageUrl: string,
|
||
prefix: string = "alibaba-image"
|
||
): Promise<{ thumbnail: string; original: string }> {
|
||
try {
|
||
console.log("[Alibaba Image] Downloading image from:", imageUrl.substring(0, 100) + "...");
|
||
|
||
const response = await fetch(imageUrl);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to download image: ${response.status}`);
|
||
}
|
||
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
console.log(`[Alibaba Image] Downloaded image, size: ${buffer.length} bytes`);
|
||
|
||
// 创建缩略图
|
||
const thumbnailBuffer = await createThumbnail(buffer);
|
||
|
||
// 上传原图（Buffer 转 ArrayBuffer）
|
||
const originalFilename = generateFilename(`${prefix}-orig`, "png");
|
||
const originalResult = await uploadFile(originalFilename, buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
|
||
console.log(`[Alibaba Image] Uploaded original to: ${originalResult.url}`);
|
||
|
||
// 上传缩略图
|
||
const thumbnailFilename = generateFilename(`${prefix}-thumb`, "jpg");
|
||
const thumbnailArrayBuffer = new Uint8Array(thumbnailBuffer).buffer;
|
||
const thumbnailResult = await uploadFile(thumbnailFilename, thumbnailArrayBuffer);
|
||
console.log(`[Alibaba Image] Uploaded thumbnail to: ${thumbnailResult.url}`);
|
||
|
||
return {
|
||
original: originalResult.url,
|
||
thumbnail: thumbnailResult.url,
|
||
};
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Failed to download/upload image:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 将 Base64 图片上传到 S3 获取 URL
|
||
*/
|
||
async function uploadBase64ToS3(base64Data: string): Promise<string> {
|
||
try {
|
||
// 移除 data URL 前缀
|
||
const base64Clean = base64Data.replace(/^data:image\/\w+;base64,/, "");
|
||
const buffer = Buffer.from(base64Clean, "base64");
|
||
|
||
const filename = generateFilename("alibaba-ref", "png");
|
||
const result = await uploadFile(filename, buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
|
||
console.log(`[Alibaba Image] Uploaded reference image to: ${result.url}`);
|
||
return result.url;
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Failed to upload reference image:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 发送阿里生图 API 请求
|
||
*/
|
||
async function sendAlibabaImageRequest(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
modelId: string,
|
||
prompt: string,
|
||
imageUrls: string[],
|
||
params: AlibabaImageParams
|
||
): Promise<AlibabaImageResponse> {
|
||
// 构建消息内容
|
||
const content: Array<{ image?: string; text?: string }> = [];
|
||
|
||
// 添加输入图片（最多 3 张）
|
||
for (const url of imageUrls.slice(0, 3)) {
|
||
content.push({ image: url });
|
||
}
|
||
|
||
// 添加编辑指令
|
||
content.push({ text: prompt });
|
||
|
||
const requestBody = {
|
||
model: modelId,
|
||
input: {
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content,
|
||
},
|
||
],
|
||
},
|
||
parameters: {
|
||
n: params.n || 1,
|
||
...(params.negativePrompt && { negative_prompt: params.negativePrompt }),
|
||
...(params.size && { size: params.size }),
|
||
prompt_extend: params.promptExtend ?? true,
|
||
watermark: params.watermark ?? false,
|
||
...(params.seed !== undefined && { seed: params.seed }),
|
||
},
|
||
};
|
||
|
||
console.log("[Alibaba Image] Sending request to:", baseURL);
|
||
console.log("[Alibaba Image] Model:", modelId);
|
||
console.log("[Alibaba Image] Parameters:", JSON.stringify(requestBody.parameters));
|
||
|
||
const response = await fetch(baseURL, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
});
|
||
|
||
const data = await response.json();
|
||
|
||
if (!response.ok || data.code) {
|
||
console.error("[Alibaba Image] API Error:", data);
|
||
throw new Error(data.message || `API request failed: ${response.status}`);
|
||
}
|
||
|
||
console.log("[Alibaba Image] Response received, request_id:", data.request_id);
|
||
return data;
|
||
}
|
||
|
||
/**
|
||
* 处理 API 响应，提取并上传图片
|
||
*/
|
||
async function processResponse(
|
||
response: AlibabaImageResponse
|
||
): Promise<Array<{ thumbnail: string; original: string }>> {
|
||
const images: Array<{ thumbnail: string; original: string }> = [];
|
||
|
||
const choices = response.output?.choices;
|
||
if (!choices || choices.length === 0) {
|
||
throw new Error("No images in response");
|
||
}
|
||
|
||
for (const choice of choices) {
|
||
const content = choice.message?.content;
|
||
if (!content) continue;
|
||
|
||
for (const item of content) {
|
||
if (item.image) {
|
||
try {
|
||
const uploaded = await downloadAndUploadToS3(item.image);
|
||
images.push(uploaded);
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Failed to process image:", error);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (images.length === 0) {
|
||
throw new Error("Failed to process any images");
|
||
}
|
||
|
||
console.log(`[Alibaba Image] Processed ${images.length} images`);
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* 格式化图片响应为 Markdown
|
||
* 使用与其他生图模型一致的格式：thumb|||originalUrl|||size
|
||
* 这样 saveGeneratedImages 函数可以正确解析并保存到数据库
|
||
*/
|
||
function formatImageResponse(
|
||
images: Array<{ thumbnail: string; original: string }>,
|
||
size?: string
|
||
): string {
|
||
if (images.length === 0) {
|
||
return "图片生成失败，请重试。";
|
||
}
|
||
|
||
// 将分辨率转换为简短格式（如 "1024*1024" -> "1K"）
|
||
const sizeLabel = size ? getSizeLabel(size) : "2K";
|
||
|
||
return images
|
||
.map((img) => {
|
||
// 使用与 Gemini/火山引擎一致的格式：thumb|||originalUrl|||size
|
||
const altText = `thumb|||${img.original}|||${sizeLabel}`;
|
||
return ``;
|
||
})
|
||
.join("\n\n");
|
||
}
|
||
|
||
/**
|
||
* 将分辨率字符串转换为简短标签
|
||
*/
|
||
function getSizeLabel(size: string): string {
|
||
const sizeMap: Record<string, string> = {
|
||
"1024*1024": "1K",
|
||
"1280*1280": "1K",
|
||
"1536*1536": "2K",
|
||
"2048*2048": "4K",
|
||
};
|
||
return sizeMap[size] || "2K";
|
||
}
|
||
|
||
/**
|
||
* 创建阿里云千问图片生成 Provider
|
||
*/
|
||
export function createAlibabaImage({ apiKey, baseURL }: AlibabaImageConfig) {
|
||
return {
|
||
provider: "alibaba-image",
|
||
|
||
chat: (modelId: string) => ({
|
||
provider: "alibaba-image", // 🆕 让 chat() 返回的对象也包含 provider 属性，用于 route.ts 中的类型判断
|
||
doGenerate: async (options: {
|
||
messages: Array<{ role: string; content: string | Array<{ type: string; text?: string; image?: string }> }>;
|
||
imageParams?: AlibabaImageParams & { aspectRatio?: string; imageSize?: string; imageCount?: number };
|
||
modelDisplayName?: string; // 🆕 模型显示名称，用于错误提示
|
||
}) => {
|
||
console.log("[Alibaba Image] doGenerate called");
|
||
console.log("[Alibaba Image] Model:", modelId);
|
||
console.log("[Alibaba Image] Image params:", options.imageParams);
|
||
console.log("[Alibaba Image] Messages count:", options.messages.length);
|
||
console.log("[Alibaba Image] Messages:", JSON.stringify(options.messages, null, 2));
|
||
|
||
try {
|
||
// 提取最后一条用户消息
|
||
const lastUserMessage = options.messages.filter((m) => m.role === "user").pop();
|
||
if (!lastUserMessage) {
|
||
throw new Error("No user message found");
|
||
}
|
||
|
||
console.log("[Alibaba Image] Last user message:", JSON.stringify(lastUserMessage, null, 2));
|
||
|
||
// 解析消息内容
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
|
||
if (typeof lastUserMessage.content === "string") {
|
||
prompt = lastUserMessage.content;
|
||
} else if (Array.isArray(lastUserMessage.content)) {
|
||
for (const part of lastUserMessage.content) {
|
||
if (part.type === "text" && part.text) {
|
||
prompt = part.text;
|
||
}
|
||
// 支持多种图片格式
|
||
else if (part.type === "image" && part.image) {
|
||
// 格式1: { type: "image", image: string }
|
||
if (part.image.startsWith("data:")) {
|
||
const url = await uploadBase64ToS3(part.image);
|
||
imageUrls.push(url);
|
||
} else {
|
||
imageUrls.push(part.image);
|
||
}
|
||
} else if (part.type === "image_url" && (part as any).image_url?.url) {
|
||
// 格式2: { type: "image_url", image_url: { url: string } }
|
||
const imageUrl = (part as any).image_url.url;
|
||
if (imageUrl.startsWith("data:")) {
|
||
const url = await uploadBase64ToS3(imageUrl);
|
||
imageUrls.push(url);
|
||
} else {
|
||
imageUrls.push(imageUrl);
|
||
}
|
||
} else if (part.type === "file") {
|
||
// 格式3: { type: "file", url/data: string } - 来自 ChatMessage 的原始格式
|
||
// AI SDK convertToModelMessages 会将 url 转换为 data 字段
|
||
const fileUrl = (part as any).url || (part as any).data;
|
||
if (fileUrl) {
|
||
if (fileUrl.startsWith("data:")) {
|
||
const url = await uploadBase64ToS3(fileUrl);
|
||
imageUrls.push(url);
|
||
} else {
|
||
imageUrls.push(fileUrl);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!prompt) {
|
||
throw new Error("No prompt found in message");
|
||
}
|
||
|
||
// 验证：qwen-image-edit 系列模型必须有输入图片
|
||
if (imageUrls.length === 0) {
|
||
const modelName = options.modelDisplayName || modelId;
|
||
throw new Error(`「${modelName}」是图片编辑模型，需要至少上传一张图片才能进行编辑。如需纯文生图，请使用其他生图模型。`);
|
||
}
|
||
|
||
// 构建参数
|
||
const params: AlibabaImageParams = {
|
||
n: options.imageParams?.imageCount || options.imageParams?.n || 1,
|
||
negativePrompt: options.imageParams?.negativePrompt,
|
||
promptExtend: options.imageParams?.promptExtend,
|
||
watermark: options.imageParams?.watermark,
|
||
seed: options.imageParams?.seed,
|
||
};
|
||
|
||
// 处理分辨率：结合 aspectRatio（宽高比）和 imageSize（分辨率等级）
|
||
const aspectRatio = options.imageParams?.aspectRatio || "auto";
|
||
const imageSize = options.imageParams?.imageSize || "2K";
|
||
|
||
// 根据分辨率等级确定基准像素
|
||
const baseSizeMap: Record<string, number> = {
|
||
"1K": 1024,
|
||
"2K": 1536,
|
||
"4K": 2048,
|
||
};
|
||
const baseSize = baseSizeMap[imageSize] || 1536;
|
||
|
||
// 根据宽高比计算实际分辨率
|
||
// 阿里云 API 要求宽高在 512-2048 之间
|
||
const calculateSize = (ratio: string, base: number): string => {
|
||
// 解析宽高比
|
||
const ratioMap: Record<string, [number, number]> = {
|
||
"auto": [1, 1],
|
||
"1:1": [1, 1],
|
||
"16:9": [16, 9],
|
||
"9:16": [9, 16],
|
||
"4:3": [4, 3],
|
||
"3:4": [3, 4],
|
||
"3:2": [3, 2],
|
||
"2:3": [2, 3],
|
||
"5:4": [5, 4],
|
||
"4:5": [4, 5],
|
||
"21:9": [21, 9],
|
||
};
|
||
|
||
const [w, h] = ratioMap[ratio] || [1, 1];
|
||
const aspectValue = w / h;
|
||
|
||
// 计算宽高，保持总像素接近 base*base
|
||
const totalPixels = base * base;
|
||
|
||
// height = sqrt(totalPixels / aspectValue)
|
||
// width = height * aspectValue
|
||
let height = Math.round(Math.sqrt(totalPixels / aspectValue));
|
||
let width = Math.round(height * aspectValue);
|
||
|
||
// 阿里云 API 限制：宽高在 512-2048 之间
|
||
const MIN_SIZE = 512;
|
||
const MAX_SIZE = 2048;
|
||
|
||
// 如果超出范围，按比例缩放，同时保持宽高比
|
||
if (width > MAX_SIZE) {
|
||
const scale = MAX_SIZE / width;
|
||
width = MAX_SIZE;
|
||
height = Math.round(height * scale);
|
||
}
|
||
if (height > MAX_SIZE) {
|
||
const scale = MAX_SIZE / height;
|
||
height = MAX_SIZE;
|
||
width = Math.round(width * scale);
|
||
}
|
||
if (width < MIN_SIZE) {
|
||
const scale = MIN_SIZE / width;
|
||
width = MIN_SIZE;
|
||
height = Math.round(height * scale);
|
||
}
|
||
if (height < MIN_SIZE) {
|
||
const scale = MIN_SIZE / height;
|
||
height = MIN_SIZE;
|
||
width = Math.round(width * scale);
|
||
}
|
||
|
||
// 最终 clamp 确保在范围内
|
||
width = Math.max(MIN_SIZE, Math.min(MAX_SIZE, width));
|
||
height = Math.max(MIN_SIZE, Math.min(MAX_SIZE, height));
|
||
|
||
// 确保是偶数（某些图像处理可能需要）
|
||
width = Math.round(width / 2) * 2;
|
||
height = Math.round(height / 2) * 2;
|
||
|
||
return `${width}*${height}`;
|
||
};
|
||
|
||
// 如果用户直接指定了 size，使用用户指定的
|
||
if (options.imageParams?.size) {
|
||
params.size = options.imageParams.size;
|
||
} else {
|
||
// 否则根据 aspectRatio 和 imageSize 计算
|
||
params.size = calculateSize(aspectRatio, baseSize);
|
||
}
|
||
|
||
console.log(`[Alibaba Image] Calculated size: aspectRatio=${aspectRatio}, imageSize=${imageSize} => size=${params.size}`);
|
||
|
||
// 根据图片数量确定编辑模式
|
||
const editMode = imageUrls.length === 1 ? "单图编辑" : `多图融合 (${imageUrls.length}张)`;
|
||
console.log("[Alibaba Image] 编辑模式:", editMode);
|
||
console.log("[Alibaba Image] Prompt:", prompt.substring(0, 100) + (prompt.length > 100 ? "..." : ""));
|
||
console.log("[Alibaba Image] Input images:", imageUrls.length);
|
||
console.log("[Alibaba Image] Final params:", params);
|
||
|
||
// 发送请求
|
||
const response = await sendAlibabaImageRequest(
|
||
baseURL,
|
||
apiKey,
|
||
modelId,
|
||
prompt,
|
||
imageUrls,
|
||
params
|
||
);
|
||
|
||
// 处理响应
|
||
const images = await processResponse(response);
|
||
const markdownText = formatImageResponse(images, params.size);
|
||
|
||
console.log(`[Alibaba Image] Generated ${images.length} images successfully`);
|
||
|
||
return {
|
||
text: markdownText,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Generation failed:", error);
|
||
throw error;
|
||
}
|
||
},
|
||
|
||
doStream: async function* (options: {
|
||
messages: Array<{ role: string; content: string | Array<{ type: string; text?: string; image?: string }> }>;
|
||
imageParams?: AlibabaImageParams & { aspectRatio?: string; imageSize?: string; imageCount?: number };
|
||
modelDisplayName?: string;
|
||
}) {
|
||
console.log("[Alibaba Image] doStream called (using doGenerate internally)");
|
||
|
||
// 阿里生图 API 不支持流式，直接调用 doGenerate
|
||
const result = await this.doGenerate(options);
|
||
|
||
// 模拟流式输出
|
||
yield {
|
||
type: "text-delta" as const,
|
||
textDelta: result.text,
|
||
};
|
||
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "stop" as const,
|
||
usage: result.usage,
|
||
};
|
||
},
|
||
}),
|
||
};
|
||
}
|
||
|
||
export default createAlibabaImage;
|
||
|