855 lines
29 KiB
TypeScript
855 lines
29 KiB
TypeScript
/**
|
||
* 阿里云千问图片生成/编辑 Provider
|
||
*
|
||
* 支持模型：
|
||
* - qwen-image-edit-plus 系列：支持 1-6 张输出，自定义分辨率，提示词优化
|
||
* - qwen-image-edit：仅支持 1 张输出
|
||
*
|
||
* API 文档：https://help.aliyun.com/zh/model-studio/developer-reference/qwen-vl-plus
|
||
*/
|
||
|
||
import sharp from "sharp";
|
||
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
|
||
import { isAlibabaImageTextToImageModelId } from "@/lib/ai/model-classification";
|
||
import { buildReferenceImageOrderPrompt } from "@/lib/ai/providers/reference-image-order";
|
||
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
interface AlibabaImageConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
// 阿里生图请求参数
|
||
export interface AlibabaImageParams {
|
||
n?: number; // 输出图片数量，1-6（qwen-image-edit-plus）或 1（qwen-image-edit）
|
||
size?: string; // 输出分辨率，如 "1024*1024"，宽高 512-2048
|
||
negativePrompt?: string; // 反向提示词
|
||
promptExtend?: boolean; // 是否开启提示词智能优化
|
||
watermark?: boolean; // 是否添加水印
|
||
seed?: number; // 随机种子
|
||
}
|
||
|
||
// 阿里生图 API 响应
|
||
interface AlibabaImageResponse {
|
||
output?: {
|
||
choices?: Array<{
|
||
finish_reason: string;
|
||
message: {
|
||
role: string;
|
||
content: Array<{ image?: string }>;
|
||
};
|
||
}>;
|
||
};
|
||
usage?: {
|
||
width: number;
|
||
height: number;
|
||
image_count: number;
|
||
};
|
||
request_id?: string;
|
||
code?: string;
|
||
message?: string;
|
||
}
|
||
|
||
const ALIBABA_IMAGE_FETCH_TIMEOUT_MS = 20000;
|
||
const ALIBABA_IMAGE_GENERATION_TIMEOUT_MS = 10 * 60 * 1000;
|
||
const ALIBABA_MIN_SIZE = 512;
|
||
const ALIBABA_MAX_SIZE = 2048;
|
||
const ALIBABA_OUTPUT_SIZE_MULTIPLE = 2;
|
||
const ALIBABA_MAX_INPUT_SIDE = 4096;
|
||
const ALIBABA_DEFAULT_MAX_INPUT_BYTES = 18 * 1024 * 1024;
|
||
const ALIBABA_QWEN_MAX_INPUT_BYTES = 9.5 * 1024 * 1024;
|
||
|
||
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
|
||
const normalized = String(imageSize || "").trim().toUpperCase();
|
||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") return normalized;
|
||
return "2K";
|
||
}
|
||
|
||
function getAlibabaMaxInputBytes(modelId: string) {
|
||
const normalized = modelId.trim().toLowerCase();
|
||
if (normalized.includes("qwen-image")) {
|
||
return ALIBABA_QWEN_MAX_INPUT_BYTES;
|
||
}
|
||
return ALIBABA_DEFAULT_MAX_INPUT_BYTES;
|
||
}
|
||
|
||
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||
return buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
}
|
||
|
||
function createAbortSignal(timeoutMs: number) {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||
return {
|
||
signal: controller.signal,
|
||
clear: () => clearTimeout(timeoutId),
|
||
};
|
||
}
|
||
|
||
async function fetchImageBufferWithRetry(
|
||
imageUrl: string,
|
||
timeoutMs = ALIBABA_IMAGE_FETCH_TIMEOUT_MS,
|
||
attempts = 2
|
||
): Promise<{ buffer: Buffer; contentType: string | null }> {
|
||
let lastError: unknown = null;
|
||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||
const ctx = createAbortSignal(timeoutMs);
|
||
try {
|
||
const response = await fetch(imageUrl, {
|
||
cache: "no-store",
|
||
redirect: "follow",
|
||
signal: ctx.signal,
|
||
});
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to download image: ${response.status}`);
|
||
}
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
if (!buffer.length) {
|
||
throw new Error("Downloaded image is empty");
|
||
}
|
||
return {
|
||
buffer,
|
||
contentType: response.headers.get("content-type"),
|
||
};
|
||
} catch (error) {
|
||
lastError = error;
|
||
if (attempt < attempts) {
|
||
await new Promise((resolve) => setTimeout(resolve, attempt * 500));
|
||
}
|
||
} finally {
|
||
ctx.clear();
|
||
}
|
||
}
|
||
|
||
throw lastError instanceof Error ? lastError : new Error(String(lastError || "下载图片失败"));
|
||
}
|
||
|
||
function calculateAlibabaSizeByAspect(aspectValue: number, baseSize: number): string {
|
||
const safeAspect = Number.isFinite(aspectValue) && aspectValue > 0 ? aspectValue : 1;
|
||
const totalPixels = baseSize * baseSize;
|
||
|
||
let height = Math.round(Math.sqrt(totalPixels / safeAspect));
|
||
let width = Math.round(height * safeAspect);
|
||
|
||
({ width, height } = constrainAlibabaOutputDimensions({ width, height }));
|
||
|
||
return `${width}*${height}`;
|
||
}
|
||
|
||
async function inferAspectRatioFromImageUrl(imageUrl: string): Promise<number | null> {
|
||
if (!/^https?:\/\//i.test(imageUrl)) return null;
|
||
|
||
try {
|
||
const { buffer } = await fetchImageBufferWithRetry(imageUrl);
|
||
if (!buffer.length) return null;
|
||
|
||
const metadata = await sharp(buffer).metadata();
|
||
if (!metadata.width || !metadata.height || metadata.height <= 0) return null;
|
||
return metadata.width / metadata.height;
|
||
} catch (error) {
|
||
console.warn("[Alibaba Image] Failed to infer aspect ratio from reference image:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function inferImageDimensionsFromImageUrl(
|
||
imageUrl: string,
|
||
): Promise<{ width: number; height: number; aspectRatio: number } | null> {
|
||
if (!/^https?:\/\//i.test(imageUrl)) return null;
|
||
|
||
try {
|
||
const { buffer } = await fetchImageBufferWithRetry(imageUrl);
|
||
if (!buffer.length) return null;
|
||
|
||
const metadata = await sharp(buffer).metadata();
|
||
if (!metadata.width || !metadata.height || metadata.height <= 0) return null;
|
||
|
||
return {
|
||
width: metadata.width,
|
||
height: metadata.height,
|
||
aspectRatio: metadata.width / metadata.height,
|
||
};
|
||
} catch (error) {
|
||
console.warn("[Alibaba Image] Failed to infer image dimensions from reference image:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function calculateAlibabaSizeFromDimensions(
|
||
width: number,
|
||
height: number,
|
||
baseSize: number,
|
||
): string {
|
||
const safeWidth = Math.max(1, Math.round(width));
|
||
const safeHeight = Math.max(1, Math.round(height));
|
||
const longestSide = Math.max(safeWidth, safeHeight);
|
||
const scale = baseSize / longestSide;
|
||
|
||
let scaledWidth = Math.round(safeWidth * scale);
|
||
let scaledHeight = Math.round(safeHeight * scale);
|
||
|
||
({ width: scaledWidth, height: scaledHeight } = constrainAlibabaOutputDimensions({
|
||
width: scaledWidth,
|
||
height: scaledHeight,
|
||
}));
|
||
|
||
return `${scaledWidth}*${scaledHeight}`;
|
||
}
|
||
|
||
function constrainAlibabaOutputDimensions(dimensions: { width: number; height: number }) {
|
||
return constrainImageDimensions(dimensions, {
|
||
maxEdge: ALIBABA_MAX_SIZE,
|
||
maxPixels: ALIBABA_MAX_SIZE * ALIBABA_MAX_SIZE,
|
||
minEdge: ALIBABA_MIN_SIZE,
|
||
multiple: ALIBABA_OUTPUT_SIZE_MULTIPLE,
|
||
});
|
||
}
|
||
|
||
/**
|
||
* 创建缩略图
|
||
*/
|
||
async function createThumbnail(buffer: Buffer): Promise<Buffer> {
|
||
try {
|
||
const thumbnailBuffer = await sharp(buffer)
|
||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
console.log(`[Alibaba Image] Thumbnail created, size: ${thumbnailBuffer.length} bytes`);
|
||
return thumbnailBuffer;
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Failed to create thumbnail:", error);
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 生成唯一文件名
|
||
*/
|
||
function generateFilename(prefix: string, extension: string): string {
|
||
const timestamp = Date.now();
|
||
const random = Math.random().toString(36).substring(2, 8);
|
||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||
}
|
||
|
||
async function uploadPreparedAlibabaInputImage(
|
||
buffer: Buffer,
|
||
extension: "png" | "jpg"
|
||
): Promise<string> {
|
||
const filename = generateFilename("alibaba-input", extension);
|
||
const result = await uploadFile(filename, bufferToArrayBuffer(buffer));
|
||
console.log(`[Alibaba Image] Prepared input image uploaded to: ${result.url}`);
|
||
return result.url;
|
||
}
|
||
|
||
async function prepareAlibabaInputImageUrl(
|
||
imageUrl: string,
|
||
modelId: string
|
||
): Promise<string> {
|
||
const normalizedUrl = String(imageUrl || "").trim();
|
||
if (!/^https?:\/\//i.test(normalizedUrl)) return normalizedUrl;
|
||
|
||
const maxInputBytes = getAlibabaMaxInputBytes(modelId);
|
||
const { buffer, contentType } = await fetchImageBufferWithRetry(normalizedUrl);
|
||
let metadata: sharp.Metadata;
|
||
try {
|
||
metadata = await sharp(buffer, { failOn: "none" }).metadata();
|
||
} catch (error) {
|
||
throw new Error(
|
||
`输入图片无法识别，请转换为 JPG/PNG 后重试: ${
|
||
error instanceof Error ? error.message : String(error)
|
||
}`
|
||
);
|
||
}
|
||
|
||
const format = String(metadata.format || "").toLowerCase();
|
||
const type = String(contentType || "").toLowerCase();
|
||
const cleanUrl = normalizedUrl.split("?")[0]?.toLowerCase() || "";
|
||
const isGif = format === "gif" || type.includes("gif") || cleanUrl.endsWith(".gif");
|
||
const isTooLarge = buffer.length > maxInputBytes;
|
||
const isTooWide = Number(metadata.width || 0) > ALIBABA_MAX_INPUT_SIDE;
|
||
const isTooTall = Number(metadata.height || 0) > ALIBABA_MAX_INPUT_SIDE;
|
||
|
||
if (!isGif && !isTooLarge && !isTooWide && !isTooTall) {
|
||
return normalizedUrl;
|
||
}
|
||
|
||
console.warn("[Alibaba Image] Normalizing input image before provider request:", {
|
||
bytes: buffer.length,
|
||
contentType,
|
||
format,
|
||
height: metadata.height,
|
||
isGif,
|
||
isTooLarge,
|
||
maxInputBytes,
|
||
modelId,
|
||
sourceHost: (() => {
|
||
try {
|
||
return new URL(normalizedUrl).host;
|
||
} catch {
|
||
return "";
|
||
}
|
||
})(),
|
||
width: metadata.width,
|
||
});
|
||
|
||
const basePipeline = sharp(buffer, {
|
||
animated: false,
|
||
failOn: "none",
|
||
})
|
||
.rotate()
|
||
.resize(ALIBABA_MAX_INPUT_SIDE, ALIBABA_MAX_INPUT_SIDE, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
});
|
||
|
||
let prepared = await basePipeline.png({ compressionLevel: 8 }).toBuffer();
|
||
let extension: "png" | "jpg" = "png";
|
||
|
||
if (prepared.length > maxInputBytes) {
|
||
prepared = await sharp(buffer, { animated: false, failOn: "none" })
|
||
.rotate()
|
||
.resize(ALIBABA_MAX_INPUT_SIDE, ALIBABA_MAX_INPUT_SIDE, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 88, mozjpeg: true })
|
||
.toBuffer();
|
||
extension = "jpg";
|
||
}
|
||
|
||
if (prepared.length > maxInputBytes) {
|
||
prepared = await sharp(buffer, { animated: false, failOn: "none" })
|
||
.rotate()
|
||
.resize(3072, 3072, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 82, mozjpeg: true })
|
||
.toBuffer();
|
||
extension = "jpg";
|
||
}
|
||
|
||
if (prepared.length > maxInputBytes) {
|
||
prepared = await sharp(buffer, { animated: false, failOn: "none" })
|
||
.rotate()
|
||
.resize(2048, 2048, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 76, mozjpeg: true })
|
||
.toBuffer();
|
||
extension = "jpg";
|
||
}
|
||
|
||
if (prepared.length > maxInputBytes) {
|
||
const limitMb = Math.floor(maxInputBytes / 1024 / 1024);
|
||
throw new Error(`输入图片压缩后仍超过 ${limitMb}MB，请先压缩图片后重试`);
|
||
}
|
||
|
||
return uploadPreparedAlibabaInputImage(prepared, extension);
|
||
}
|
||
|
||
/**
|
||
* 下载图片并上传到 S3
|
||
*/
|
||
async function downloadAndUploadToS3(
|
||
imageUrl: string,
|
||
prefix: string = "alibaba-image"
|
||
): Promise<{ thumbnail: string; original: string }> {
|
||
try {
|
||
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||
providerType: "alibaba-image",
|
||
});
|
||
if (deferred) return deferred;
|
||
|
||
console.log("[Alibaba Image] Downloading image from:", imageUrl.substring(0, 100) + "...");
|
||
|
||
const { buffer } = await fetchImageBufferWithRetry(imageUrl, 30000, 3);
|
||
console.log(`[Alibaba Image] Downloaded image, size: ${buffer.length} bytes`);
|
||
|
||
// 创建缩略图
|
||
const thumbnailBuffer = await createThumbnail(buffer);
|
||
|
||
// 上传原图（Buffer 转 ArrayBuffer）
|
||
const originalFilename = generateFilename(`${prefix}-orig`, "png");
|
||
const originalResult = await uploadFile(originalFilename, bufferToArrayBuffer(buffer));
|
||
console.log(`[Alibaba Image] Uploaded original to: ${originalResult.url}`);
|
||
|
||
// 上传缩略图
|
||
const thumbnailFilename = generateFilename(`${prefix}-thumb`, "jpg");
|
||
const thumbnailArrayBuffer = new Uint8Array(thumbnailBuffer).buffer;
|
||
const thumbnailResult = await uploadFile(thumbnailFilename, thumbnailArrayBuffer);
|
||
console.log(`[Alibaba Image] Uploaded thumbnail to: ${thumbnailResult.url}`);
|
||
|
||
return {
|
||
original: originalResult.url,
|
||
thumbnail: thumbnailResult.url,
|
||
};
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Failed to download/upload image:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 将 Base64 图片上传到 S3 获取 URL
|
||
*/
|
||
async function uploadBase64ToS3(base64Data: string): Promise<string> {
|
||
try {
|
||
// 移除 data URL 前缀
|
||
const base64Clean = base64Data.replace(/^data:image\/\w+;base64,/, "");
|
||
const buffer = Buffer.from(base64Clean, "base64");
|
||
|
||
const filename = generateFilename("alibaba-ref", "png");
|
||
const result = await uploadFile(filename, bufferToArrayBuffer(buffer));
|
||
console.log(`[Alibaba Image] Uploaded reference image to: ${result.url}`);
|
||
return result.url;
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Failed to upload reference image:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 发送阿里生图 API 请求
|
||
*/
|
||
async function sendAlibabaImageRequest(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
modelId: string,
|
||
prompt: string,
|
||
imageUrls: string[],
|
||
params: AlibabaImageParams
|
||
): Promise<AlibabaImageResponse> {
|
||
// 构建消息内容
|
||
const content: Array<{ image?: string; text?: string }> = [];
|
||
|
||
// 添加输入图片（最多 3 张）
|
||
for (const url of imageUrls.slice(0, 3)) {
|
||
content.push({ image: url });
|
||
}
|
||
|
||
// 添加编辑指令
|
||
content.push({ text: prompt });
|
||
|
||
const requestBody = {
|
||
model: modelId,
|
||
input: {
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content,
|
||
},
|
||
],
|
||
},
|
||
parameters: {
|
||
n: params.n || 1,
|
||
...(params.negativePrompt && { negative_prompt: params.negativePrompt }),
|
||
...(params.size && { size: params.size }),
|
||
prompt_extend: params.promptExtend ?? true,
|
||
watermark: params.watermark ?? false,
|
||
...(params.seed !== undefined && { seed: params.seed }),
|
||
},
|
||
};
|
||
|
||
console.log("[Alibaba Image] Sending request to:", baseURL);
|
||
console.log("[Alibaba Image] Model:", modelId);
|
||
console.log("[Alibaba Image] Parameters:", JSON.stringify(requestBody.parameters));
|
||
|
||
const ctx = createAbortSignal(ALIBABA_IMAGE_GENERATION_TIMEOUT_MS);
|
||
|
||
try {
|
||
const response = await fetch(baseURL, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
signal: ctx.signal,
|
||
});
|
||
|
||
const data = await response.json();
|
||
|
||
if (!response.ok || data.code) {
|
||
console.error("[Alibaba Image] API Error:", data);
|
||
throw new Error(data.message || `API request failed: ${response.status}`);
|
||
}
|
||
|
||
console.log("[Alibaba Image] Response received, request_id:", data.request_id);
|
||
return data;
|
||
} catch (error) {
|
||
if (error instanceof Error && error.name === "AbortError") {
|
||
throw new Error("阿里图片生成请求超时（600秒），请稍后重试");
|
||
}
|
||
throw error;
|
||
} finally {
|
||
ctx.clear();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 处理 API 响应，提取并上传图片
|
||
*/
|
||
async function processResponse(
|
||
response: AlibabaImageResponse
|
||
): Promise<Array<{ thumbnail: string; original: string }>> {
|
||
const images: Array<{ thumbnail: string; original: string }> = [];
|
||
|
||
const choices = response.output?.choices;
|
||
if (!choices || choices.length === 0) {
|
||
throw new Error("No images in response");
|
||
}
|
||
|
||
for (const choice of choices) {
|
||
const content = choice.message?.content;
|
||
if (!content) continue;
|
||
|
||
for (const item of content) {
|
||
if (item.image) {
|
||
try {
|
||
const uploaded = await downloadAndUploadToS3(item.image);
|
||
images.push(uploaded);
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Failed to process image:", error);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (images.length === 0) {
|
||
throw new Error("Failed to process any images");
|
||
}
|
||
|
||
console.log(`[Alibaba Image] Processed ${images.length} images`);
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* 格式化图片响应为 Markdown
|
||
* 使用与其他生图模型一致的格式：thumb|||originalUrl|||size
|
||
* 这样 saveGeneratedImages 函数可以正确解析并保存到数据库
|
||
*/
|
||
function formatImageResponse(
|
||
images: Array<{ thumbnail: string; original: string }>,
|
||
size?: string,
|
||
requestedSizeLabel?: string,
|
||
): string {
|
||
if (images.length === 0) {
|
||
return "图片生成失败，请重试。";
|
||
}
|
||
|
||
// 将分辨率转换为简短格式（如 "1024*1024" -> "1K"）
|
||
const normalizedRequestedSize = normalizeImageSizeLabel(requestedSizeLabel);
|
||
const sizeLabel = requestedSizeLabel
|
||
? normalizedRequestedSize
|
||
: size
|
||
? getSizeLabel(size)
|
||
: "2K";
|
||
|
||
return images
|
||
.map((img) => {
|
||
// 使用与 Gemini/火山引擎一致的格式：thumb|||originalUrl|||size
|
||
const altText = `thumb|||${img.original}|||${sizeLabel}`;
|
||
return ``;
|
||
})
|
||
.join("\n\n");
|
||
}
|
||
|
||
/**
|
||
* 将分辨率字符串转换为简短标签
|
||
*/
|
||
function getSizeLabel(size: string): string {
|
||
const sizeMap: Record<string, string> = {
|
||
"1024*1024": "1K",
|
||
"1280*1280": "1K",
|
||
"1536*1536": "2K",
|
||
"2048*2048": "4K",
|
||
};
|
||
if (sizeMap[size]) return sizeMap[size];
|
||
|
||
const matched = size.match(/^(\d+)\*(\d+)$/);
|
||
if (matched) {
|
||
const width = Number.parseInt(matched[1], 10);
|
||
const height = Number.parseInt(matched[2], 10);
|
||
const maxSide = Math.max(width, height);
|
||
if (maxSide >= 1900) return "4K";
|
||
if (maxSide >= 1450) return "2K";
|
||
if (maxSide >= 950) return "1K";
|
||
}
|
||
|
||
return "2K";
|
||
}
|
||
|
||
/**
|
||
* 创建阿里云千问图片生成 Provider
|
||
*/
|
||
export function createAlibabaImage({ apiKey, baseURL }: AlibabaImageConfig) {
|
||
return {
|
||
provider: "alibaba-image",
|
||
|
||
chat: (modelId: string) => ({
|
||
provider: "alibaba-image", // 🆕 让 chat() 返回的对象也包含 provider 属性，用于 route.ts 中的类型判断
|
||
doGenerate: async (options: {
|
||
messages: Array<{ role: string; content: string | Array<{ type: string; text?: string; image?: string }> }>;
|
||
imageParams?: AlibabaImageParams & { aspectRatio?: string; imageSize?: string; imageCount?: number };
|
||
modelDisplayName?: string; // 🆕 模型显示名称，用于错误提示
|
||
overrideModelId?: string; // 🆕 备用模型 ID
|
||
}) => {
|
||
console.log("[Alibaba Image] doGenerate called");
|
||
const activeModelId = options.overrideModelId || modelId;
|
||
console.log("[Alibaba Image] Model:", activeModelId);
|
||
console.log("[Alibaba Image] Image params:", options.imageParams);
|
||
console.log("[Alibaba Image] Messages count:", options.messages.length);
|
||
console.log("[Alibaba Image] Messages:", JSON.stringify(options.messages, null, 2));
|
||
|
||
try {
|
||
// 提取最后一条用户消息
|
||
const lastUserMessage = options.messages.filter((m) => m.role === "user").pop();
|
||
if (!lastUserMessage) {
|
||
throw new Error("No user message found");
|
||
}
|
||
|
||
console.log("[Alibaba Image] Last user message:", JSON.stringify(lastUserMessage, null, 2));
|
||
|
||
// 解析消息内容
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
|
||
if (typeof lastUserMessage.content === "string") {
|
||
prompt = lastUserMessage.content;
|
||
} else if (Array.isArray(lastUserMessage.content)) {
|
||
for (const part of lastUserMessage.content) {
|
||
if (part.type === "text" && part.text) {
|
||
prompt = part.text;
|
||
}
|
||
// 支持多种图片格式
|
||
else if (part.type === "image" && part.image) {
|
||
// 格式1: { type: "image", image: string }
|
||
if (part.image.startsWith("data:")) {
|
||
const url = await uploadBase64ToS3(part.image);
|
||
imageUrls.push(url);
|
||
} else {
|
||
imageUrls.push(part.image);
|
||
}
|
||
} else if (part.type === "image_url" && (part as any).image_url?.url) {
|
||
// 格式2: { type: "image_url", image_url: { url: string } }
|
||
const imageUrl = (part as any).image_url.url;
|
||
if (imageUrl.startsWith("data:")) {
|
||
const url = await uploadBase64ToS3(imageUrl);
|
||
imageUrls.push(url);
|
||
} else {
|
||
imageUrls.push(imageUrl);
|
||
}
|
||
} else if (part.type === "file") {
|
||
// 格式3: { type: "file", url/data: string } - 来自 ChatMessage 的原始格式
|
||
// AI SDK convertToModelMessages 会将 url 转换为 data 字段
|
||
const fileUrl = (part as any).url || (part as any).data;
|
||
if (fileUrl) {
|
||
if (fileUrl.startsWith("data:")) {
|
||
const url = await uploadBase64ToS3(fileUrl);
|
||
imageUrls.push(url);
|
||
} else {
|
||
imageUrls.push(fileUrl);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!prompt) {
|
||
throw new Error("No prompt found in message");
|
||
}
|
||
|
||
const preparedImageUrls: string[] = [];
|
||
for (const imageUrl of Array.from(new Set(imageUrls)).slice(0, 3)) {
|
||
preparedImageUrls.push(await prepareAlibabaInputImageUrl(imageUrl, activeModelId));
|
||
}
|
||
|
||
// qwen-image-3.0 同时支持文生图与图生图；旧编辑模型仍必须上传参考图。
|
||
if (
|
||
preparedImageUrls.length === 0 &&
|
||
!isAlibabaImageTextToImageModelId(activeModelId)
|
||
) {
|
||
const modelName = options.modelDisplayName || modelId;
|
||
throw new Error(`「${modelName}」是图片编辑模型，需要至少上传一张图片才能进行编辑。如需纯文生图，请使用其他生图模型。`);
|
||
}
|
||
|
||
// 构建参数
|
||
const params: AlibabaImageParams = {
|
||
n: options.imageParams?.imageCount || options.imageParams?.n || 1,
|
||
negativePrompt: options.imageParams?.negativePrompt,
|
||
promptExtend: options.imageParams?.promptExtend,
|
||
watermark: options.imageParams?.watermark,
|
||
seed: options.imageParams?.seed,
|
||
};
|
||
|
||
// 处理分辨率：结合 aspectRatio（宽高比）和 imageSize（分辨率等级）
|
||
const aspectRatio = options.imageParams?.aspectRatio || "auto";
|
||
const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize);
|
||
|
||
// 根据分辨率等级确定基准像素
|
||
const isQwenImage3 = isAlibabaImageTextToImageModelId(activeModelId);
|
||
const baseSizeMap: Record<string, number> = {
|
||
"1K": 1024,
|
||
"2K": isQwenImage3 ? 2048 : 1536,
|
||
"4K": 2048,
|
||
};
|
||
const baseSize = baseSizeMap[imageSize] || 1536;
|
||
|
||
// 根据宽高比计算实际分辨率
|
||
// 阿里云 API 要求宽高在 512-2048 之间
|
||
const calculateSize = (ratio: string, base: number): string | null => {
|
||
// 解析宽高比
|
||
const ratioMap: Record<string, [number, number]> = {
|
||
"1:1": [1, 1],
|
||
"16:9": [16, 9],
|
||
"9:16": [9, 16],
|
||
"4:3": [4, 3],
|
||
"3:4": [3, 4],
|
||
"3:2": [3, 2],
|
||
"2:3": [2, 3],
|
||
"5:4": [5, 4],
|
||
"4:5": [4, 5],
|
||
"21:9": [21, 9],
|
||
};
|
||
|
||
if (!ratioMap[ratio]) return null;
|
||
const [w, h] = ratioMap[ratio];
|
||
const aspectValue = w / h;
|
||
return calculateAlibabaSizeByAspect(aspectValue, base);
|
||
};
|
||
|
||
// 处理 size 参数
|
||
// 1. 如果用户直接指定了 size，使用用户指定的
|
||
// 2. 如果是 auto 模式，按参考图宽高比计算 size，确保清晰度档位生效
|
||
// 3. 如果指定了具体比例，根据 aspectRatio 和 imageSize 计算
|
||
if (options.imageParams?.size) {
|
||
params.size = options.imageParams.size;
|
||
console.log(`[Alibaba Image] Using user-specified size: ${params.size}`);
|
||
} else {
|
||
if (aspectRatio === "auto") {
|
||
// 阿里文档在 auto 下默认接近 1024*1024，这会让 4K 选择失效。
|
||
// 这里改为根据参考图比例计算 size，确保 1K/2K/4K 档位真正生效。
|
||
const referenceImageUrl = preparedImageUrls[0];
|
||
if (!referenceImageUrl) {
|
||
params.size = `${baseSize}*${baseSize}`;
|
||
console.log(
|
||
`[Alibaba Image] Text-to-image auto mode resolved size: imageSize=${imageSize} => size=${params.size}`
|
||
);
|
||
} else {
|
||
const inferredDimensions = await inferImageDimensionsFromImageUrl(referenceImageUrl);
|
||
if (inferredDimensions) {
|
||
params.size = calculateAlibabaSizeFromDimensions(
|
||
inferredDimensions.width,
|
||
inferredDimensions.height,
|
||
baseSize,
|
||
);
|
||
console.log(
|
||
`[Alibaba Image] Auto mode resolved size: imageSize=${imageSize}, reference=${inferredDimensions.width}x${inferredDimensions.height}, inferredAspect=${inferredDimensions.aspectRatio} => size=${params.size}`
|
||
);
|
||
} else {
|
||
const inferredAspectRatio = await inferAspectRatioFromImageUrl(referenceImageUrl);
|
||
if (inferredAspectRatio) {
|
||
params.size = calculateAlibabaSizeByAspect(inferredAspectRatio, baseSize);
|
||
console.log(
|
||
`[Alibaba Image] Auto mode resolved size via aspect fallback: imageSize=${imageSize}, inferredAspect=${inferredAspectRatio} => size=${params.size}`
|
||
);
|
||
} else {
|
||
console.log(
|
||
`[Alibaba Image] Auto mode keeps provider default size: imageSize=${imageSize}, inferredAspect=unavailable`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
const calculatedSize = calculateSize(aspectRatio, baseSize);
|
||
if (calculatedSize) {
|
||
params.size = calculatedSize;
|
||
console.log(`[Alibaba Image] Calculated size: aspectRatio=${aspectRatio}, imageSize=${imageSize} => size=${params.size}`);
|
||
} else {
|
||
params.size = `${baseSize}*${baseSize}`;
|
||
console.log(
|
||
`[Alibaba Image] Unknown aspect ratio "${aspectRatio}", fallback size=${params.size}`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 根据图片数量确定编辑模式
|
||
const editMode = preparedImageUrls.length === 0
|
||
? "文生图"
|
||
: preparedImageUrls.length === 1
|
||
? "单图编辑"
|
||
: `多图融合 (${preparedImageUrls.length}张)`;
|
||
const promptForRequest = buildReferenceImageOrderPrompt(
|
||
prompt,
|
||
preparedImageUrls
|
||
);
|
||
console.log("[Alibaba Image] 编辑模式:", editMode);
|
||
console.log("[Alibaba Image] Prompt:", promptForRequest.substring(0, 100) + (promptForRequest.length > 100 ? "..." : ""));
|
||
console.log("[Alibaba Image] Input images:", preparedImageUrls.length);
|
||
console.log("[Alibaba Image] Final params:", params);
|
||
|
||
// 发送请求
|
||
const response = await sendAlibabaImageRequest(
|
||
baseURL,
|
||
apiKey,
|
||
activeModelId,
|
||
promptForRequest,
|
||
preparedImageUrls,
|
||
params
|
||
);
|
||
|
||
// 处理响应
|
||
const images = await processResponse(response);
|
||
const markdownText = formatImageResponse(images, params.size, imageSize);
|
||
|
||
console.log(`[Alibaba Image] Generated ${images.length} images successfully`);
|
||
|
||
return {
|
||
text: markdownText,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
console.error("[Alibaba Image] Generation failed:", error);
|
||
throw error;
|
||
}
|
||
},
|
||
|
||
doStream: async function* (options: {
|
||
messages: Array<{ role: string; content: string | Array<{ type: string; text?: string; image?: string }> }>;
|
||
imageParams?: AlibabaImageParams & { aspectRatio?: string; imageSize?: string; imageCount?: number };
|
||
modelDisplayName?: string;
|
||
}) {
|
||
console.log("[Alibaba Image] doStream called (using doGenerate internally)");
|
||
|
||
// 阿里生图 API 不支持流式，直接调用 doGenerate
|
||
const result = await this.doGenerate(options);
|
||
|
||
// 模拟流式输出
|
||
yield {
|
||
type: "text-delta" as const,
|
||
textDelta: result.text,
|
||
};
|
||
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "stop" as const,
|
||
usage: result.usage,
|
||
};
|
||
},
|
||
}),
|
||
};
|
||
}
|
||
|
||
export default createAlibabaImage;
|