570 lines
18 KiB
TypeScript
570 lines
18 KiB
TypeScript
import {
|
||
createMediaAssetRelation,
|
||
getAiProvidersByType,
|
||
getMediaAssetsByUserAndUrls,
|
||
getSystemSetting,
|
||
upsertMediaAsset,
|
||
} from "@/lib/db/queries";
|
||
import { type ExtendProvider, resolveAlibabaExtendApiKey, resolveFalExtendApiKey } from "@/lib/ai/extend/config";
|
||
import { expandWithFalOutpaint, FAL_OUTPAINT_MODEL } from "@/lib/ai/extend/fal-outpaint";
|
||
import { executeConfiguredVip98kTask, getVip98kRuntimeProvider } from "@/lib/ai/vip98k-runtime";
|
||
import { expandWithVip98kOutpaint, VIP98K_OUTPAINT_MODEL } from "@/lib/ai/extend/vip98k-outpaint";
|
||
import { db } from "@/lib/db";
|
||
import { generatedImage } from "@/lib/db/schema";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import { archiveTemporaryImage } from "@/lib/storage/image-archive";
|
||
import { randomUUID } from "node:crypto";
|
||
|
||
type ExtendConfig = {
|
||
provider?: ExtendProvider;
|
||
apiKey?: string;
|
||
xScale?: number;
|
||
yScale?: number;
|
||
modelId?: string;
|
||
routeOptionId?: string;
|
||
};
|
||
|
||
const SOURCE_DOWNLOAD_TIMEOUT_MS = 120_000;
|
||
const DASHSCOPE_MIN_IMAGE_EDGE = 512;
|
||
const DASHSCOPE_MAX_IMAGE_EDGE = 4096;
|
||
const DASHSCOPE_MAX_IMAGE_BYTES = 10 * 1024 * 1024;
|
||
const DASHSCOPE_SUPPORTED_IMAGE_FORMATS = new Set([
|
||
"jpeg",
|
||
"jpg",
|
||
"png",
|
||
"heif",
|
||
"webp",
|
||
]);
|
||
|
||
export interface ExecuteExtendJobParams {
|
||
userId: string;
|
||
imageUrl?: string | null;
|
||
imageBase64?: string | null;
|
||
prompt?: string;
|
||
chatId?: string | null;
|
||
messageId?: string | null;
|
||
xScale?: number;
|
||
yScale?: number;
|
||
includeBase64?: boolean;
|
||
skipGallerySave?: boolean;
|
||
sourceImageUrl?: string | null;
|
||
requestId?: string | null;
|
||
}
|
||
|
||
export interface ExecuteExtendJobResult {
|
||
image_base64?: string;
|
||
thumbnail_url: string;
|
||
original_url: string;
|
||
processing_time: number;
|
||
mediaAssetId?: string | null;
|
||
generatedImageId?: string | null;
|
||
}
|
||
|
||
export class ExtendServiceError extends Error {
|
||
status: number;
|
||
detail?: string;
|
||
|
||
constructor(message: string, status = 500, detail?: string) {
|
||
super(message);
|
||
this.name = "ExtendServiceError";
|
||
this.status = status;
|
||
this.detail = detail;
|
||
}
|
||
}
|
||
|
||
async function createThumbnail(buffer: Buffer): Promise<Buffer> {
|
||
try {
|
||
const sharp = await import("sharp");
|
||
return await sharp.default(buffer)
|
||
.resize(500, null, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
} catch (error) {
|
||
console.error("[Extend Service] Failed to create thumbnail:", error);
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
async function uploadBufferPairToS3(buffer: Buffer, originalExtension = "jpg"): Promise<{ thumbnail: string; original: string }> {
|
||
try {
|
||
const timestamp = Date.now();
|
||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||
const thumbnailFilename = `extend-thumb-${timestamp}-${randomStr}.jpg`;
|
||
const originalFilename = `extend-orig-${timestamp}-${randomStr}.${originalExtension}`;
|
||
const thumbnailBuffer = await createThumbnail(buffer);
|
||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||
thumbnailBuffer.byteOffset,
|
||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength,
|
||
) as ArrayBuffer;
|
||
const originalArrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength,
|
||
) as ArrayBuffer;
|
||
const [thumbnailResult, originalResult] = await Promise.all([
|
||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||
uploadFile(originalFilename, originalArrayBuffer),
|
||
]);
|
||
return {
|
||
thumbnail: thumbnailResult.url,
|
||
original: originalResult.url,
|
||
};
|
||
} catch (error) {
|
||
console.error("[Extend Service] Failed to upload to S3:", error);
|
||
throw new ExtendServiceError(
|
||
"图片上传到存储桶失败",
|
||
500,
|
||
error instanceof Error ? error.message : "未知错误",
|
||
);
|
||
}
|
||
}
|
||
|
||
async function uploadImageToS3ForAPI(imageBase64: string): Promise<string> {
|
||
try {
|
||
const buffer = Buffer.from(imageBase64, "base64");
|
||
const timestamp = Date.now();
|
||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||
const filename = `extend-input-${timestamp}-${randomStr}.jpg`;
|
||
const arrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength,
|
||
) as ArrayBuffer;
|
||
const result = await uploadFile(filename, arrayBuffer);
|
||
return result.url;
|
||
} catch (error) {
|
||
console.error("[Extend Service] Failed to upload input to S3:", error);
|
||
throw new ExtendServiceError(
|
||
"图片上传到存储桶失败",
|
||
500,
|
||
error instanceof Error ? error.message : "未知错误",
|
||
);
|
||
}
|
||
}
|
||
|
||
async function normalizeDashscopeExpansionInput(imageUrl: string): Promise<string> {
|
||
const response = await fetch(imageUrl, {
|
||
cache: "no-store",
|
||
redirect: "follow",
|
||
signal: AbortSignal.timeout(SOURCE_DOWNLOAD_TIMEOUT_MS),
|
||
});
|
||
if (!response.ok) {
|
||
throw new ExtendServiceError(
|
||
"下载待拓展图片失败",
|
||
502,
|
||
`${response.status} ${response.statusText}`,
|
||
);
|
||
}
|
||
|
||
const sourceBuffer = Buffer.from(await response.arrayBuffer());
|
||
const sharp = await import("sharp");
|
||
const image = sharp.default(sourceBuffer, { failOn: "none" });
|
||
const metadata = await image.metadata();
|
||
const width = Number(metadata.width || 0);
|
||
const height = Number(metadata.height || 0);
|
||
const format = String(metadata.format || "").toLowerCase();
|
||
if (!width || !height) {
|
||
throw new ExtendServiceError("无法识别待拓展图片尺寸", 422);
|
||
}
|
||
|
||
const withinEdgeLimits =
|
||
width >= DASHSCOPE_MIN_IMAGE_EDGE &&
|
||
height >= DASHSCOPE_MIN_IMAGE_EDGE &&
|
||
width <= DASHSCOPE_MAX_IMAGE_EDGE &&
|
||
height <= DASHSCOPE_MAX_IMAGE_EDGE;
|
||
const withinFileLimit = sourceBuffer.length <= DASHSCOPE_MAX_IMAGE_BYTES;
|
||
if (
|
||
withinEdgeLimits &&
|
||
withinFileLimit &&
|
||
DASHSCOPE_SUPPORTED_IMAGE_FORMATS.has(format)
|
||
) {
|
||
return imageUrl;
|
||
}
|
||
|
||
const downscale = Math.min(
|
||
1,
|
||
DASHSCOPE_MAX_IMAGE_EDGE / width,
|
||
DASHSCOPE_MAX_IMAGE_EDGE / height,
|
||
);
|
||
let targetWidth = Math.max(1, Math.round(width * downscale));
|
||
let targetHeight = Math.max(1, Math.round(height * downscale));
|
||
if (Math.min(targetWidth, targetHeight) < DASHSCOPE_MIN_IMAGE_EDGE) {
|
||
const upscale =
|
||
DASHSCOPE_MIN_IMAGE_EDGE / Math.min(targetWidth, targetHeight);
|
||
targetWidth = Math.max(1, Math.round(targetWidth * upscale));
|
||
targetHeight = Math.max(1, Math.round(targetHeight * upscale));
|
||
}
|
||
|
||
const resizeScale = Math.min(
|
||
1,
|
||
DASHSCOPE_MAX_IMAGE_EDGE / targetWidth,
|
||
DASHSCOPE_MAX_IMAGE_EDGE / targetHeight,
|
||
);
|
||
targetWidth = Math.max(1, Math.round(targetWidth * resizeScale));
|
||
targetHeight = Math.max(1, Math.round(targetHeight * resizeScale));
|
||
|
||
const canvasWidth = Math.max(DASHSCOPE_MIN_IMAGE_EDGE, targetWidth);
|
||
const canvasHeight = Math.max(DASHSCOPE_MIN_IMAGE_EDGE, targetHeight);
|
||
const left = Math.floor((canvasWidth - targetWidth) / 2);
|
||
const top = Math.floor((canvasHeight - targetHeight) / 2);
|
||
const normalizedBuffer = await image
|
||
.rotate()
|
||
.resize(targetWidth, targetHeight, { fit: "fill" })
|
||
.flatten({ background: "#ffffff" })
|
||
.extend({
|
||
left,
|
||
right: canvasWidth - targetWidth - left,
|
||
top,
|
||
bottom: canvasHeight - targetHeight - top,
|
||
background: "#ffffff",
|
||
})
|
||
.jpeg({ quality: 88, mozjpeg: true })
|
||
.toBuffer();
|
||
|
||
const normalizedArrayBuffer = normalizedBuffer.buffer.slice(
|
||
normalizedBuffer.byteOffset,
|
||
normalizedBuffer.byteOffset + normalizedBuffer.byteLength,
|
||
) as ArrayBuffer;
|
||
const uploaded = await uploadFile(
|
||
`extend-dashscope-input-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.jpg`,
|
||
normalizedArrayBuffer,
|
||
);
|
||
console.info("[Extend Service] Normalized DashScope input image", {
|
||
sourceWidth: width,
|
||
sourceHeight: height,
|
||
sourceBytes: sourceBuffer.length,
|
||
sourceFormat: format,
|
||
targetWidth: canvasWidth,
|
||
targetHeight: canvasHeight,
|
||
targetBytes: normalizedBuffer.length,
|
||
});
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function extendImage(
|
||
imageUrl: string,
|
||
config: ExtendConfig,
|
||
xScale: number,
|
||
yScale: number,
|
||
requestId?: string | null,
|
||
): Promise<string> {
|
||
if (config.provider === "vip98k") {
|
||
const result = await executeConfiguredVip98kTask({
|
||
modelId: "image-expansion",
|
||
kind: "image-tool",
|
||
imageUrls: [imageUrl],
|
||
imageParams: { xScale, yScale },
|
||
});
|
||
const outputUrl = result.outputUrls[0];
|
||
if (!outputUrl) {
|
||
throw new ExtendServiceError("VIP98K 图像拓展未返回图片", 502);
|
||
}
|
||
return outputUrl;
|
||
}
|
||
|
||
if (!config.apiKey) {
|
||
throw new ExtendServiceError("图像拓展API密钥未配置", 400);
|
||
}
|
||
const requestBody = {
|
||
model: "image-out-painting",
|
||
input: {
|
||
image_url: imageUrl,
|
||
},
|
||
parameters: {
|
||
x_scale: xScale,
|
||
y_scale: yScale,
|
||
best_quality: false,
|
||
limit_image_size: true,
|
||
add_watermark: false,
|
||
},
|
||
};
|
||
|
||
const response = await fetch(
|
||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/image2image/out-painting",
|
||
{
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${config.apiKey}`,
|
||
"X-DashScope-Async": "enable",
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
},
|
||
);
|
||
|
||
const initialResult = await response.json().catch(() => null);
|
||
if (!response.ok || initialResult?.code) {
|
||
const detail =
|
||
initialResult?.message ||
|
||
initialResult?.code ||
|
||
`${response.status} ${response.statusText}`;
|
||
if (response.status === 401 || initialResult?.code === "InvalidApiKey") {
|
||
throw new ExtendServiceError(
|
||
"图像拓展服务鉴权失败，请在AI模型设置中更新阿里云官方生图Key",
|
||
503,
|
||
detail,
|
||
);
|
||
}
|
||
throw new ExtendServiceError(
|
||
"图像拓展API错误",
|
||
502,
|
||
detail,
|
||
);
|
||
}
|
||
|
||
const taskId = initialResult?.output?.task_id;
|
||
if (!taskId) {
|
||
throw new ExtendServiceError("图像拓展API未返回任务ID", 502);
|
||
}
|
||
console.info("[Extend Service] DashScope task accepted", {
|
||
requestId,
|
||
taskId,
|
||
xScale,
|
||
yScale,
|
||
});
|
||
const taskStatusUrl = `https://dashscope.aliyuncs.com/api/v1/tasks/${taskId}`;
|
||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||
|
||
for (let attempt = 0; attempt < 60; attempt += 1) {
|
||
const statusResponse = await fetch(taskStatusUrl, {
|
||
headers: {
|
||
Authorization: `Bearer ${config.apiKey}`,
|
||
},
|
||
});
|
||
const result = await statusResponse.json().catch(() => null);
|
||
if (!statusResponse.ok || result?.code) {
|
||
throw new ExtendServiceError(
|
||
"查询图像拓展任务失败",
|
||
502,
|
||
result?.message ||
|
||
result?.code ||
|
||
`${statusResponse.status} ${statusResponse.statusText}`,
|
||
);
|
||
}
|
||
if (result?.output?.task_status === "SUCCEEDED") {
|
||
if (!result.output.output_image_url) {
|
||
throw new ExtendServiceError("图像拓展任务未返回图片", 502);
|
||
}
|
||
return result.output.output_image_url;
|
||
}
|
||
if (result?.output?.task_status === "FAILED") {
|
||
throw new ExtendServiceError(
|
||
"图像拓展任务失败",
|
||
500,
|
||
result.output.message || result.output.code || "未知错误",
|
||
);
|
||
}
|
||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||
}
|
||
|
||
throw new ExtendServiceError("图像拓展任务超时", 500);
|
||
}
|
||
|
||
export function inferExtendOptionsFromText(text: string): Pick<ExecuteExtendJobParams, "xScale" | "yScale"> {
|
||
const normalizedText = String(text || "").toLowerCase();
|
||
const scaleMatch = normalizedText.match(/([1-4](?:\.\d+)?)\s*倍/);
|
||
const scale = scaleMatch ? Number(scaleMatch[1]) : undefined;
|
||
return {
|
||
xScale: scale,
|
||
yScale: scale,
|
||
};
|
||
}
|
||
|
||
async function getValidatedExtendConfig(): Promise<ExtendConfig> {
|
||
const extendConfig = await getSystemSetting({ key: "extend_api_config" });
|
||
const config = (extendConfig?.value || {}) as ExtendConfig;
|
||
if (config.provider === "vip98k") {
|
||
return config;
|
||
}
|
||
if (config.provider === "fal") {
|
||
const [imageProviders, videoProviders] = await Promise.all([
|
||
getAiProvidersByType({ type: "fal-image", enabledOnly: true }),
|
||
getAiProvidersByType({ type: "fal-video", enabledOnly: true }),
|
||
]);
|
||
const apiKey = resolveFalExtendApiKey([...imageProviders, ...videoProviders]);
|
||
if (!apiKey) throw new ExtendServiceError("扩图服务未配置可用的 fal 密钥", 503);
|
||
return { ...config, apiKey };
|
||
}
|
||
|
||
const alibabaProviders = await getAiProvidersByType({
|
||
type: "alibaba-image",
|
||
enabledOnly: true,
|
||
});
|
||
const apiKey = resolveAlibabaExtendApiKey(alibabaProviders, config.apiKey);
|
||
if (!apiKey) {
|
||
throw new ExtendServiceError(
|
||
"图像拓展API密钥未配置，请在AI模型设置中配置阿里云官方生图服务商",
|
||
400,
|
||
);
|
||
}
|
||
|
||
return {
|
||
...config,
|
||
provider: "dashscope",
|
||
apiKey,
|
||
};
|
||
}
|
||
|
||
export async function executeExtendJob(
|
||
params: ExecuteExtendJobParams,
|
||
): Promise<ExecuteExtendJobResult> {
|
||
const {
|
||
userId,
|
||
imageUrl = null,
|
||
imageBase64 = null,
|
||
prompt = "图像拓展处理",
|
||
chatId = null,
|
||
messageId = null,
|
||
includeBase64 = false,
|
||
skipGallerySave = false,
|
||
sourceImageUrl = imageUrl || null,
|
||
requestId = null,
|
||
} = params;
|
||
|
||
if (!imageUrl && !imageBase64) {
|
||
throw new ExtendServiceError("缺少 image_url 或 image_base64 参数", 400);
|
||
}
|
||
|
||
const config = await getValidatedExtendConfig();
|
||
const xScale = Math.max(1.1, Math.min(Number(params.xScale || config.xScale || 1.5), 4));
|
||
const yScale = Math.max(1.1, Math.min(Number(params.yScale || config.yScale || 1.5), 4));
|
||
const inputImageUrl = imageUrl || (imageBase64 ? await uploadImageToS3ForAPI(imageBase64) : "");
|
||
|
||
if (!inputImageUrl) {
|
||
throw new ExtendServiceError("无法获取输入图片URL", 400);
|
||
}
|
||
|
||
const startTime = Date.now();
|
||
let buffer!: Buffer;
|
||
let uploadResult: { thumbnail: string; original: string } | undefined;
|
||
let archivedBytes: number | undefined;
|
||
let downloadAsset: Awaited<ReturnType<typeof archiveTemporaryImage>>["downloadAsset"];
|
||
let modelId: string;
|
||
if (config.provider === "fal") {
|
||
buffer = await expandWithFalOutpaint({ imageUrl: inputImageUrl, apiKey: config.apiKey!, xScale, yScale });
|
||
modelId = FAL_OUTPAINT_MODEL;
|
||
} else if (config.provider === "vip98k" && config.modelId === VIP98K_OUTPAINT_MODEL) {
|
||
if (!config.routeOptionId) throw new ExtendServiceError("98K 扩图线路未配置", 503);
|
||
const provider = await getVip98kRuntimeProvider();
|
||
buffer = await expandWithVip98kOutpaint({
|
||
imageUrl: inputImageUrl, baseUrl: provider.baseUrl, apiKey: provider.apiKey,
|
||
routeOptionId: config.routeOptionId, xScale, yScale,
|
||
});
|
||
modelId = VIP98K_OUTPAINT_MODEL;
|
||
} else {
|
||
const prepared = config.provider === "vip98k" ? inputImageUrl : await normalizeDashscopeExpansionInput(inputImageUrl);
|
||
const url = await extendImage(prepared, config, xScale, yScale, requestId);
|
||
if (config.provider === "vip98k" && !includeBase64) {
|
||
const archived = await archiveTemporaryImage(url, requestId || randomUUID());
|
||
uploadResult = { original: archived.url, thumbnail: archived.thumbnailUrl };
|
||
archivedBytes = archived.bytes;
|
||
downloadAsset = archived.downloadAsset;
|
||
} else {
|
||
const imageResponse = await fetch(url);
|
||
if (!imageResponse.ok) {
|
||
throw new ExtendServiceError("下载拓展图片失败", 500, `${imageResponse.status} ${imageResponse.statusText}`);
|
||
}
|
||
buffer = Buffer.from(await imageResponse.arrayBuffer());
|
||
}
|
||
modelId = config.provider === "vip98k" ? "image-expansion" : "extend-1.0";
|
||
}
|
||
const imageBase64Result = includeBase64 ? buffer.toString("base64") : undefined;
|
||
uploadResult ??= await uploadBufferPairToS3(buffer, config.provider === "fal" || config.provider === "vip98k" ? "png" : "jpg");
|
||
console.log("[Extend Service] Result uploaded:", {
|
||
requestId,
|
||
includeBase64,
|
||
inputImageUrl: inputImageUrl.slice(0, 120),
|
||
modelId,
|
||
outputBytes: archivedBytes ?? buffer.length,
|
||
});
|
||
let mediaAssetId: string | null = null;
|
||
let generatedImageId: string | null = null;
|
||
|
||
if (!skipGallerySave) {
|
||
try {
|
||
const [savedImage] = await db
|
||
.insert(generatedImage)
|
||
.values({
|
||
userId,
|
||
chatId,
|
||
messageId,
|
||
thumbnailUrl: uploadResult.thumbnail,
|
||
originalUrl: uploadResult.original,
|
||
prompt,
|
||
modelId,
|
||
imageSize: "2K",
|
||
})
|
||
.returning();
|
||
generatedImageId = savedImage?.id || null;
|
||
|
||
const sourceAssets = sourceImageUrl
|
||
? await getMediaAssetsByUserAndUrls({
|
||
userId,
|
||
urls: [sourceImageUrl],
|
||
})
|
||
: [];
|
||
|
||
const media = await upsertMediaAsset({
|
||
userId,
|
||
chatId,
|
||
mediaType: "image",
|
||
sourceType: "edited",
|
||
status: "ready",
|
||
originalUrl: uploadResult.original,
|
||
previewUrl: uploadResult.original,
|
||
thumbnailUrl: uploadResult.thumbnail,
|
||
displayName: prompt.slice(0, 80),
|
||
prompt,
|
||
modelId,
|
||
sourceMessageId: messageId,
|
||
sourceTable: "GeneratedImage",
|
||
sourceRowId: generatedImageId,
|
||
primaryParentAssetId: sourceAssets[0]?.id ?? null,
|
||
metadata: {
|
||
xScale,
|
||
yScale,
|
||
imageSize: "2K",
|
||
...(downloadAsset ? { downloadAsset } : {}),
|
||
},
|
||
});
|
||
mediaAssetId = media?.id || null;
|
||
|
||
if (media?.id && sourceAssets.length > 0) {
|
||
for (const sourceAsset of sourceAssets) {
|
||
await createMediaAssetRelation({
|
||
userId,
|
||
chatId,
|
||
fromAssetId: sourceAsset.id,
|
||
toAssetId: media.id,
|
||
relationType: "edited_from",
|
||
sourceMessageId: messageId,
|
||
metadata: {
|
||
prompt,
|
||
tool: "extend",
|
||
xScale,
|
||
yScale,
|
||
},
|
||
});
|
||
}
|
||
}
|
||
} catch (error) {
|
||
console.error("[Extend Service] Failed to save result:", error);
|
||
throw new ExtendServiceError(
|
||
"图像拓展成功但保存结果失败",
|
||
500,
|
||
error instanceof Error ? error.message : "未知错误",
|
||
);
|
||
}
|
||
}
|
||
|
||
return {
|
||
image_base64: imageBase64Result,
|
||
thumbnail_url: uploadResult.thumbnail,
|
||
original_url: uploadResult.original,
|
||
processing_time: Number(((Date.now() - startTime) / 1000).toFixed(2)),
|
||
mediaAssetId,
|
||
generatedImageId,
|
||
};
|
||
}
|