feat: upgrade agent executor and canvas sync
This commit is contained in:
Your Name
parent
3badf859b9
commit
864b6d41e3
File diff suppressed because it is too large
Load Diff
@ -1,465 +1,110 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { getSystemSetting } from "@/lib/db/queries";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { db } from "@/lib/db";
|
||||
import { generatedImage } from "@/lib/db/schema";
|
||||
import {
|
||||
executeExtendJob,
|
||||
ExtendServiceError,
|
||||
} from "@/lib/ai/extend/service";
|
||||
|
||||
/**
|
||||
* 压缩图片生成缩略图（使用 sharp）
|
||||
*/
|
||||
async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer> {
|
||||
try {
|
||||
const sharp = await import("sharp");
|
||||
|
||||
// 创建缩略图（宽度 500px，高度按比例，质量 80%）
|
||||
const thumbnailBuffer = await sharp.default(buffer)
|
||||
.resize(500, null, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality: 80 }) // JPEG格式
|
||||
.toBuffer();
|
||||
|
||||
console.log(`[Extend] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`);
|
||||
return thumbnailBuffer;
|
||||
} catch (error) {
|
||||
console.error("[Extend] Failed to create thumbnail:", error);
|
||||
// 如果失败，返回原图
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 base64 图片上传到 S3，返回缩略图和原图的 URL
|
||||
*/
|
||||
async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jpeg"): Promise<{ thumbnail: string; original: string }> {
|
||||
try {
|
||||
// 解析 base64 数据
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
|
||||
// 生成文件名
|
||||
const ext = mimeType.split("/")[1] || "jpg";
|
||||
const timestamp = Date.now();
|
||||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||||
const thumbnailFilename = `extend-thumb-${timestamp}-${randomStr}.${ext}`;
|
||||
const originalFilename = `extend-orig-${timestamp}-${randomStr}.${ext}`;
|
||||
|
||||
console.log(`[Extend] Original size: ${buffer.length} bytes`);
|
||||
|
||||
// 生成缩略图
|
||||
const thumbnailBuffer = await createThumbnail(buffer, mimeType);
|
||||
|
||||
// 转换为 ArrayBuffer
|
||||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
|
||||
// 并行上传缩略图和原图到 S3
|
||||
console.log(`[Extend] Uploading thumbnail and original to S3...`);
|
||||
const [thumbnailResult, originalResult] = await Promise.all([
|
||||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||||
uploadFile(originalFilename, originalArrayBuffer),
|
||||
]);
|
||||
|
||||
console.log(`[Extend] Uploaded to S3:`);
|
||||
console.log(` - Thumbnail: ${thumbnailResult.url}`);
|
||||
console.log(` - Original: ${originalResult.url}`);
|
||||
|
||||
return {
|
||||
thumbnail: thumbnailResult.url,
|
||||
original: originalResult.url,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[Extend] Failed to upload to S3:", error);
|
||||
throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`);
|
||||
}
|
||||
}
|
||||
|
||||
// 设置API路由最大执行时间为3分钟（180秒）
|
||||
export const maxDuration = 180;
|
||||
|
||||
/**
|
||||
* 将图片上传到S3并返回公网URL（用于API调用）
|
||||
*/
|
||||
async function uploadImageToS3ForAPI(imageBase64: string, mimeType: string = "image/jpeg"): Promise<string> {
|
||||
try {
|
||||
// 解析 base64 数据
|
||||
const buffer = Buffer.from(imageBase64, "base64");
|
||||
|
||||
// 生成文件名
|
||||
const ext = mimeType.split("/")[1] || "jpg";
|
||||
const timestamp = Date.now();
|
||||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||||
const filename = `extend-input-${timestamp}-${randomStr}.${ext}`;
|
||||
|
||||
console.log(`[Extend API] 上传原图到S3，大小: ${(buffer.length / 1024).toFixed(2)}KB`);
|
||||
|
||||
// 转换为 ArrayBuffer
|
||||
const arrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
|
||||
// 上传到 S3
|
||||
const result = await uploadFile(filename, arrayBuffer);
|
||||
|
||||
console.log(`[Extend API] ✅ 原图已上传到S3: ${result.url}`);
|
||||
|
||||
return result.url;
|
||||
} catch (error) {
|
||||
console.error("[Extend API] 上传到S3失败:", error);
|
||||
throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 图像拓展核心实现（使用URL方式，避免Base64长度限制）
|
||||
* @param imageUrl - 图片的公网URL
|
||||
* @param apiKey - Dashscope API Key
|
||||
* @param xScale - X轴放大倍数（默认1.5）
|
||||
* @param yScale - Y轴放大倍数（默认1.5）
|
||||
* @returns 拓展后的图片URL
|
||||
*/
|
||||
async function extendImage(
|
||||
imageUrl: string,
|
||||
apiKey: string,
|
||||
xScale: number = 1.5,
|
||||
yScale: number = 1.5
|
||||
): Promise<string> {
|
||||
console.log("[Extend API] 开始图像拓展...");
|
||||
console.log("[Extend API] 使用图片URL:", imageUrl);
|
||||
|
||||
// 1. 提交拓展任务（使用URL方式，避免Base64长度限制）
|
||||
const requestBody = {
|
||||
model: "image-out-painting",
|
||||
input: {
|
||||
image_url: imageUrl,
|
||||
},
|
||||
parameters: {
|
||||
x_scale: xScale,
|
||||
y_scale: yScale,
|
||||
best_quality: false,
|
||||
limit_image_size: true,
|
||||
add_watermark: false,
|
||||
},
|
||||
};
|
||||
|
||||
console.log("[Extend API] 提交拓展任务...");
|
||||
const response = await fetch(
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/image2image/out-painting",
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"X-DashScope-Async": "enable",
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
}
|
||||
);
|
||||
|
||||
const initialResult = await response.json();
|
||||
|
||||
if (initialResult.code) {
|
||||
console.error("[Extend API] API调用失败:", initialResult);
|
||||
throw new Error(
|
||||
`图像拓展API错误: ${initialResult.message || JSON.stringify(initialResult)}`
|
||||
);
|
||||
}
|
||||
|
||||
// 2. 获取任务ID
|
||||
const taskId = initialResult.output.task_id;
|
||||
const taskStatusUrl = `https://dashscope.aliyuncs.com/api/v1/tasks/${taskId}`;
|
||||
console.log(`[Extend API] 任务已提交，任务ID: ${taskId}`);
|
||||
|
||||
// 3. 等待3秒后开始轮询
|
||||
console.log("[Extend API] 等待API初始化...");
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
|
||||
// 4. 轮询获取结果
|
||||
let attempts = 0;
|
||||
const maxAttempts = 60; // 最多轮询60次
|
||||
const pollInterval = 2000; // 每2秒轮询一次
|
||||
|
||||
console.log("[Extend API] 开始轮询任务状态...");
|
||||
while (attempts < maxAttempts) {
|
||||
const statusResponse = await fetch(taskStatusUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await statusResponse.json();
|
||||
|
||||
if (attempts === 0 || attempts % 5 === 0) {
|
||||
const elapsedTime = attempts * 2 + 3;
|
||||
console.log(`[Extend API] [${elapsedTime}秒] 任务状态: ${result.output?.task_status || "UNKNOWN"}`);
|
||||
}
|
||||
|
||||
if (result.output?.task_status === "SUCCEEDED") {
|
||||
const elapsedTime = attempts * 2 + 3;
|
||||
console.log(`[Extend API] ✅ 任务完成（总用时 ${elapsedTime}秒）`);
|
||||
return result.output.output_image_url;
|
||||
} else if (result.output?.task_status === "FAILED") {
|
||||
console.error("[Extend API] 任务失败:", result.output);
|
||||
throw new Error(
|
||||
`图像拓展任务失败: ${result.output.message || result.output.code || "未知错误"}`
|
||||
);
|
||||
}
|
||||
|
||||
attempts++;
|
||||
if (attempts < maxAttempts) {
|
||||
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`图像拓展任务超时（等待${maxAttempts * 2 + 3}秒）`);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 图像拓展API路由
|
||||
* 从数据库读取图像拓展API配置，调用Dashscope API进行图像拓展处理
|
||||
* 将处理后的图片上传到S3并保存到数据库
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
console.log("[Extend API] ===== 收到请求 =====");
|
||||
try {
|
||||
// 0. 获取用户会话
|
||||
console.log("[Extend API] 开始获取用户会话...");
|
||||
const session = await auth();
|
||||
console.log("[Extend API] 用户会话获取完成:", session?.user?.id ? "已登录" : "未登录");
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json(
|
||||
{ error: "未授权，请先登录" },
|
||||
{ status: 401 }
|
||||
);
|
||||
return NextResponse.json({ error: "未授权，请先登录" }, { status: 401 });
|
||||
}
|
||||
|
||||
const userId = session.user.id;
|
||||
console.log("[Extend API] 用户ID:", userId);
|
||||
|
||||
// 1. 从数据库获取图像拓展API配置
|
||||
console.log("[Extend API] 开始获取配置...");
|
||||
const extendConfig = await getSystemSetting({ key: "extend_api_config" });
|
||||
console.log("[Extend API] 配置获取完成:", extendConfig ? "已找到" : "未找到");
|
||||
|
||||
if (!extendConfig || !extendConfig.value) {
|
||||
return NextResponse.json(
|
||||
{ error: "图像拓展API配置未设置，请在系统设置中配置" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const config = extendConfig.value as {
|
||||
apiKey?: string;
|
||||
xScale?: number;
|
||||
yScale?: number;
|
||||
};
|
||||
|
||||
if (!config.apiKey) {
|
||||
return NextResponse.json(
|
||||
{ error: "图像拓展API密钥未配置" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// 2. 获取请求体（Base64图片或FormData）
|
||||
console.log("[Extend API] 开始获取请求体...");
|
||||
const contentType = request.headers.get("content-type") || "";
|
||||
console.log("[Extend API] Content-Type:", contentType);
|
||||
|
||||
let imageBase64: string;
|
||||
let prompt = ""; // 用户提示词（可选）
|
||||
let chatId: string | null = null; // 聊天ID（可选）
|
||||
|
||||
let imageUrl: string | null = null;
|
||||
let imageBase64: string | null = null;
|
||||
let prompt = "图像拓展处理";
|
||||
let chatId: string | null = null;
|
||||
let messageId: string | null = null;
|
||||
|
||||
if (contentType.includes("application/json")) {
|
||||
// JSON格式：支持Base64图片或图片URL
|
||||
const body = await request.json();
|
||||
const imageUrl = body.image_url; // 优先使用URL
|
||||
imageBase64 = body.image_base64; // 如果没有URL，使用Base64
|
||||
prompt = body.prompt || "图像拓展处理";
|
||||
imageUrl = body.image_url ? String(body.image_url).trim() : null;
|
||||
imageBase64 = body.image_base64 ? String(body.image_base64).trim() : null;
|
||||
prompt = body.prompt || prompt;
|
||||
chatId = body.chatId || null;
|
||||
messageId = body.messageId || null;
|
||||
|
||||
// 如果提供了URL，直接使用；否则需要Base64
|
||||
if (imageUrl) {
|
||||
console.log("[Extend API] 使用图片URL:", imageUrl);
|
||||
// 直接使用URL，跳过Base64转换和上传步骤
|
||||
const startTime = Date.now();
|
||||
const xScale = config.xScale || 1.5;
|
||||
const yScale = config.yScale || 1.5;
|
||||
|
||||
console.log(`[Extend API] 拓展参数: X=${xScale}, Y=${yScale}`);
|
||||
|
||||
const extendedImageUrl = await extendImage(imageUrl, config.apiKey, xScale, yScale);
|
||||
console.log("[Extend API] 拓展后图片URL:", extendedImageUrl);
|
||||
|
||||
// 下载拓展后的图片
|
||||
console.log("[Extend API] 下载拓展后的图片...");
|
||||
const imageResponse = await fetch(extendedImageUrl);
|
||||
if (!imageResponse.ok) {
|
||||
throw new Error(`下载拓展图片失败: ${imageResponse.status} ${imageResponse.statusText}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await imageResponse.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const extendedImageBase64 = buffer.toString("base64");
|
||||
|
||||
// 上传到S3
|
||||
console.log("[Extend API] 上传到S3...");
|
||||
const { thumbnail, original } = await uploadBase64ToS3(extendedImageBase64, "image/jpeg");
|
||||
|
||||
// 保存到数据库（相册和生图记录）
|
||||
console.log("[Extend API] 保存到数据库...");
|
||||
let savedImage;
|
||||
try {
|
||||
[savedImage] = await db
|
||||
.insert(generatedImage)
|
||||
.values({
|
||||
userId,
|
||||
chatId: chatId || null,
|
||||
messageId: messageId || null,
|
||||
thumbnailUrl: thumbnail,
|
||||
originalUrl: original,
|
||||
prompt: prompt || "图像拓展处理",
|
||||
modelId: "extend-1.0",
|
||||
imageSize: "2K",
|
||||
})
|
||||
.returning();
|
||||
console.log(`[Extend API] ✅ 图片已保存到相册和生图记录，ID: ${savedImage.id}`);
|
||||
} catch (dbError) {
|
||||
console.error("[Extend API] ❌ 保存到数据库失败:", dbError);
|
||||
}
|
||||
|
||||
const processingTime = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
console.log(`[Extend API] ✅ 图像拓展完成，处理时间: ${processingTime}秒`);
|
||||
|
||||
// 返回结果
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
original_url: original,
|
||||
thumbnail_url: thumbnail,
|
||||
base64: extendedImageBase64,
|
||||
processing_time: parseFloat(processingTime),
|
||||
});
|
||||
}
|
||||
|
||||
if (!imageBase64) {
|
||||
if (!imageUrl && !imageBase64) {
|
||||
return NextResponse.json(
|
||||
{ error: "缺少image_base64或image_url参数" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
} else if (contentType.includes("multipart/form-data")) {
|
||||
// FormData格式：文件上传
|
||||
const formData = await request.formData();
|
||||
const file = formData.get("file") as File;
|
||||
|
||||
if (!file) {
|
||||
return NextResponse.json(
|
||||
{ error: "缺少file参数" },
|
||||
{ status: 400 }
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
// 将文件转换为Base64
|
||||
const result = await executeExtendJob({
|
||||
userId: session.user.id,
|
||||
imageUrl,
|
||||
imageBase64,
|
||||
prompt,
|
||||
chatId,
|
||||
messageId,
|
||||
xScale: body.xScale ? Number(body.xScale) : undefined,
|
||||
yScale: body.yScale ? Number(body.yScale) : undefined,
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
original_url: result.original_url,
|
||||
thumbnail_url: result.thumbnail_url,
|
||||
base64: result.image_base64,
|
||||
processing_time: result.processing_time,
|
||||
});
|
||||
}
|
||||
|
||||
if (contentType.includes("multipart/form-data")) {
|
||||
const formData = await request.formData();
|
||||
const file = formData.get("file") as File | null;
|
||||
if (!file) {
|
||||
return NextResponse.json({ error: "缺少file参数" }, { status: 400 });
|
||||
}
|
||||
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
imageBase64 = buffer.toString("base64");
|
||||
prompt = (formData.get("prompt") as string) || "图像拓展处理";
|
||||
chatId = (formData.get("chatId") as string) || null;
|
||||
messageId = (formData.get("messageId") as string) || null;
|
||||
} else {
|
||||
const result = await executeExtendJob({
|
||||
userId: session.user.id,
|
||||
imageBase64: buffer.toString("base64"),
|
||||
prompt: String(formData.get("prompt") || prompt),
|
||||
chatId: String(formData.get("chatId") || "") || null,
|
||||
messageId: String(formData.get("messageId") || "") || null,
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
original_url: result.original_url,
|
||||
thumbnail_url: result.thumbnail_url,
|
||||
base64: result.image_base64,
|
||||
processing_time: result.processing_time,
|
||||
});
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: "不支持的Content-Type，请使用application/json或multipart/form-data" },
|
||||
{ status: 400 },
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof ExtendServiceError) {
|
||||
return NextResponse.json(
|
||||
{ error: "不支持的Content-Type，请使用application/json或multipart/form-data" },
|
||||
{ status: 400 }
|
||||
{
|
||||
error: error.message,
|
||||
detail: error.detail || error.message,
|
||||
},
|
||||
{ status: error.status },
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`[Extend API] 图片Base64长度: ${imageBase64.length} 字符`);
|
||||
|
||||
// 3. 先将图片上传到S3获取公网URL（使用URL方式避免Base64长度限制）
|
||||
console.log("[Extend API] 上传原图到S3获取公网URL...");
|
||||
const inputImageUrl = await uploadImageToS3ForAPI(imageBase64, "image/jpeg");
|
||||
console.log("[Extend API] 原图URL:", inputImageUrl);
|
||||
|
||||
// 4. 调用图像拓展API（使用URL方式）
|
||||
const startTime = Date.now();
|
||||
const xScale = config.xScale || 1.5;
|
||||
const yScale = config.yScale || 1.5;
|
||||
|
||||
console.log(`[Extend API] 拓展参数: X=${xScale}, Y=${yScale}`);
|
||||
|
||||
const extendedImageUrl = await extendImage(inputImageUrl, config.apiKey, xScale, yScale);
|
||||
console.log("[Extend API] 拓展后图片URL:", extendedImageUrl);
|
||||
|
||||
// 4. 下载拓展后的图片
|
||||
console.log("[Extend API] 下载拓展后的图片...");
|
||||
const imageResponse = await fetch(extendedImageUrl);
|
||||
if (!imageResponse.ok) {
|
||||
throw new Error(`下载拓展图片失败: ${imageResponse.status} ${imageResponse.statusText}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await imageResponse.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const extendedImageBase64 = buffer.toString("base64");
|
||||
|
||||
// 5. 上传到S3
|
||||
console.log("[Extend API] 上传到S3...");
|
||||
const { thumbnail, original } = await uploadBase64ToS3(extendedImageBase64, "image/jpeg");
|
||||
|
||||
// 6. 保存到数据库（相册和生图记录）
|
||||
console.log("[Extend API] 保存到数据库...");
|
||||
let savedImage;
|
||||
try {
|
||||
[savedImage] = await db
|
||||
.insert(generatedImage)
|
||||
.values({
|
||||
userId,
|
||||
chatId: chatId || null,
|
||||
messageId: messageId || null,
|
||||
thumbnailUrl: thumbnail,
|
||||
originalUrl: original,
|
||||
prompt: prompt || "图像拓展处理",
|
||||
modelId: "extend-1.0",
|
||||
imageSize: "2K",
|
||||
})
|
||||
.returning();
|
||||
console.log(`[Extend API] ✅ 图片已保存到相册和生图记录，ID: ${savedImage.id}`);
|
||||
} catch (dbError) {
|
||||
console.error("[Extend API] ❌ 保存到数据库失败:", dbError);
|
||||
// 数据库保存失败时仍然返回成功，但记录错误（图片已上传到S3，用户可以访问）
|
||||
}
|
||||
|
||||
const processingTime = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
console.log(`[Extend API] ✅ 图像拓展完成，处理时间: ${processingTime}秒`);
|
||||
|
||||
// 7. 返回结果
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
original_url: original,
|
||||
thumbnail_url: thumbnail,
|
||||
base64: extendedImageBase64,
|
||||
processing_time: parseFloat(processingTime),
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Extend API] 错误:", error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "图像拓展失败",
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
{ status: 500 }
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,225 +1,66 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { db } from "@/lib/db";
|
||||
import { generatedImage } from "@/lib/db/schema";
|
||||
import { upsertMediaAsset } from "@/lib/db/queries";
|
||||
import {
|
||||
executeResizeJob,
|
||||
ResizeServiceError,
|
||||
} from "@/lib/ai/resize/service";
|
||||
|
||||
/**
|
||||
* 压缩图片生成缩略图（使用 sharp）
|
||||
*/
|
||||
async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer> {
|
||||
try {
|
||||
const sharp = await import("sharp");
|
||||
|
||||
// 创建缩略图（宽度 500px，高度按比例，质量 80%）
|
||||
const thumbnailBuffer = await sharp.default(buffer)
|
||||
.resize(500, null, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.png({ quality: 80 }) // PNG格式保持透明背景
|
||||
.toBuffer();
|
||||
|
||||
console.log(`[Resize] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`);
|
||||
return thumbnailBuffer;
|
||||
} catch (error) {
|
||||
console.error("[Resize] Failed to create thumbnail:", error);
|
||||
// 如果失败，返回原图
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 base64 图片上传到 S3，返回缩略图和原图的 URL
|
||||
*/
|
||||
async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/png"): Promise<{ thumbnail: string; original: string }> {
|
||||
try {
|
||||
// 解析 base64 数据
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
|
||||
// 生成文件名
|
||||
const ext = mimeType.split("/")[1] || "png";
|
||||
const timestamp = Date.now();
|
||||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||||
const thumbnailFilename = `resize-thumb-${timestamp}-${randomStr}.${ext}`;
|
||||
const originalFilename = `resize-orig-${timestamp}-${randomStr}.${ext}`;
|
||||
|
||||
console.log(`[Resize] Original size: ${buffer.length} bytes`);
|
||||
|
||||
// 生成缩略图
|
||||
const thumbnailBuffer = await createThumbnail(buffer, mimeType);
|
||||
|
||||
// 转换为 ArrayBuffer
|
||||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
|
||||
// 并行上传缩略图和原图到 S3
|
||||
console.log(`[Resize] Uploading thumbnail and original to S3...`);
|
||||
const [thumbnailResult, originalResult] = await Promise.all([
|
||||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||||
uploadFile(originalFilename, originalArrayBuffer),
|
||||
]);
|
||||
|
||||
console.log(`[Resize] Uploaded to S3:`);
|
||||
console.log(` - Thumbnail: ${thumbnailResult.url}`);
|
||||
console.log(` - Original: ${originalResult.url}`);
|
||||
|
||||
return {
|
||||
thumbnail: thumbnailResult.url,
|
||||
original: originalResult.url,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[Resize] Failed to upload to S3:", error);
|
||||
throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 尺寸调整API路由
|
||||
* 接收前端处理后的Base64图片，上传到S3并保存到数据库
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
// 0. 获取用户会话
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json(
|
||||
{ error: "未授权，请先登录" },
|
||||
{ status: 401 }
|
||||
);
|
||||
return NextResponse.json({ error: "未授权，请先登录" }, { status: 401 });
|
||||
}
|
||||
|
||||
const userId = session.user.id;
|
||||
|
||||
// 1. 获取请求体
|
||||
const contentType = request.headers.get("content-type") || "";
|
||||
|
||||
let imageBase64: string;
|
||||
let prompt = "尺寸调整处理";
|
||||
let chatId: string | null = null;
|
||||
let messageId: string | null = null;
|
||||
let canvasWidth: number;
|
||||
let canvasHeight: number;
|
||||
let mimeType = "image/png";
|
||||
|
||||
if (contentType.includes("application/json")) {
|
||||
const body = await request.json();
|
||||
imageBase64 = body.image_base64;
|
||||
prompt = body.prompt || "尺寸调整处理";
|
||||
chatId = body.chatId || null;
|
||||
messageId = body.messageId || null;
|
||||
canvasWidth = body.canvasWidth;
|
||||
canvasHeight = body.canvasHeight;
|
||||
mimeType = body.mimeType || "image/png";
|
||||
|
||||
if (!imageBase64) {
|
||||
return NextResponse.json(
|
||||
{ error: "缺少image_base64参数" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (!canvasWidth || !canvasHeight) {
|
||||
return NextResponse.json(
|
||||
{ error: "缺少canvasWidth或canvasHeight参数" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if (!contentType.includes("application/json")) {
|
||||
return NextResponse.json(
|
||||
{ error: "不支持的Content-Type，请使用application/json" },
|
||||
{ status: 400 }
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`[Resize API] 图片Base64长度: ${imageBase64.length} 字符`);
|
||||
console.log(`[Resize API] 画布尺寸: ${canvasWidth}x${canvasHeight}`);
|
||||
const body = await request.json();
|
||||
const imageBase64 = String(body.image_base64 || "").trim();
|
||||
const canvasWidth = Number(body.canvasWidth);
|
||||
const canvasHeight = Number(body.canvasHeight);
|
||||
|
||||
// 2. 上传处理后的图片到S3存储桶
|
||||
let thumbnailUrl: string;
|
||||
let originalUrl: string;
|
||||
|
||||
try {
|
||||
const uploadResult = await uploadBase64ToS3(imageBase64, mimeType);
|
||||
thumbnailUrl = uploadResult.thumbnail;
|
||||
originalUrl = uploadResult.original;
|
||||
console.log(`[Resize] ✅ Images uploaded to S3 successfully`);
|
||||
} catch (uploadError) {
|
||||
console.error("[Resize] ❌ Failed to upload to S3:", uploadError);
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "图片上传到存储桶失败",
|
||||
detail: uploadError instanceof Error ? uploadError.message : "未知错误"
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
if (!imageBase64) {
|
||||
return NextResponse.json({ error: "缺少image_base64参数" }, { status: 400 });
|
||||
}
|
||||
|
||||
// 3. 保存图片记录到数据库（相册和生图记录）
|
||||
try {
|
||||
const [savedImage] = await db.insert(generatedImage).values({
|
||||
userId,
|
||||
chatId: chatId || null,
|
||||
messageId: messageId || null,
|
||||
thumbnailUrl,
|
||||
originalUrl,
|
||||
imageSize: `${canvasWidth}x${canvasHeight}`, // 使用实际尺寸
|
||||
prompt: prompt || "尺寸调整处理",
|
||||
modelId: "resize-1.0", // 标记为尺寸调整处理
|
||||
}).returning();
|
||||
console.log(`[Resize] ✅ 图片已保存到相册和生图记录，ID: ${savedImage.id}`);
|
||||
const result = await executeResizeJob({
|
||||
userId: session.user.id,
|
||||
imageBuffer: Buffer.from(imageBase64, "base64"),
|
||||
prompt: body.prompt || "尺寸调整处理",
|
||||
chatId: body.chatId || null,
|
||||
messageId: body.messageId || null,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
mimeType: body.mimeType || "image/png",
|
||||
transparentBackground: body.transparentBackground === true,
|
||||
backgroundColor: body.backgroundColor || undefined,
|
||||
});
|
||||
|
||||
try {
|
||||
await upsertMediaAsset({
|
||||
userId,
|
||||
chatId,
|
||||
mediaType: "image",
|
||||
sourceType: "edited",
|
||||
status: "ready",
|
||||
originalUrl,
|
||||
previewUrl: originalUrl,
|
||||
thumbnailUrl,
|
||||
displayName: (prompt || "尺寸调整处理").slice(0, 80),
|
||||
prompt: prompt || "尺寸调整处理",
|
||||
modelId: "resize-1.0",
|
||||
sourceMessageId: messageId,
|
||||
sourceTable: "GeneratedImage",
|
||||
sourceRowId: savedImage.id,
|
||||
metadata: {
|
||||
imageSize: `${canvasWidth}x${canvasHeight}`,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
},
|
||||
});
|
||||
} catch (assetError) {
|
||||
console.error("[Resize] ⚠️ MediaAsset 注册失败:", assetError);
|
||||
}
|
||||
} catch (dbError) {
|
||||
console.error("[Resize] ❌ 保存到数据库失败:", dbError);
|
||||
// 数据库保存失败时仍然返回成功，但记录错误（图片已上传到S3，用户可以访问）
|
||||
}
|
||||
|
||||
// 4. 返回结果（包含S3 URL和Base64，前端可以选择使用）
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
image_base64: imageBase64, // 保留Base64用于前端立即显示
|
||||
thumbnail_url: thumbnailUrl, // S3缩略图URL
|
||||
original_url: originalUrl, // S3原图URL
|
||||
canvas_width: canvasWidth,
|
||||
canvas_height: canvasHeight,
|
||||
...result,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ResizeServiceError) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: error.message,
|
||||
detail: error.detail || error.message,
|
||||
},
|
||||
{ status: error.status },
|
||||
);
|
||||
}
|
||||
|
||||
console.error("[Resize API] 处理错误:", error);
|
||||
return NextResponse.json(
|
||||
{ error: "服务器内部错误", detail: error instanceof Error ? error.message : "未知错误" },
|
||||
{ status: 500 }
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -82,6 +82,21 @@ interface TagPreviewInfo {
|
||||
};
|
||||
}
|
||||
|
||||
type CanvasMediaStreamPayload = {
|
||||
messageId?: string;
|
||||
imageUrls?: string[];
|
||||
videoUrls?: string[];
|
||||
prompt?: string;
|
||||
sourceImageUrls?: string[];
|
||||
};
|
||||
|
||||
type ImageGenerationStatusStreamPayload = {
|
||||
phase?: "queued" | "processing" | "finalizing" | "failed" | "completed";
|
||||
providerType?: string;
|
||||
modelName?: string;
|
||||
imageCount?: number;
|
||||
};
|
||||
|
||||
interface CanvasChatPanelProps {
|
||||
chatId: string;
|
||||
initialMessages?: ChatMessage[];
|
||||
@ -1665,6 +1680,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
unitCost: number;
|
||||
imageCount: number;
|
||||
} | null>(null);
|
||||
const [streamImageGenerationState, setStreamImageGenerationState] =
|
||||
useState<ImageGenerationStatusStreamPayload | null>(null);
|
||||
const imageParamsRef = useRef(imageParams);
|
||||
const videoParamsRef = useRef(videoParams);
|
||||
const deepThinkingEnabledRef = useRef(deepThinkingEnabled);
|
||||
@ -2723,22 +2740,27 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
currentModel.isMultimodal &&
|
||||
!VIDEO_PROVIDER_TYPES.has(currentModel.providerType)
|
||||
);
|
||||
const isAgentModel = !isDetailGenerationMode && (selectedModelId === "auto" || isConversationModel);
|
||||
const agentSelectableModels = useMemo(
|
||||
() => [SMART_AUTO_MODEL, ...conversationOnlyModels],
|
||||
[SMART_AUTO_MODEL, conversationOnlyModels]
|
||||
);
|
||||
const selectableModels = useMemo(() => {
|
||||
if (isDetailGenerationMode) {
|
||||
return [SMART_AUTO_MODEL];
|
||||
}
|
||||
if (isConversationModel) {
|
||||
return conversationOnlyModels;
|
||||
}
|
||||
if (isVideoModel) {
|
||||
return videoOnlyModels;
|
||||
}
|
||||
return [SMART_AUTO_MODEL, ...imageOnlyModels];
|
||||
if (isAgentModel) {
|
||||
return agentSelectableModels;
|
||||
}
|
||||
return imageOnlyModels;
|
||||
}, [
|
||||
SMART_AUTO_MODEL,
|
||||
conversationOnlyModels,
|
||||
agentSelectableModels,
|
||||
imageOnlyModels,
|
||||
isConversationModel,
|
||||
isAgentModel,
|
||||
isDetailGenerationMode,
|
||||
isVideoModel,
|
||||
videoOnlyModels,
|
||||
@ -2751,8 +2773,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
currentModel ||
|
||||
SMART_AUTO_MODEL;
|
||||
const agentDisplayName =
|
||||
modelsData?.agentModel?.displayName || selectDisplayModel?.displayName || "Agent模式";
|
||||
const agentAvatarUrl = modelsData?.agentModel?.avatarUrl || selectDisplayModel?.avatarUrl || null;
|
||||
currentModel?.displayName || selectDisplayModel?.displayName || "Agent模式";
|
||||
const agentAvatarUrl = currentModel?.avatarUrl || selectDisplayModel?.avatarUrl || null;
|
||||
const videoFrameSlots = useMemo(() => {
|
||||
if (!isVideoModel || !videoParams) return [] as string[];
|
||||
if (videoParams.mode === "first-last-frame") return ["首帧", "尾帧"] as string[];
|
||||
@ -2901,28 +2923,21 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
() => imageOnlyModels[0]?.id || null,
|
||||
[imageOnlyModels]
|
||||
);
|
||||
const firstConversationModelId = useMemo(
|
||||
() => conversationOnlyModels[0]?.id || null,
|
||||
[conversationOnlyModels]
|
||||
);
|
||||
const firstVideoModelId = useMemo(
|
||||
() => videoOnlyModels[0]?.id || null,
|
||||
[videoOnlyModels]
|
||||
);
|
||||
const activeComposerMode = useMemo<"agent" | "chat" | "image" | "video" | "detail">(() => {
|
||||
const activeComposerMode = useMemo<"agent" | "image" | "video" | "detail">(() => {
|
||||
if (isDetailGenerationMode) return "detail";
|
||||
if (selectedModelId === "auto") return "agent";
|
||||
if (isVideoModel) return "video";
|
||||
if (isConversationModel) return "chat";
|
||||
if (isAgentModel) return "agent";
|
||||
return "image";
|
||||
}, [isConversationModel, isDetailGenerationMode, isVideoModel, selectedModelId]);
|
||||
}, [isAgentModel, isDetailGenerationMode, isVideoModel]);
|
||||
const activeModeLabel =
|
||||
activeComposerMode === "video"
|
||||
? "视频"
|
||||
: activeComposerMode === "detail"
|
||||
? "AI详情"
|
||||
: activeComposerMode === "chat"
|
||||
? "对话"
|
||||
: activeComposerMode === "image"
|
||||
? "生图"
|
||||
: "Agent";
|
||||
@ -3025,10 +3040,17 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
}, [currentModel, isConversationModel, isDetailGenerationMode, isVideoModel]);
|
||||
|
||||
const handleSelectGenerationMode = useCallback(
|
||||
(mode: "agent" | "chat" | "image" | "video" | "detail") => {
|
||||
(mode: "agent" | "image" | "video" | "detail") => {
|
||||
if (mode === "agent") {
|
||||
currentModelIdRef.current = "auto";
|
||||
setSelectedModelId("auto");
|
||||
const preferredAgentModelId =
|
||||
allModels.find(
|
||||
(model) =>
|
||||
model.id === lastConversationModelId &&
|
||||
model.isMultimodal &&
|
||||
!VIDEO_PROVIDER_TYPES.has(model.providerType)
|
||||
)?.id || "auto";
|
||||
currentModelIdRef.current = preferredAgentModelId;
|
||||
setSelectedModelId(preferredAgentModelId);
|
||||
setIsDetailGenerationMode(false);
|
||||
return;
|
||||
}
|
||||
@ -3045,25 +3067,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
return;
|
||||
}
|
||||
|
||||
if (mode === "chat") {
|
||||
const preferredConversationModelId =
|
||||
allModels.find(
|
||||
(model) =>
|
||||
model.id === lastConversationModelId &&
|
||||
model.isMultimodal &&
|
||||
!VIDEO_PROVIDER_TYPES.has(model.providerType)
|
||||
)?.id || firstConversationModelId;
|
||||
|
||||
if (!preferredConversationModelId) {
|
||||
toast.error("当前没有可用的多模态模型");
|
||||
return;
|
||||
}
|
||||
currentModelIdRef.current = preferredConversationModelId;
|
||||
setSelectedModelId(preferredConversationModelId);
|
||||
setIsDetailGenerationMode(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (mode === "image") {
|
||||
const preferredImageModelId =
|
||||
allModels.find(
|
||||
@ -3100,7 +3103,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
[
|
||||
aiDetailGenerationEnabled,
|
||||
allModels,
|
||||
firstConversationModelId,
|
||||
firstImageModelId,
|
||||
firstVideoModelId,
|
||||
lastConversationModelId,
|
||||
@ -3264,6 +3266,159 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
|
||||
// 合并 initialMessages 和历史消息
|
||||
const mergedInitialMessages = historyData?.messages ?? (activeChatId === chatId ? initialMessages : []);
|
||||
|
||||
const appendStreamMediaToCanvas = useCallback(
|
||||
async (payload: CanvasMediaStreamPayload) => {
|
||||
const imageUrls = Array.isArray(payload.imageUrls)
|
||||
? payload.imageUrls.filter((url): url is string => typeof url === "string" && /^https?:\/\//i.test(url))
|
||||
: [];
|
||||
const videoUrls = Array.isArray(payload.videoUrls)
|
||||
? payload.videoUrls.filter((url): url is string => typeof url === "string" && /^https?:\/\//i.test(url))
|
||||
: [];
|
||||
if (imageUrls.length === 0 && videoUrls.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const latestNodes = useCanvasStore.getState().nodes;
|
||||
const existingImageUrlKeys = new Set(
|
||||
latestNodes
|
||||
.filter((node) => node.type === "image" && (node.data as any)?.src)
|
||||
.map((node) => normalizeImageIdentityUrl(String((node.data as any)?.src || "")))
|
||||
.filter(Boolean)
|
||||
);
|
||||
const existingMessageImageKeys = new Set(
|
||||
latestNodes
|
||||
.filter((node) => node.type === "image")
|
||||
.map((node) => {
|
||||
const data = ((node.data as any) || {}) as Record<string, any>;
|
||||
const metadata = ((data.metadata as any) || {}) as Record<string, any>;
|
||||
const sourceMessageId = String(metadata.sourceMessageId || "").trim();
|
||||
const sourceImageKey = normalizeImageIdentityUrl(
|
||||
String(metadata.sourceImageKey || data.src || "").trim()
|
||||
);
|
||||
if (!sourceMessageId || !sourceImageKey) return "";
|
||||
return `${sourceMessageId}::${sourceImageKey}`;
|
||||
})
|
||||
.filter(Boolean)
|
||||
);
|
||||
const existingVideoUrlKeys = new Set(
|
||||
latestNodes
|
||||
.filter((node) => node.type === "video" && (node.data as any)?.src)
|
||||
.map((node) => normalizeMediaUrl(String((node.data as any)?.src || "")))
|
||||
.filter(Boolean)
|
||||
);
|
||||
|
||||
const sourceImageUrls = Array.isArray(payload.sourceImageUrls)
|
||||
? payload.sourceImageUrls.filter((url): url is string => typeof url === "string" && /^https?:\/\//i.test(url))
|
||||
: [];
|
||||
const messageId = String(payload.messageId || "").trim();
|
||||
const generationPrompt = String(payload.prompt || "").trim() || "对话生成";
|
||||
let addedImageCount = 0;
|
||||
let addedVideoCount = 0;
|
||||
|
||||
for (let index = 0; index < imageUrls.length; index += 1) {
|
||||
const url = imageUrls[index]!;
|
||||
const sourceImageKey = normalizeImageIdentityUrl(url);
|
||||
if (!sourceImageKey || existingImageUrlKeys.has(sourceImageKey)) {
|
||||
continue;
|
||||
}
|
||||
const messageImageKey = messageId ? `${messageId}::${sourceImageKey}` : "";
|
||||
if (messageImageKey && existingMessageImageKeys.has(messageImageKey)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
existingImageUrlKeys.add(sourceImageKey);
|
||||
if (messageImageKey) {
|
||||
existingMessageImageKeys.add(messageImageKey);
|
||||
}
|
||||
|
||||
const dimensions = await getImageDimensions(url);
|
||||
const currentNodes = useCanvasStore.getState().nodes;
|
||||
const existingNodes = currentNodes.length + addedImageCount + addedVideoCount;
|
||||
const spacing = 20;
|
||||
const x = 100 + (existingNodes % 3) * (dimensions.width + spacing);
|
||||
const y = 100 + Math.floor(existingNodes / 3) * (dimensions.height + spacing);
|
||||
const compareSourceUrlRaw =
|
||||
sourceImageUrls[Math.min(index, Math.max(0, sourceImageUrls.length - 1))] ||
|
||||
sourceImageUrls[0] ||
|
||||
"";
|
||||
const compareSourceUrl = String(compareSourceUrlRaw || "").trim();
|
||||
const normalizedCompareSource = normalizeImageIdentityUrl(compareSourceUrl);
|
||||
const canUseCompareSource =
|
||||
Boolean(normalizedCompareSource) &&
|
||||
normalizedCompareSource !== sourceImageKey;
|
||||
const sourceNodeId = canUseCompareSource
|
||||
? useCanvasStore
|
||||
.getState()
|
||||
.nodes.find((node) => {
|
||||
if (node.type !== "image") return false;
|
||||
const src = String(((node.data as any) || {}).src || "").trim();
|
||||
return normalizeImageIdentityUrl(src) === normalizedCompareSource;
|
||||
})?.id || ""
|
||||
: "";
|
||||
|
||||
const nodeId = addNode({
|
||||
type: "image",
|
||||
position: { x, y },
|
||||
size: { width: dimensions.width, height: dimensions.height },
|
||||
data: {
|
||||
src: url,
|
||||
status: "completed",
|
||||
title: "图片",
|
||||
prompt: generationPrompt,
|
||||
originalSrc: canUseCompareSource ? compareSourceUrl : undefined,
|
||||
metadata: {
|
||||
originalWidth: dimensions.originalWidth,
|
||||
originalHeight: dimensions.originalHeight,
|
||||
generationPrompt,
|
||||
sourceMessageId: messageId || undefined,
|
||||
sourceImageKey,
|
||||
compareSourceUrl: canUseCompareSource ? compareSourceUrl : undefined,
|
||||
},
|
||||
},
|
||||
connections: sourceNodeId ? [sourceNodeId] : [],
|
||||
});
|
||||
void autoNameGeneratedImageNode(nodeId, url);
|
||||
addedImageCount += 1;
|
||||
}
|
||||
|
||||
for (let index = 0; index < videoUrls.length; index += 1) {
|
||||
const url = videoUrls[index]!;
|
||||
const key = normalizeMediaUrl(url);
|
||||
if (!key || existingVideoUrlKeys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
existingVideoUrlKeys.add(key);
|
||||
const currentNodes = useCanvasStore.getState().nodes;
|
||||
const existingNodes = currentNodes.length + addedImageCount + addedVideoCount;
|
||||
const x = 120 + (existingNodes % 2) * 420;
|
||||
const y = 120 + Math.floor(existingNodes / 2) * 260;
|
||||
addNode({
|
||||
type: "video",
|
||||
position: { x, y },
|
||||
size: { width: 380, height: 220 },
|
||||
data: {
|
||||
src: url,
|
||||
status: "completed",
|
||||
prompt: generationPrompt,
|
||||
},
|
||||
connections: [],
|
||||
});
|
||||
addedVideoCount += 1;
|
||||
}
|
||||
|
||||
if (addedImageCount > 0 || addedVideoCount > 0) {
|
||||
const notice = [
|
||||
addedImageCount > 0 ? `${addedImageCount} 张图片` : "",
|
||||
addedVideoCount > 0 ? `${addedVideoCount} 条视频` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("、");
|
||||
toast.success(`已将 ${notice} 添加到画布`);
|
||||
}
|
||||
},
|
||||
[addNode, autoNameGeneratedImageNode]
|
||||
);
|
||||
|
||||
const {
|
||||
messages,
|
||||
@ -3332,12 +3487,79 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
},
|
||||
}),
|
||||
onData: (dataPart: any) => {
|
||||
if (dataPart.type === "data-imageGenerationStatus") {
|
||||
try {
|
||||
const payload: ImageGenerationStatusStreamPayload =
|
||||
typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data;
|
||||
if (!payload || typeof payload !== "object") return;
|
||||
if (payload.phase === "completed") {
|
||||
setStreamImageGenerationState(null);
|
||||
return;
|
||||
}
|
||||
setStreamImageGenerationState({
|
||||
phase: payload.phase || "processing",
|
||||
providerType: payload.providerType,
|
||||
modelName: payload.modelName,
|
||||
imageCount: Number(payload.imageCount || 0) || undefined,
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("[CanvasChat] Failed to parse data-imageGenerationStatus:", e);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (dataPart.type === "data-canvasMedia") {
|
||||
try {
|
||||
const payload: CanvasMediaStreamPayload =
|
||||
typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data;
|
||||
if (!payload || typeof payload !== "object") return;
|
||||
void appendStreamMediaToCanvas(payload);
|
||||
} catch (e) {
|
||||
console.error("[CanvasChat] Failed to parse data-canvasMedia:", e);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 处理后端返回的自定义消息
|
||||
if (dataPart.type === "data-appendMessage") {
|
||||
try {
|
||||
const messageData = typeof dataPart.data === 'string'
|
||||
? JSON.parse(dataPart.data)
|
||||
: dataPart.data;
|
||||
const hasImageGenStartMarker = Array.isArray(messageData.parts)
|
||||
? messageData.parts.some(
|
||||
(part: any) =>
|
||||
part?.type === "text" &&
|
||||
typeof part?.text === "string" &&
|
||||
part.text.includes("<!-- IMAGE_GENERATION_START -->")
|
||||
)
|
||||
: false;
|
||||
const hasImageGenEndMarker = Array.isArray(messageData.parts)
|
||||
? messageData.parts.some(
|
||||
(part: any) =>
|
||||
part?.type === "text" &&
|
||||
typeof part?.text === "string" &&
|
||||
part.text.includes("<!-- IMAGE_GENERATION_END -->")
|
||||
)
|
||||
: false;
|
||||
if (hasImageGenStartMarker) {
|
||||
setStreamImageGenerationState((prev) => ({
|
||||
phase: "processing",
|
||||
providerType:
|
||||
prev?.providerType ||
|
||||
(activeComposerMode === "agent" ? "alibaba-image" : currentModel?.providerType),
|
||||
modelName:
|
||||
prev?.modelName ||
|
||||
(activeComposerMode === "agent"
|
||||
? "智能处理"
|
||||
: currentModel?.displayName || undefined),
|
||||
imageCount:
|
||||
prev?.imageCount || Math.max(1, Number(imageParamsRef.current?.imageCount || 1)),
|
||||
}));
|
||||
}
|
||||
if (hasImageGenEndMarker) {
|
||||
setStreamImageGenerationState(null);
|
||||
}
|
||||
const normalizedParts = Array.isArray(messageData.parts) ? messageData.parts : [];
|
||||
const normalizedAttachments = Array.isArray(messageData.attachments)
|
||||
? messageData.attachments
|
||||
@ -5950,7 +6172,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
]);
|
||||
|
||||
const ensureImagePointsBeforeSend = useCallback(() => {
|
||||
if (isVideoModel || activeComposerMode === "chat") return true;
|
||||
if (isVideoModel || activeComposerMode === "agent") return true;
|
||||
|
||||
const pointsEnabled = membershipSummary?.pointsSystemConfig?.enabled === true;
|
||||
const blockOnInsufficient =
|
||||
@ -6169,14 +6391,15 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
return;
|
||||
}
|
||||
|
||||
sendGuardRef.current = {
|
||||
inFlight: true,
|
||||
signature: sendSignature,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
sendGuardRef.current = {
|
||||
inFlight: true,
|
||||
signature: sendSignature,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
setStreamImageGenerationState(null);
|
||||
|
||||
// 使用与 multimodal-input 相同的格式发送消息
|
||||
try {
|
||||
// 使用与 multimodal-input 相同的格式发送消息
|
||||
try {
|
||||
sendMessage({
|
||||
role: "user",
|
||||
parts: parts,
|
||||
@ -6633,10 +6856,27 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
(part: any) => part.type === "text" && part.text?.includes("<!-- IMAGE_GENERATION_END -->")
|
||||
)
|
||||
);
|
||||
const shouldShowImageLoading = hasGenerationStart && !hasGenerationEnd && !!currentModel;
|
||||
const shouldShowImageLoading =
|
||||
((hasGenerationStart && !hasGenerationEnd) || Boolean(streamImageGenerationState)) &&
|
||||
!!currentModel;
|
||||
const hasActiveTypewriter = Object.keys(activeTypingSections).length > 0;
|
||||
const shouldRenderImageLoading = shouldShowImageLoading && !hasActiveTypewriter;
|
||||
const loadingImageCount = imageParams.imageCount || 1;
|
||||
const loadingImageCount = streamImageGenerationState?.imageCount || imageParams.imageCount || 1;
|
||||
const loadingProviderType =
|
||||
streamImageGenerationState?.providerType ||
|
||||
(activeComposerMode === "agent" ? "alibaba-image" : currentModel?.providerType);
|
||||
const loadingModelName =
|
||||
streamImageGenerationState?.modelName ||
|
||||
(activeComposerMode === "agent"
|
||||
? "智能处理"
|
||||
: currentModel?.displayName || undefined);
|
||||
const loadingPhase =
|
||||
streamImageGenerationState?.phase === "queued" ||
|
||||
streamImageGenerationState?.phase === "processing" ||
|
||||
streamImageGenerationState?.phase === "finalizing" ||
|
||||
streamImageGenerationState?.phase === "failed"
|
||||
? streamImageGenerationState.phase
|
||||
: "processing";
|
||||
const renderableVisibleMessages = useMemo(() => {
|
||||
return visibleMessages.reduce<ChatMessage[]>((acc, message) => {
|
||||
const rawParts = (message.parts || []) as any[];
|
||||
@ -7474,8 +7714,9 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
<ImageGenerationLoading
|
||||
imageCount={loadingImageCount}
|
||||
maxImages={6}
|
||||
providerType={currentModel?.providerType}
|
||||
modelName={currentModel?.displayName}
|
||||
providerType={loadingProviderType}
|
||||
modelName={loadingModelName}
|
||||
phase={loadingPhase}
|
||||
/>
|
||||
)}
|
||||
|
||||
@ -8498,8 +8739,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
? "border border-blue-500 bg-blue-50 text-blue-600 hover:bg-blue-100"
|
||||
: activeComposerMode === "detail"
|
||||
? "border border-violet-500 bg-violet-50 text-violet-700 hover:bg-violet-100"
|
||||
: activeComposerMode === "chat"
|
||||
? "border border-sky-500 bg-sky-50 text-sky-700 hover:bg-sky-100"
|
||||
: activeComposerMode === "image"
|
||||
? "border border-purple-500 bg-purple-50 text-purple-700 hover:bg-purple-100"
|
||||
: "border border-[#9eb3ff] bg-white text-zinc-700 hover:bg-zinc-50"
|
||||
@ -8510,8 +8749,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
<Video className="mr-1 h-4 w-4" />
|
||||
) : activeComposerMode === "detail" ? (
|
||||
<LayoutGrid className="mr-1 h-4 w-4" />
|
||||
) : activeComposerMode === "chat" ? (
|
||||
<MessageCircle className="mr-1 h-4 w-4" />
|
||||
) : activeComposerMode === "image" ? (
|
||||
<ImagePlus className="mr-1 h-4 w-4" />
|
||||
) : (
|
||||
@ -8538,17 +8775,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
<span className="flex-1">Agent</span>
|
||||
{activeComposerMode === "agent" ? <Check className="h-4 w-4 text-[#3b82f6]" /> : null}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => handleSelectGenerationMode("chat")}
|
||||
className={cn(
|
||||
"h-8 cursor-pointer rounded-lg px-2 text-[13px] font-medium text-[#2e3238]",
|
||||
activeComposerMode === "chat" ? "bg-white/80" : "hover:bg-white/70"
|
||||
)}
|
||||
>
|
||||
<MessageCircle className="h-4 w-4" />
|
||||
<span className="flex-1">对话模式</span>
|
||||
{activeComposerMode === "chat" ? <Check className="h-4 w-4 text-[#3b82f6]" /> : null}
|
||||
</DropdownMenuItem>
|
||||
{aiDetailGenerationEnabled ? (
|
||||
<DropdownMenuItem
|
||||
onClick={() => handleSelectGenerationMode("detail")}
|
||||
@ -8612,10 +8838,10 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
) : activeComposerMode !== "chat" ? (
|
||||
) : (
|
||||
<div className="shrink-0">
|
||||
<ImageParamsBar
|
||||
modelName={currentModel?.displayName}
|
||||
modelName={activeComposerMode === "agent" ? agentDisplayName : currentModel?.displayName}
|
||||
initialParams={imageParams}
|
||||
onParamsChange={handleImageParamsChange}
|
||||
inline={true}
|
||||
@ -8624,7 +8850,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
hideAdvancedOptions={activeComposerMode === "image"}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2 flex-shrink-0">
|
||||
@ -8671,7 +8897,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
</Button>
|
||||
) : null}
|
||||
|
||||
{activeComposerMode !== "agent" ? (
|
||||
{activeComposerMode !== "detail" ? (
|
||||
<Select
|
||||
value={modelSelectValue}
|
||||
onValueChange={(value) => {
|
||||
|
||||
@ -3,6 +3,11 @@ import { generateText } from "ai";
|
||||
import sharp from "sharp";
|
||||
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
|
||||
import type { RouteResult } from "@/lib/ai/model-router";
|
||||
import {
|
||||
getEnabledAgentToolDefinitionsForContext,
|
||||
type AgentProjectToolDefinition,
|
||||
type AgentProjectToolName,
|
||||
} from "@/lib/ai/tools-registry";
|
||||
import {
|
||||
getAiModelAgentCodingModelId,
|
||||
isAiModelMultimodal,
|
||||
@ -47,12 +52,90 @@ export interface AgentVideoPlan {
|
||||
recommendedVideoParams?: AgentRecommendedVideoParams;
|
||||
}
|
||||
|
||||
export type AgentTargetResolutionMode =
|
||||
| "explicit_selection"
|
||||
| "current_upload"
|
||||
| "text_reference"
|
||||
| "working_memory"
|
||||
| "latest_result"
|
||||
| "clarification"
|
||||
| "none";
|
||||
|
||||
export type AgentPreferredCapability =
|
||||
| "respond"
|
||||
| "analyze"
|
||||
| "generic_edit"
|
||||
| "remove_background"
|
||||
| "enhance"
|
||||
| "extend"
|
||||
| "resize"
|
||||
| "box_cutout"
|
||||
| "generate_image"
|
||||
| "generate_video"
|
||||
| "publish_canvas";
|
||||
|
||||
export type AgentPlanNextAction =
|
||||
| "respond"
|
||||
| "clarify"
|
||||
| "analyze_image"
|
||||
| "generate_image"
|
||||
| "edit_image"
|
||||
| "generate_video";
|
||||
|
||||
export type AgentPlanResponseMode =
|
||||
| "direct_answer"
|
||||
| "tool_then_answer"
|
||||
| "tool_then_execute"
|
||||
| "clarify";
|
||||
|
||||
export interface AgentPlannedToolCall {
|
||||
name: AgentProjectToolName;
|
||||
reason?: string;
|
||||
args?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface AgentExecutionPlan {
|
||||
goal: string;
|
||||
nextAction: AgentPlanNextAction;
|
||||
intent: AnalysisResult["intent"];
|
||||
subIntent?: string;
|
||||
targetSource: AgentTargetResolutionMode;
|
||||
preferredCapability: AgentPreferredCapability;
|
||||
responseMode: AgentPlanResponseMode;
|
||||
shouldUseHistoryImage: boolean;
|
||||
shouldUseStructuredReply: boolean;
|
||||
confidence: number;
|
||||
toolCalls: AgentPlannedToolCall[];
|
||||
}
|
||||
|
||||
export interface AgentPreparedExecution {
|
||||
mode: AgentPlanNextAction;
|
||||
routeIntent: AnalysisResult["intent"];
|
||||
routeSubIntent?: string;
|
||||
targetSource: AgentTargetResolutionMode;
|
||||
preferredCapability: AgentPreferredCapability;
|
||||
shouldUseStructuredReply: boolean;
|
||||
shouldOptimizePrompt: boolean;
|
||||
shouldApplyImagePipeline: boolean;
|
||||
shouldApplyVideoPipeline: boolean;
|
||||
shouldStayConversational: boolean;
|
||||
shouldPublishToCanvas: boolean;
|
||||
toolCalls: AgentPlannedToolCall[];
|
||||
availableTools: AgentProjectToolName[];
|
||||
}
|
||||
|
||||
export interface AutoAgentInput {
|
||||
text: string;
|
||||
hasImage: boolean;
|
||||
imageUrls?: string[];
|
||||
assetIds?: string[];
|
||||
imageFromHistory?: boolean;
|
||||
targetResolutionMode?: AgentTargetResolutionMode;
|
||||
mediaSource?: "current_message" | "canvas_selection" | "history" | "none";
|
||||
hasCanvasSelection?: boolean;
|
||||
selectedImageCount?: number;
|
||||
historyImageCount?: number;
|
||||
currentUploadImageCount?: number;
|
||||
historyMessages?: HistoryMessage[];
|
||||
context?: string;
|
||||
preferredModelId?: string;
|
||||
@ -60,6 +143,8 @@ export interface AutoAgentInput {
|
||||
|
||||
export interface AutoAgentResult {
|
||||
analysis?: AnalysisResult;
|
||||
plan?: AgentExecutionPlan;
|
||||
execution?: AgentPreparedExecution;
|
||||
friendlyMessage?: string;
|
||||
optimizedPrompt?: string;
|
||||
recommendedImageParams?: AgentRecommendedImageParams;
|
||||
@ -73,10 +158,19 @@ const AutoAgentState = Annotation.Root({
|
||||
imageUrls: Annotation<string[]>,
|
||||
assetIds: Annotation<string[]>,
|
||||
imageFromHistory: Annotation<boolean>,
|
||||
targetResolutionMode: Annotation<AgentTargetResolutionMode>,
|
||||
mediaSource: Annotation<"current_message" | "canvas_selection" | "history" | "none">,
|
||||
hasCanvasSelection: Annotation<boolean>,
|
||||
selectedImageCount: Annotation<number>,
|
||||
historyImageCount: Annotation<number>,
|
||||
currentUploadImageCount: Annotation<number>,
|
||||
historyMessages: Annotation<HistoryMessage[]>,
|
||||
context: Annotation<string | null>,
|
||||
preferredModelId: Annotation<string | null>,
|
||||
analysis: Annotation<AnalysisResult | null>,
|
||||
availableTools: Annotation<AgentProjectToolDefinition[]>,
|
||||
plan: Annotation<AgentExecutionPlan | null>,
|
||||
execution: Annotation<AgentPreparedExecution | null>,
|
||||
friendlyMessage: Annotation<string | null>,
|
||||
optimizedPrompt: Annotation<string | null>,
|
||||
recommendedImageParams: Annotation<AgentRecommendedImageParams | null>,
|
||||
@ -92,6 +186,244 @@ function compactText(text: string, max = 48) {
|
||||
return `${normalized.slice(0, max)}...`;
|
||||
}
|
||||
|
||||
function createPlanGoalSummary(text: string, hasImage: boolean): string {
|
||||
const compact = compactText(text || "", 72);
|
||||
if (compact) return compact;
|
||||
return hasImage ? "基于当前图片继续处理" : "继续当前对话";
|
||||
}
|
||||
|
||||
function describeTargetSource(mode: AgentTargetResolutionMode): string {
|
||||
switch (mode) {
|
||||
case "explicit_selection":
|
||||
return "画布选中图";
|
||||
case "current_upload":
|
||||
return "当前上传图";
|
||||
case "text_reference":
|
||||
return "用户文本指定图";
|
||||
case "working_memory":
|
||||
return "会话工作记忆图";
|
||||
case "latest_result":
|
||||
return "最近生成结果图";
|
||||
case "clarification":
|
||||
return "目标待澄清";
|
||||
case "none":
|
||||
default:
|
||||
return "无明确图像目标";
|
||||
}
|
||||
}
|
||||
|
||||
function inferPreferredCapability(
|
||||
analysis: AnalysisResult,
|
||||
toolCalls?: AgentPlannedToolCall[],
|
||||
): AgentPreferredCapability {
|
||||
const firstTool = toolCalls?.[0]?.name;
|
||||
if (firstTool === "removeBackground") return "remove_background";
|
||||
if (firstTool === "enhanceImage") return "enhance";
|
||||
if (firstTool === "extendImage") return "extend";
|
||||
if (firstTool === "resizeImage") return "resize";
|
||||
if (firstTool === "boxCutout") return "box_cutout";
|
||||
if (firstTool === "publishToCanvas") return "publish_canvas";
|
||||
if (firstTool === "generateVideo") return "generate_video";
|
||||
if (firstTool === "generateImage") return "generate_image";
|
||||
if (firstTool === "analyzeImage") return "analyze";
|
||||
|
||||
if (analysis.intent === "video_gen") return "generate_video";
|
||||
if (analysis.intent === "image_gen") return "generate_image";
|
||||
if (analysis.intent === "vision") return "analyze";
|
||||
if (analysis.intent === "chat") return "respond";
|
||||
|
||||
switch (analysis.subIntent) {
|
||||
case "remove_background":
|
||||
return "remove_background";
|
||||
case "enhance":
|
||||
return "enhance";
|
||||
case "extend":
|
||||
return "extend";
|
||||
case "resize":
|
||||
return "resize";
|
||||
default:
|
||||
return "generic_edit";
|
||||
}
|
||||
}
|
||||
|
||||
function defaultToolForAction(action: AgentPlanNextAction): AgentProjectToolName {
|
||||
switch (action) {
|
||||
case "analyze_image":
|
||||
return "analyzeImage";
|
||||
case "edit_image":
|
||||
return "editImage";
|
||||
case "generate_image":
|
||||
return "generateImage";
|
||||
case "generate_video":
|
||||
return "generateVideo";
|
||||
case "clarify":
|
||||
case "respond":
|
||||
default:
|
||||
return "respondDirect";
|
||||
}
|
||||
}
|
||||
|
||||
function mapIntentToAction(intent: AnalysisResult["intent"]): AgentPlanNextAction {
|
||||
switch (intent) {
|
||||
case "vision":
|
||||
return "analyze_image";
|
||||
case "image_edit":
|
||||
return "edit_image";
|
||||
case "image_gen":
|
||||
return "generate_image";
|
||||
case "video_gen":
|
||||
return "generate_video";
|
||||
case "chat":
|
||||
default:
|
||||
return "respond";
|
||||
}
|
||||
}
|
||||
|
||||
function mapActionToIntent(
|
||||
action: AgentPlanNextAction,
|
||||
fallbackIntent: AnalysisResult["intent"],
|
||||
): AnalysisResult["intent"] {
|
||||
switch (action) {
|
||||
case "analyze_image":
|
||||
return "vision";
|
||||
case "edit_image":
|
||||
return "image_edit";
|
||||
case "generate_image":
|
||||
return "image_gen";
|
||||
case "generate_video":
|
||||
return "video_gen";
|
||||
case "clarify":
|
||||
case "respond":
|
||||
return fallbackIntent === "vision" ? "vision" : "chat";
|
||||
default:
|
||||
return fallbackIntent;
|
||||
}
|
||||
}
|
||||
|
||||
function defaultResponseMode(action: AgentPlanNextAction): AgentPlanResponseMode {
|
||||
switch (action) {
|
||||
case "clarify":
|
||||
return "clarify";
|
||||
case "respond":
|
||||
return "direct_answer";
|
||||
case "analyze_image":
|
||||
return "tool_then_answer";
|
||||
case "edit_image":
|
||||
case "generate_image":
|
||||
case "generate_video":
|
||||
return "tool_then_execute";
|
||||
default:
|
||||
return "direct_answer";
|
||||
}
|
||||
}
|
||||
|
||||
function shouldUseStructuredReply(action: AgentPlanNextAction): boolean {
|
||||
return action === "edit_image" || action === "generate_image" || action === "generate_video";
|
||||
}
|
||||
|
||||
function sanitizePlannedToolCalls(
|
||||
rawToolCalls: unknown,
|
||||
availableTools: AgentProjectToolDefinition[],
|
||||
fallbackAction: AgentPlanNextAction,
|
||||
): AgentPlannedToolCall[] {
|
||||
const availableNames = new Set(availableTools.map((tool) => tool.name));
|
||||
const list = Array.isArray(rawToolCalls) ? rawToolCalls : [];
|
||||
const sanitized = list
|
||||
.map((item) => {
|
||||
if (!item || typeof item !== "object") return null;
|
||||
const rawName = String((item as { name?: unknown }).name || "").trim() as AgentProjectToolName;
|
||||
if (!availableNames.has(rawName)) return null;
|
||||
const args = (item as { args?: unknown }).args;
|
||||
return {
|
||||
name: rawName,
|
||||
reason: String((item as { reason?: unknown }).reason || "").trim() || undefined,
|
||||
args: args && typeof args === "object" && !Array.isArray(args) ? (args as Record<string, unknown>) : undefined,
|
||||
} satisfies AgentPlannedToolCall;
|
||||
})
|
||||
.filter(Boolean) as AgentPlannedToolCall[];
|
||||
|
||||
if (sanitized.length > 0) return sanitized;
|
||||
|
||||
const fallbackTool = defaultToolForAction(fallbackAction);
|
||||
if (availableNames.has(fallbackTool)) {
|
||||
return [{ name: fallbackTool }];
|
||||
}
|
||||
if (availableNames.has("respondDirect")) {
|
||||
return [{ name: "respondDirect" }];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function buildFallbackPlan(params: {
|
||||
analysis: AnalysisResult;
|
||||
text: string;
|
||||
hasImage: boolean;
|
||||
imageFromHistory: boolean;
|
||||
targetResolutionMode: AgentTargetResolutionMode;
|
||||
availableTools: AgentProjectToolDefinition[];
|
||||
}): AgentExecutionPlan {
|
||||
const baseAction = params.analysis.needsClarification
|
||||
? "clarify"
|
||||
: mapIntentToAction(params.analysis.intent);
|
||||
const toolCalls = sanitizePlannedToolCalls([], params.availableTools, baseAction);
|
||||
return {
|
||||
goal: createPlanGoalSummary(params.text, params.hasImage),
|
||||
nextAction: baseAction,
|
||||
intent: mapActionToIntent(baseAction, params.analysis.intent),
|
||||
subIntent: baseAction === "edit_image" ? params.analysis.subIntent || "enhance" : undefined,
|
||||
targetSource: params.targetResolutionMode,
|
||||
preferredCapability: inferPreferredCapability(params.analysis, toolCalls),
|
||||
responseMode: defaultResponseMode(baseAction),
|
||||
shouldUseHistoryImage: params.imageFromHistory,
|
||||
shouldUseStructuredReply: shouldUseStructuredReply(baseAction),
|
||||
confidence: Math.max(params.analysis.confidence || 0.6, 0.68),
|
||||
toolCalls,
|
||||
};
|
||||
}
|
||||
|
||||
function alignAnalysisWithPlan(
|
||||
analysis: AnalysisResult,
|
||||
plan: AgentExecutionPlan,
|
||||
): AnalysisResult {
|
||||
const intent = plan.intent || mapActionToIntent(plan.nextAction, analysis.intent);
|
||||
return {
|
||||
...analysis,
|
||||
intent,
|
||||
subIntent: intent === "image_edit" ? plan.subIntent || analysis.subIntent : undefined,
|
||||
needsClarification: plan.nextAction === "clarify",
|
||||
clarifyingQuestions:
|
||||
plan.nextAction === "clarify"
|
||||
? analysis.clarifyingQuestions?.length
|
||||
? analysis.clarifyingQuestions
|
||||
: ["我先确认一下你希望我下一步执行什么。"]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
function prepareExecutionFromPlan(params: {
|
||||
plan: AgentExecutionPlan;
|
||||
availableTools: AgentProjectToolDefinition[];
|
||||
}): AgentPreparedExecution {
|
||||
const availableToolNames = params.availableTools.map((tool) => tool.name);
|
||||
const mode = params.plan.nextAction;
|
||||
return {
|
||||
mode,
|
||||
routeIntent: params.plan.intent,
|
||||
routeSubIntent: params.plan.subIntent,
|
||||
targetSource: params.plan.targetSource,
|
||||
preferredCapability: params.plan.preferredCapability,
|
||||
shouldUseStructuredReply: params.plan.shouldUseStructuredReply,
|
||||
shouldOptimizePrompt:
|
||||
mode === "edit_image" || mode === "generate_image" || mode === "generate_video",
|
||||
shouldApplyImagePipeline: mode === "edit_image" || mode === "generate_image",
|
||||
shouldApplyVideoPipeline: mode === "generate_video",
|
||||
shouldStayConversational: mode === "respond" || mode === "analyze_image" || mode === "clarify",
|
||||
shouldPublishToCanvas: params.plan.toolCalls.some((tool) => tool.name === "publishToCanvas"),
|
||||
toolCalls: params.plan.toolCalls,
|
||||
availableTools: availableToolNames,
|
||||
};
|
||||
}
|
||||
|
||||
function getEnvInt(name: string, fallback: number, min: number, max: number): number {
|
||||
const raw = process.env[name];
|
||||
const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
|
||||
@ -760,6 +1092,244 @@ function resolveAgentCodingModelOverride(
|
||||
return getAiModelAgentCodingModelId(configuredAgentModel?.model || {}) || null;
|
||||
}
|
||||
|
||||
function sanitizeAgentPlan(
|
||||
raw: unknown,
|
||||
fallbackPlan: AgentExecutionPlan,
|
||||
availableTools: AgentProjectToolDefinition[],
|
||||
): AgentExecutionPlan {
|
||||
if (!raw || typeof raw !== "object") {
|
||||
return fallbackPlan;
|
||||
}
|
||||
|
||||
const payload = raw as Record<string, unknown>;
|
||||
const rawAction = String(payload.nextAction || "").trim() as AgentPlanNextAction;
|
||||
const nextAction: AgentPlanNextAction = [
|
||||
"respond",
|
||||
"clarify",
|
||||
"analyze_image",
|
||||
"generate_image",
|
||||
"edit_image",
|
||||
"generate_video",
|
||||
].includes(rawAction)
|
||||
? rawAction
|
||||
: fallbackPlan.nextAction;
|
||||
|
||||
const fallbackIntent = fallbackPlan.intent;
|
||||
const rawIntent = String(payload.intent || "").trim() as AnalysisResult["intent"];
|
||||
const intent: AnalysisResult["intent"] = [
|
||||
"chat",
|
||||
"vision",
|
||||
"image_edit",
|
||||
"image_gen",
|
||||
"video_gen",
|
||||
].includes(rawIntent)
|
||||
? rawIntent
|
||||
: mapActionToIntent(nextAction, fallbackIntent);
|
||||
|
||||
const rawResponseMode = String(payload.responseMode || "").trim() as AgentPlanResponseMode;
|
||||
const responseMode: AgentPlanResponseMode = [
|
||||
"direct_answer",
|
||||
"tool_then_answer",
|
||||
"tool_then_execute",
|
||||
"clarify",
|
||||
].includes(rawResponseMode)
|
||||
? rawResponseMode
|
||||
: defaultResponseMode(nextAction);
|
||||
|
||||
const confidenceNumber = Number(payload.confidence);
|
||||
const confidence = Number.isFinite(confidenceNumber)
|
||||
? Math.max(0.3, Math.min(0.99, confidenceNumber))
|
||||
: fallbackPlan.confidence;
|
||||
|
||||
const goal =
|
||||
typeof payload.goal === "string" && payload.goal.trim()
|
||||
? payload.goal.trim().slice(0, 160)
|
||||
: fallbackPlan.goal;
|
||||
|
||||
const rawSubIntent =
|
||||
typeof payload.subIntent === "string" && payload.subIntent.trim()
|
||||
? payload.subIntent.trim().slice(0, 48)
|
||||
: undefined;
|
||||
const rawTargetSource = String(payload.targetSource || "").trim() as AgentTargetResolutionMode;
|
||||
const targetSource: AgentTargetResolutionMode = [
|
||||
"explicit_selection",
|
||||
"current_upload",
|
||||
"text_reference",
|
||||
"working_memory",
|
||||
"latest_result",
|
||||
"clarification",
|
||||
"none",
|
||||
].includes(rawTargetSource)
|
||||
? rawTargetSource
|
||||
: fallbackPlan.targetSource;
|
||||
const sanitizedToolCalls = sanitizePlannedToolCalls(payload.toolCalls, availableTools, nextAction);
|
||||
const rawCapability = String(payload.preferredCapability || "").trim() as AgentPreferredCapability;
|
||||
const derivedAnalysis: AnalysisResult = {
|
||||
intent,
|
||||
subIntent: intent === "image_edit" ? rawSubIntent || fallbackPlan.subIntent : undefined,
|
||||
confidence,
|
||||
};
|
||||
const preferredCapability: AgentPreferredCapability = [
|
||||
"respond",
|
||||
"analyze",
|
||||
"generic_edit",
|
||||
"remove_background",
|
||||
"enhance",
|
||||
"extend",
|
||||
"resize",
|
||||
"box_cutout",
|
||||
"generate_image",
|
||||
"generate_video",
|
||||
"publish_canvas",
|
||||
].includes(rawCapability)
|
||||
? rawCapability
|
||||
: inferPreferredCapability(derivedAnalysis, sanitizedToolCalls);
|
||||
|
||||
return {
|
||||
goal,
|
||||
nextAction,
|
||||
intent,
|
||||
subIntent: intent === "image_edit" ? rawSubIntent || fallbackPlan.subIntent : undefined,
|
||||
targetSource,
|
||||
preferredCapability,
|
||||
responseMode,
|
||||
shouldUseHistoryImage:
|
||||
typeof payload.shouldUseHistoryImage === "boolean"
|
||||
? payload.shouldUseHistoryImage
|
||||
: fallbackPlan.shouldUseHistoryImage,
|
||||
shouldUseStructuredReply:
|
||||
typeof payload.shouldUseStructuredReply === "boolean"
|
||||
? payload.shouldUseStructuredReply
|
||||
: shouldUseStructuredReply(nextAction),
|
||||
confidence,
|
||||
toolCalls: sanitizedToolCalls,
|
||||
};
|
||||
}
|
||||
|
||||
async function buildExecutionPlanWithAgent(
|
||||
state: AutoAgentStateType,
|
||||
analysis: AnalysisResult,
|
||||
availableTools: AgentProjectToolDefinition[],
|
||||
): Promise<AgentExecutionPlan> {
|
||||
const fallbackPlan = buildFallbackPlan({
|
||||
analysis,
|
||||
text: state.text,
|
||||
hasImage: state.hasImage,
|
||||
imageFromHistory: state.imageFromHistory,
|
||||
targetResolutionMode: state.targetResolutionMode,
|
||||
availableTools,
|
||||
});
|
||||
|
||||
try {
|
||||
const {
|
||||
getEnabledAiModelsWithProvider,
|
||||
getAgentAiModelWithProvider,
|
||||
} = await import("@/lib/db/queries");
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const agentModel = await getAgentAiModelWithProvider();
|
||||
const primaryAgentModel = resolvePrimaryAgentModel({
|
||||
preferredModelId: state.preferredModelId,
|
||||
configuredAgentModel: agentModel,
|
||||
allModels,
|
||||
});
|
||||
const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels);
|
||||
if (!plannerModelConfig) {
|
||||
return fallbackPlan;
|
||||
}
|
||||
const plannerModel = await getLanguageModelById(
|
||||
plannerModelConfig.model.id,
|
||||
resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel),
|
||||
);
|
||||
if (!plannerModel) {
|
||||
return fallbackPlan;
|
||||
}
|
||||
|
||||
const toolCatalog = availableTools
|
||||
.map((tool) => `- ${tool.name}: ${tool.title}｜${tool.description}`)
|
||||
.join("\n");
|
||||
const historyContext = (state.historyMessages || [])
|
||||
.slice(-6)
|
||||
.map((item) => `${item.role}: ${compactText(item.content || "", 120)}`)
|
||||
.join("\n");
|
||||
|
||||
const { text } = await generateText({
|
||||
model: plannerModel,
|
||||
system: `你是一个多模态 Agent Planner。
|
||||
你的职责不是直接回答，而是决定当前回合下一步该走哪条能力路径。
|
||||
只输出 JSON，不要输出解释，不要输出 markdown。
|
||||
|
||||
可选 nextAction：
|
||||
- respond：纯对话、策略讨论、解释说明、普通问答
|
||||
- clarify：信息不足，需要先澄清
|
||||
- analyze_image：用户要你看图、识图、评估、分析、给建议
|
||||
- edit_image：基于当前图/历史图做修改、抠图、增强、换背景、调色、尺寸调整、局部重绘
|
||||
- generate_image：生成新图
|
||||
- generate_video：生成视频
|
||||
|
||||
规则：
|
||||
1. 用户只是聊天、问图、讨论方案时，绝不能走生图/生视频。
|
||||
2. 只有用户明确要求“生成/修改/做视频”等执行任务时，才走执行型 action。
|
||||
3. 如果用户上传了图片但只是问“这是什么/适不适合/帮我分析一下”，应选择 analyze_image 或 respond。
|
||||
4. 优先保持一次只选择一个主工具；toolCalls 只从提供的工具列表里选。
|
||||
5. shouldUseStructuredReply 只在 edit_image / generate_image / generate_video 时设为 true。
|
||||
6. targetSource 要忠实反映当前目标图来源：explicit_selection/current_upload/text_reference/working_memory/latest_result/clarification/none。
|
||||
7. preferredCapability 要尽量具体，例如 remove_background / enhance / extend / resize / box_cutout / publish_canvas。
|
||||
|
||||
输出格式：
|
||||
{
|
||||
"goal": "一句话概括当前回合目标",
|
||||
"nextAction": "respond|clarify|analyze_image|edit_image|generate_image|generate_video",
|
||||
"intent": "chat|vision|image_edit|image_gen|video_gen",
|
||||
"subIntent": "可选，仅 image_edit 时填写",
|
||||
"targetSource": "explicit_selection|current_upload|text_reference|working_memory|latest_result|clarification|none",
|
||||
"preferredCapability": "respond|analyze|generic_edit|remove_background|enhance|extend|resize|box_cutout|generate_image|generate_video|publish_canvas",
|
||||
"responseMode": "direct_answer|tool_then_answer|tool_then_execute|clarify",
|
||||
"shouldUseHistoryImage": true,
|
||||
"shouldUseStructuredReply": false,
|
||||
"confidence": 0.92,
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "analyzeImage",
|
||||
"reason": "为什么选这个工具",
|
||||
"args": {}
|
||||
}
|
||||
]
|
||||
}`,
|
||||
prompt: [
|
||||
`用户输入：${state.text || "（空）"}`,
|
||||
`有当前或历史图片：${state.hasImage ? "是" : "否"}`,
|
||||
`图片来自历史：${state.imageFromHistory ? "是" : "否"}`,
|
||||
`目标图来源：${state.targetResolutionMode}（${describeTargetSource(state.targetResolutionMode)}）`,
|
||||
`媒体输入来源：${state.mediaSource}`,
|
||||
`当前上传图数量：${state.currentUploadImageCount}`,
|
||||
`历史图数量：${state.historyImageCount}`,
|
||||
`画布选中图数量：${state.selectedImageCount}`,
|
||||
`基础分析：intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`,
|
||||
state.context ? `画布上下文：${compactText(state.context, 600)}` : "",
|
||||
historyContext ? `最近对话：\n${historyContext}` : "",
|
||||
`可用工具：\n${toolCatalog}`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
temperature: 0.1,
|
||||
maxTokens: 420,
|
||||
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
|
||||
} as any);
|
||||
|
||||
const parsed = parseJsonBlock<Record<string, unknown>>(text);
|
||||
const planned = sanitizeAgentPlan(parsed, fallbackPlan, availableTools);
|
||||
console.log(
|
||||
`[Auto Agent] Planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})`,
|
||||
);
|
||||
console.log("[Auto Agent] Execution plan:", planned);
|
||||
return planned;
|
||||
} catch (error) {
|
||||
console.error("[Auto Agent] Planner failed, fallback to heuristic plan:", error);
|
||||
return fallbackPlan;
|
||||
}
|
||||
}
|
||||
|
||||
async function generateAgentConversation(
|
||||
analysis: AnalysisResult,
|
||||
state: AutoAgentStateType
|
||||
@ -1191,38 +1761,72 @@ const analyzeNode = async (state: AutoAgentStateType) => {
|
||||
preferredModelId: state.preferredModelId ?? undefined,
|
||||
});
|
||||
|
||||
const fallbackQuestion = analysis.needsClarification
|
||||
? analysis.clarifyingQuestions?.[0]
|
||||
: undefined;
|
||||
return { analysis };
|
||||
};
|
||||
|
||||
const strategicMessage = await buildStrategicFriendlyMessage(analysis, state);
|
||||
const planNode = async (state: AutoAgentStateType) => {
|
||||
if (!state.analysis) return {};
|
||||
const availableTools = await getEnabledAgentToolDefinitionsForContext({
|
||||
hasImage: state.hasImage,
|
||||
hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0,
|
||||
hasHistoryImage: state.historyImageCount > 0 || state.imageFromHistory,
|
||||
hasCurrentUpload: state.currentUploadImageCount > 0,
|
||||
});
|
||||
const plan = await buildExecutionPlanWithAgent(state, state.analysis, availableTools);
|
||||
const alignedAnalysis = alignAnalysisWithPlan(state.analysis, plan);
|
||||
|
||||
return {
|
||||
analysis,
|
||||
friendlyMessage: strategicMessage || analysis.conversationalReply || fallbackQuestion || null,
|
||||
availableTools,
|
||||
plan,
|
||||
analysis: alignedAnalysis,
|
||||
};
|
||||
};
|
||||
|
||||
const prepareExecutionNode = async (state: AutoAgentStateType) => {
|
||||
if (!state.analysis || !state.plan) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const execution = prepareExecutionFromPlan({
|
||||
plan: state.plan,
|
||||
availableTools: state.availableTools || [],
|
||||
});
|
||||
|
||||
const fallbackQuestion = state.analysis.needsClarification
|
||||
? state.analysis.clarifyingQuestions?.[0]
|
||||
: undefined;
|
||||
|
||||
let friendlyMessage =
|
||||
state.analysis.conversationalReply || fallbackQuestion || null;
|
||||
|
||||
if (state.plan.nextAction === "clarify") {
|
||||
friendlyMessage =
|
||||
friendlyMessage || "我先确认一下你希望我下一步执行什么。";
|
||||
} else if (execution.shouldUseStructuredReply) {
|
||||
const strategicMessage = await buildStrategicFriendlyMessage(state.analysis, state);
|
||||
friendlyMessage =
|
||||
strategicMessage ||
|
||||
state.analysis.conversationalReply ||
|
||||
friendlyMessage;
|
||||
}
|
||||
|
||||
return {
|
||||
execution,
|
||||
friendlyMessage,
|
||||
};
|
||||
};
|
||||
|
||||
const shouldOptimize = (state: AutoAgentStateType) => {
|
||||
if (!state.analysis) return "route";
|
||||
if (!state.analysis || !state.execution) return "route";
|
||||
if (state.analysis.needsClarification) return "route";
|
||||
if (state.analysis.intent === "video_gen") {
|
||||
return "optimize";
|
||||
}
|
||||
if (state.analysis.intent === "image_gen") {
|
||||
return "optimize";
|
||||
}
|
||||
if (state.analysis.intent === "image_edit" && state.analysis.subIntent) {
|
||||
return "optimize";
|
||||
}
|
||||
return "route";
|
||||
return state.execution.shouldOptimizePrompt ? "optimize" : "route";
|
||||
};
|
||||
|
||||
const optimizeNode = async (state: AutoAgentStateType) => {
|
||||
if (!state.analysis) {
|
||||
if (!state.analysis || !state.execution) {
|
||||
return {};
|
||||
}
|
||||
if (state.analysis.intent === "video_gen") {
|
||||
if (state.execution.mode === "generate_video") {
|
||||
const videoPlan = await recommendVideoPlanWithAgent(state);
|
||||
const storyboardPreview = videoPlan.storyboard
|
||||
.slice(0, 4)
|
||||
@ -1250,11 +1854,11 @@ const optimizeNode = async (state: AutoAgentStateType) => {
|
||||
videoPlan,
|
||||
};
|
||||
}
|
||||
if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) {
|
||||
if (state.execution.mode === "edit_image" && !state.analysis.subIntent) {
|
||||
const recommendedImageParams = heuristicImageParams(state.text, state.analysis);
|
||||
return { recommendedImageParams };
|
||||
}
|
||||
if (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen") {
|
||||
if (!state.execution.shouldApplyImagePipeline) {
|
||||
return {};
|
||||
}
|
||||
const { optimizePrompt, optimizePromptWithLLM } = await import("./prompt-optimizer");
|
||||
@ -1297,11 +1901,14 @@ const optimizeNode = async (state: AutoAgentStateType) => {
|
||||
|
||||
const routeNode = async (state: AutoAgentStateType) => {
|
||||
const { routeToModel } = await import("./model-router");
|
||||
const route = await routeToModel({
|
||||
intent: state.analysis?.intent || "chat",
|
||||
subIntent: state.analysis?.subIntent,
|
||||
let route = await routeToModel({
|
||||
intent: state.execution?.routeIntent || state.analysis?.intent || "chat",
|
||||
subIntent: state.execution?.routeSubIntent || state.analysis?.subIntent,
|
||||
hasImage: state.hasImage,
|
||||
});
|
||||
if (state.execution?.shouldApplyImagePipeline) {
|
||||
route = await refineRouteWithAgent(state, route);
|
||||
}
|
||||
return { route };
|
||||
};
|
||||
|
||||
@ -1310,10 +1917,14 @@ let compiledGraph: ReturnType<typeof buildGraph> | null = null;
|
||||
function buildGraph() {
|
||||
return new StateGraph(AutoAgentState)
|
||||
.addNode("analyze", analyzeNode)
|
||||
.addNode("plan", planNode)
|
||||
.addNode("prepareExecution", prepareExecutionNode)
|
||||
.addNode("optimize", optimizeNode)
|
||||
.addNode("routeModel", routeNode)
|
||||
.addEdge(START, "analyze")
|
||||
.addConditionalEdges("analyze", shouldOptimize, {
|
||||
.addEdge("analyze", "plan")
|
||||
.addEdge("plan", "prepareExecution")
|
||||
.addConditionalEdges("prepareExecution", shouldOptimize, {
|
||||
optimize: "optimize",
|
||||
route: "routeModel",
|
||||
})
|
||||
@ -1337,10 +1948,19 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAge
|
||||
imageUrls: params.imageUrls ?? [],
|
||||
assetIds: params.assetIds ?? [],
|
||||
imageFromHistory: params.imageFromHistory ?? false,
|
||||
targetResolutionMode: params.targetResolutionMode ?? "none",
|
||||
mediaSource: params.mediaSource ?? "none",
|
||||
hasCanvasSelection: params.hasCanvasSelection ?? false,
|
||||
selectedImageCount: params.selectedImageCount ?? 0,
|
||||
historyImageCount: params.historyImageCount ?? 0,
|
||||
currentUploadImageCount: params.currentUploadImageCount ?? 0,
|
||||
historyMessages: params.historyMessages ?? [],
|
||||
context: params.context ?? null,
|
||||
preferredModelId: params.preferredModelId ?? null,
|
||||
analysis: null,
|
||||
availableTools: [],
|
||||
plan: null,
|
||||
execution: null,
|
||||
friendlyMessage: null,
|
||||
optimizedPrompt: null,
|
||||
recommendedImageParams: null,
|
||||
@ -1350,6 +1970,8 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAge
|
||||
|
||||
return {
|
||||
analysis: result.analysis ?? undefined,
|
||||
plan: result.plan ?? undefined,
|
||||
execution: result.execution ?? undefined,
|
||||
friendlyMessage: result.friendlyMessage ?? undefined,
|
||||
optimizedPrompt: result.optimizedPrompt ?? undefined,
|
||||
recommendedImageParams: result.recommendedImageParams ?? undefined,
|
||||
|
||||
342
lib/ai/extend/service.ts
Normal file
342
lib/ai/extend/service.ts
Normal file
@ -0,0 +1,342 @@
|
||||
import {
|
||||
createMediaAssetRelation,
|
||||
getMediaAssetsByUserAndUrls,
|
||||
getSystemSetting,
|
||||
upsertMediaAsset,
|
||||
} from "@/lib/db/queries";
|
||||
import { db } from "@/lib/db";
|
||||
import { generatedImage } from "@/lib/db/schema";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
type ExtendConfig = {
|
||||
apiKey?: string;
|
||||
xScale?: number;
|
||||
yScale?: number;
|
||||
};
|
||||
|
||||
export interface ExecuteExtendJobParams {
|
||||
userId: string;
|
||||
imageUrl?: string | null;
|
||||
imageBase64?: string | null;
|
||||
prompt?: string;
|
||||
chatId?: string | null;
|
||||
messageId?: string | null;
|
||||
xScale?: number;
|
||||
yScale?: number;
|
||||
skipGallerySave?: boolean;
|
||||
sourceImageUrl?: string | null;
|
||||
}
|
||||
|
||||
export interface ExecuteExtendJobResult {
|
||||
image_base64: string;
|
||||
thumbnail_url: string;
|
||||
original_url: string;
|
||||
processing_time: number;
|
||||
mediaAssetId?: string | null;
|
||||
generatedImageId?: string | null;
|
||||
}
|
||||
|
||||
export class ExtendServiceError extends Error {
|
||||
status: number;
|
||||
detail?: string;
|
||||
|
||||
constructor(message: string, status = 500, detail?: string) {
|
||||
super(message);
|
||||
this.name = "ExtendServiceError";
|
||||
this.status = status;
|
||||
this.detail = detail;
|
||||
}
|
||||
}
|
||||
|
||||
async function createThumbnail(buffer: Buffer): Promise<Buffer> {
|
||||
try {
|
||||
const sharp = await import("sharp");
|
||||
return await sharp.default(buffer)
|
||||
.resize(500, null, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer();
|
||||
} catch (error) {
|
||||
console.error("[Extend Service] Failed to create thumbnail:", error);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadBufferPairToS3(buffer: Buffer): Promise<{ thumbnail: string; original: string }> {
|
||||
try {
|
||||
const timestamp = Date.now();
|
||||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||||
const thumbnailFilename = `extend-thumb-${timestamp}-${randomStr}.jpg`;
|
||||
const originalFilename = `extend-orig-${timestamp}-${randomStr}.jpg`;
|
||||
const thumbnailBuffer = await createThumbnail(buffer);
|
||||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength,
|
||||
) as ArrayBuffer;
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength,
|
||||
) as ArrayBuffer;
|
||||
const [thumbnailResult, originalResult] = await Promise.all([
|
||||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||||
uploadFile(originalFilename, originalArrayBuffer),
|
||||
]);
|
||||
return {
|
||||
thumbnail: thumbnailResult.url,
|
||||
original: originalResult.url,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[Extend Service] Failed to upload to S3:", error);
|
||||
throw new ExtendServiceError(
|
||||
"图片上传到存储桶失败",
|
||||
500,
|
||||
error instanceof Error ? error.message : "未知错误",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadImageToS3ForAPI(imageBase64: string): Promise<string> {
|
||||
try {
|
||||
const buffer = Buffer.from(imageBase64, "base64");
|
||||
const timestamp = Date.now();
|
||||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||||
const filename = `extend-input-${timestamp}-${randomStr}.jpg`;
|
||||
const arrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength,
|
||||
) as ArrayBuffer;
|
||||
const result = await uploadFile(filename, arrayBuffer);
|
||||
return result.url;
|
||||
} catch (error) {
|
||||
console.error("[Extend Service] Failed to upload input to S3:", error);
|
||||
throw new ExtendServiceError(
|
||||
"图片上传到存储桶失败",
|
||||
500,
|
||||
error instanceof Error ? error.message : "未知错误",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function extendImage(
|
||||
imageUrl: string,
|
||||
apiKey: string,
|
||||
xScale: number,
|
||||
yScale: number,
|
||||
): Promise<string> {
|
||||
const requestBody = {
|
||||
model: "image-out-painting",
|
||||
input: {
|
||||
image_url: imageUrl,
|
||||
},
|
||||
parameters: {
|
||||
x_scale: xScale,
|
||||
y_scale: yScale,
|
||||
best_quality: false,
|
||||
limit_image_size: true,
|
||||
add_watermark: false,
|
||||
},
|
||||
};
|
||||
|
||||
const response = await fetch(
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/image2image/out-painting",
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"X-DashScope-Async": "enable",
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
},
|
||||
);
|
||||
|
||||
const initialResult = await response.json();
|
||||
if (initialResult.code) {
|
||||
throw new ExtendServiceError(
|
||||
"图像拓展API错误",
|
||||
500,
|
||||
initialResult.message || JSON.stringify(initialResult),
|
||||
);
|
||||
}
|
||||
|
||||
const taskId = initialResult.output.task_id;
|
||||
const taskStatusUrl = `https://dashscope.aliyuncs.com/api/v1/tasks/${taskId}`;
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
|
||||
for (let attempt = 0; attempt < 60; attempt += 1) {
|
||||
const statusResponse = await fetch(taskStatusUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
});
|
||||
const result = await statusResponse.json();
|
||||
if (result.output?.task_status === "SUCCEEDED") {
|
||||
return result.output.output_image_url;
|
||||
}
|
||||
if (result.output?.task_status === "FAILED") {
|
||||
throw new ExtendServiceError(
|
||||
"图像拓展任务失败",
|
||||
500,
|
||||
result.output.message || result.output.code || "未知错误",
|
||||
);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
}
|
||||
|
||||
throw new ExtendServiceError("图像拓展任务超时", 500);
|
||||
}
|
||||
|
||||
export function inferExtendOptionsFromText(text: string): Pick<ExecuteExtendJobParams, "xScale" | "yScale"> {
|
||||
const normalizedText = String(text || "").toLowerCase();
|
||||
const scaleMatch = normalizedText.match(/([1-4](?:\.\d+)?)\s*倍/);
|
||||
const scale = scaleMatch ? Number(scaleMatch[1]) : undefined;
|
||||
return {
|
||||
xScale: scale,
|
||||
yScale: scale,
|
||||
};
|
||||
}
|
||||
|
||||
async function getValidatedExtendConfig(): Promise<ExtendConfig> {
|
||||
const extendConfig = await getSystemSetting({ key: "extend_api_config" });
|
||||
if (!extendConfig || !extendConfig.value) {
|
||||
throw new ExtendServiceError("图像拓展API配置未设置，请在系统设置中配置", 400);
|
||||
}
|
||||
|
||||
const config = extendConfig.value as ExtendConfig;
|
||||
if (!config.apiKey) {
|
||||
throw new ExtendServiceError("图像拓展API密钥未配置", 400);
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
export async function executeExtendJob(
|
||||
params: ExecuteExtendJobParams,
|
||||
): Promise<ExecuteExtendJobResult> {
|
||||
const {
|
||||
userId,
|
||||
imageUrl = null,
|
||||
imageBase64 = null,
|
||||
prompt = "图像拓展处理",
|
||||
chatId = null,
|
||||
messageId = null,
|
||||
skipGallerySave = false,
|
||||
sourceImageUrl = imageUrl || null,
|
||||
} = params;
|
||||
|
||||
if (!imageUrl && !imageBase64) {
|
||||
throw new ExtendServiceError("缺少 image_url 或 image_base64 参数", 400);
|
||||
}
|
||||
|
||||
const config = await getValidatedExtendConfig();
|
||||
const xScale = Math.max(1.1, Math.min(Number(params.xScale || config.xScale || 1.5), 4));
|
||||
const yScale = Math.max(1.1, Math.min(Number(params.yScale || config.yScale || 1.5), 4));
|
||||
const inputImageUrl = imageUrl || (imageBase64 ? await uploadImageToS3ForAPI(imageBase64) : "");
|
||||
|
||||
if (!inputImageUrl) {
|
||||
throw new ExtendServiceError("无法获取输入图片URL", 400);
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
const extendedImageUrl = await extendImage(inputImageUrl, config.apiKey!, xScale, yScale);
|
||||
const imageResponse = await fetch(extendedImageUrl);
|
||||
if (!imageResponse.ok) {
|
||||
throw new ExtendServiceError(
|
||||
"下载拓展图片失败",
|
||||
500,
|
||||
`${imageResponse.status} ${imageResponse.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await imageResponse.arrayBuffer());
|
||||
const imageBase64Result = buffer.toString("base64");
|
||||
const uploadResult = await uploadBufferPairToS3(buffer);
|
||||
let mediaAssetId: string | null = null;
|
||||
let generatedImageId: string | null = null;
|
||||
|
||||
if (!skipGallerySave) {
|
||||
try {
|
||||
const [savedImage] = await db
|
||||
.insert(generatedImage)
|
||||
.values({
|
||||
userId,
|
||||
chatId,
|
||||
messageId,
|
||||
thumbnailUrl: uploadResult.thumbnail,
|
||||
originalUrl: uploadResult.original,
|
||||
prompt,
|
||||
modelId: "extend-1.0",
|
||||
imageSize: "2K",
|
||||
})
|
||||
.returning();
|
||||
generatedImageId = savedImage?.id || null;
|
||||
|
||||
const sourceAssets = sourceImageUrl
|
||||
? await getMediaAssetsByUserAndUrls({
|
||||
userId,
|
||||
urls: [sourceImageUrl],
|
||||
})
|
||||
: [];
|
||||
|
||||
const media = await upsertMediaAsset({
|
||||
userId,
|
||||
chatId,
|
||||
mediaType: "image",
|
||||
sourceType: "edited",
|
||||
status: "ready",
|
||||
originalUrl: uploadResult.original,
|
||||
previewUrl: uploadResult.original,
|
||||
thumbnailUrl: uploadResult.thumbnail,
|
||||
displayName: prompt.slice(0, 80),
|
||||
prompt,
|
||||
modelId: "extend-1.0",
|
||||
sourceMessageId: messageId,
|
||||
sourceTable: "GeneratedImage",
|
||||
sourceRowId: generatedImageId,
|
||||
primaryParentAssetId: sourceAssets[0]?.id ?? null,
|
||||
metadata: {
|
||||
xScale,
|
||||
yScale,
|
||||
imageSize: "2K",
|
||||
},
|
||||
});
|
||||
mediaAssetId = media?.id || null;
|
||||
|
||||
if (media?.id && sourceAssets.length > 0) {
|
||||
for (const sourceAsset of sourceAssets) {
|
||||
await createMediaAssetRelation({
|
||||
userId,
|
||||
chatId,
|
||||
fromAssetId: sourceAsset.id,
|
||||
toAssetId: media.id,
|
||||
relationType: "edited_from",
|
||||
sourceMessageId: messageId,
|
||||
metadata: {
|
||||
prompt,
|
||||
tool: "extend",
|
||||
xScale,
|
||||
yScale,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[Extend Service] Failed to save result:", error);
|
||||
throw new ExtendServiceError(
|
||||
"图像拓展成功但保存结果失败",
|
||||
500,
|
||||
error instanceof Error ? error.message : "未知错误",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
image_base64: imageBase64Result,
|
||||
thumbnail_url: uploadResult.thumbnail,
|
||||
original_url: uploadResult.original,
|
||||
processing_time: Number(((Date.now() - startTime) / 1000).toFixed(2)),
|
||||
mediaAssetId,
|
||||
generatedImageId,
|
||||
};
|
||||
}
|
||||
353
lib/ai/resize/service.ts
Normal file
353
lib/ai/resize/service.ts
Normal file
@ -0,0 +1,353 @@
|
||||
import {
|
||||
createMediaAssetRelation,
|
||||
getMediaAssetsByUserAndUrls,
|
||||
upsertMediaAsset,
|
||||
} from "@/lib/db/queries";
|
||||
import { db } from "@/lib/db";
|
||||
import { generatedImage } from "@/lib/db/schema";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||
|
||||
export interface ResizeCanvasOptions {
|
||||
canvasWidth: number;
|
||||
canvasHeight: number;
|
||||
backgroundColor?: string;
|
||||
transparentBackground?: boolean;
|
||||
mimeType?: string;
|
||||
}
|
||||
|
||||
export interface ExecuteResizeJobParams extends ResizeCanvasOptions {
|
||||
userId: string;
|
||||
imageUrl?: string | null;
|
||||
imageBuffer?: Buffer | null;
|
||||
prompt?: string;
|
||||
chatId?: string | null;
|
||||
messageId?: string | null;
|
||||
skipGallerySave?: boolean;
|
||||
sourceImageUrl?: string | null;
|
||||
}
|
||||
|
||||
export interface ExecuteResizeJobResult {
|
||||
image_base64: string;
|
||||
thumbnail_url: string;
|
||||
original_url: string;
|
||||
canvas_width: number;
|
||||
canvas_height: number;
|
||||
mediaAssetId?: string | null;
|
||||
generatedImageId?: string | null;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
export class ResizeServiceError extends Error {
|
||||
status: number;
|
||||
detail?: string;
|
||||
|
||||
constructor(message: string, status = 500, detail?: string) {
|
||||
super(message);
|
||||
this.name = "ResizeServiceError";
|
||||
this.status = status;
|
||||
this.detail = detail;
|
||||
}
|
||||
}
|
||||
|
||||
const COMMON_COLOR_MAP: Record<string, string> = {
|
||||
白: "#FFFFFF",
|
||||
白色: "#FFFFFF",
|
||||
黑: "#000000",
|
||||
黑色: "#000000",
|
||||
灰: "#D1D5DB",
|
||||
灰色: "#D1D5DB",
|
||||
浅灰: "#E5E7EB",
|
||||
深灰: "#6B7280",
|
||||
红: "#EF4444",
|
||||
红色: "#EF4444",
|
||||
蓝: "#3B82F6",
|
||||
蓝色: "#3B82F6",
|
||||
绿色: "#22C55E",
|
||||
绿: "#22C55E",
|
||||
黄色: "#FACC15",
|
||||
黄: "#FACC15",
|
||||
};
|
||||
|
||||
function normalizeHexColor(color?: string | null): string | undefined {
|
||||
const raw = String(color || "").trim();
|
||||
if (!raw) return undefined;
|
||||
if (/^#([0-9a-f]{3}|[0-9a-f]{6})$/i.test(raw)) {
|
||||
return raw.length === 4
|
||||
? `#${raw[1]}${raw[1]}${raw[2]}${raw[2]}${raw[3]}${raw[3]}`
|
||||
: raw;
|
||||
}
|
||||
return COMMON_COLOR_MAP[raw];
|
||||
}
|
||||
|
||||
function normalizeMimeType(
|
||||
mimeType: string | undefined,
|
||||
transparentBackground: boolean,
|
||||
): "image/png" | "image/jpeg" | "image/webp" {
|
||||
if (transparentBackground) return "image/png";
|
||||
const normalized = String(mimeType || "").toLowerCase().trim();
|
||||
if (normalized === "image/png" || normalized === "image/jpeg" || normalized === "image/webp") {
|
||||
return normalized;
|
||||
}
|
||||
return "image/jpeg";
|
||||
}
|
||||
|
||||
async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer> {
|
||||
try {
|
||||
const sharp = await import("sharp");
|
||||
const pipeline = sharp.default(buffer)
|
||||
.resize(500, null, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
});
|
||||
if (mimeType === "image/png") return await pipeline.png({ quality: 80 }).toBuffer();
|
||||
if (mimeType === "image/webp") return await pipeline.webp({ quality: 80 }).toBuffer();
|
||||
return await pipeline.jpeg({ quality: 80 }).toBuffer();
|
||||
} catch (error) {
|
||||
console.error("[Resize Service] Failed to create thumbnail:", error);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadBufferPairToS3(
|
||||
buffer: Buffer,
|
||||
mimeType: string,
|
||||
): Promise<{ thumbnail: string; original: string }> {
|
||||
try {
|
||||
const ext = mimeType.split("/")[1] || "png";
|
||||
const timestamp = Date.now();
|
||||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||||
const thumbnailFilename = `resize-thumb-${timestamp}-${randomStr}.${ext}`;
|
||||
const originalFilename = `resize-orig-${timestamp}-${randomStr}.${ext}`;
|
||||
const thumbnailBuffer = await createThumbnail(buffer, mimeType);
|
||||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength,
|
||||
) as ArrayBuffer;
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength,
|
||||
) as ArrayBuffer;
|
||||
const [thumbnailResult, originalResult] = await Promise.all([
|
||||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||||
uploadFile(originalFilename, originalArrayBuffer),
|
||||
]);
|
||||
return {
|
||||
thumbnail: thumbnailResult.url,
|
||||
original: originalResult.url,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[Resize Service] Failed to upload to S3:", error);
|
||||
throw new ResizeServiceError(
|
||||
"图片上传到存储桶失败",
|
||||
500,
|
||||
error instanceof Error ? error.message : "未知错误",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function renderResizedImage(params: {
|
||||
imageBuffer: Buffer;
|
||||
canvasWidth: number;
|
||||
canvasHeight: number;
|
||||
transparentBackground: boolean;
|
||||
backgroundColor?: string;
|
||||
mimeType: "image/png" | "image/jpeg" | "image/webp";
|
||||
}): Promise<Buffer> {
|
||||
const sharp = await import("sharp");
|
||||
const background = params.transparentBackground
|
||||
? { r: 255, g: 255, b: 255, alpha: 0 }
|
||||
: params.backgroundColor
|
||||
? params.backgroundColor
|
||||
: "#FFFFFF";
|
||||
|
||||
let pipeline = sharp.default(params.imageBuffer)
|
||||
.rotate()
|
||||
.resize({
|
||||
width: params.canvasWidth,
|
||||
height: params.canvasHeight,
|
||||
fit: "contain",
|
||||
background,
|
||||
withoutEnlargement: false,
|
||||
});
|
||||
|
||||
if (params.mimeType === "image/png") {
|
||||
pipeline = pipeline.png({ quality: 95 });
|
||||
} else if (params.mimeType === "image/webp") {
|
||||
pipeline = pipeline.webp({ quality: 95 });
|
||||
} else {
|
||||
pipeline = pipeline.jpeg({ quality: 95 });
|
||||
}
|
||||
|
||||
return pipeline.toBuffer();
|
||||
}
|
||||
|
||||
export function inferResizeOptionsFromText(
|
||||
text: string,
|
||||
): Partial<ResizeCanvasOptions> {
|
||||
const normalizedText = String(text || "").toLowerCase();
|
||||
const dimensionMatch = normalizedText.match(/(\d{2,5})\s*[x×*]\s*(\d{2,5})/i);
|
||||
if (dimensionMatch) {
|
||||
return {
|
||||
canvasWidth: Number(dimensionMatch[1]),
|
||||
canvasHeight: Number(dimensionMatch[2]),
|
||||
transparentBackground: /透明/.test(normalizedText),
|
||||
backgroundColor: normalizeHexColor(
|
||||
normalizedText.match(/(白色|白底|黑色|黑底|灰色|灰底|红色|蓝色|绿色|黄色)/)?.[1] || "",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const ratioMap: Record<string, { width: number; height: number }> = {
|
||||
"1:1": { width: 1080, height: 1080 },
|
||||
"4:5": { width: 1080, height: 1350 },
|
||||
"3:4": { width: 1080, height: 1440 },
|
||||
"4:3": { width: 1440, height: 1080 },
|
||||
"3:2": { width: 1440, height: 960 },
|
||||
"2:3": { width: 960, height: 1440 },
|
||||
"16:9": { width: 1600, height: 900 },
|
||||
"9:16": { width: 900, height: 1600 },
|
||||
};
|
||||
const ratioMatch = normalizedText.match(/\b(1:1|4:5|3:4|4:3|3:2|2:3|16:9|9:16)\b/);
|
||||
const ratioSize = ratioMatch ? ratioMap[ratioMatch[1]] : undefined;
|
||||
return {
|
||||
canvasWidth: ratioSize?.width,
|
||||
canvasHeight: ratioSize?.height,
|
||||
transparentBackground: /透明/.test(normalizedText),
|
||||
backgroundColor: normalizeHexColor(
|
||||
normalizedText.match(/(白色|白底|黑色|黑底|灰色|灰底|红色|蓝色|绿色|黄色)/)?.[1] || "",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeResizeJob(
|
||||
params: ExecuteResizeJobParams,
|
||||
): Promise<ExecuteResizeJobResult> {
|
||||
const {
|
||||
userId,
|
||||
imageUrl = null,
|
||||
imageBuffer = null,
|
||||
prompt = "尺寸调整处理",
|
||||
chatId = null,
|
||||
messageId = null,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
skipGallerySave = false,
|
||||
sourceImageUrl = imageUrl || null,
|
||||
} = params;
|
||||
|
||||
if (!canvasWidth || !canvasHeight) {
|
||||
throw new ResizeServiceError("缺少 canvasWidth 或 canvasHeight 参数", 400);
|
||||
}
|
||||
|
||||
const transparentBackground = params.transparentBackground === true;
|
||||
const mimeType = normalizeMimeType(params.mimeType, transparentBackground);
|
||||
const backgroundColor = normalizeHexColor(params.backgroundColor) || "#FFFFFF";
|
||||
const sourceBuffer =
|
||||
imageBuffer || (imageUrl ? await fetchImageBufferFromUrl(imageUrl) : null);
|
||||
|
||||
if (!sourceBuffer) {
|
||||
throw new ResizeServiceError("缺少可处理的图片输入", 400);
|
||||
}
|
||||
|
||||
const outputBuffer = await renderResizedImage({
|
||||
imageBuffer: sourceBuffer,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
transparentBackground,
|
||||
backgroundColor,
|
||||
mimeType,
|
||||
});
|
||||
|
||||
const imageBase64 = outputBuffer.toString("base64");
|
||||
const uploadResult = await uploadBufferPairToS3(outputBuffer, mimeType);
|
||||
let mediaAssetId: string | null = null;
|
||||
let generatedImageId: string | null = null;
|
||||
|
||||
if (!skipGallerySave) {
|
||||
try {
|
||||
const [savedImage] = await db
|
||||
.insert(generatedImage)
|
||||
.values({
|
||||
userId,
|
||||
chatId,
|
||||
messageId,
|
||||
thumbnailUrl: uploadResult.thumbnail,
|
||||
originalUrl: uploadResult.original,
|
||||
imageSize: `${canvasWidth}x${canvasHeight}`,
|
||||
prompt,
|
||||
modelId: "resize-1.0",
|
||||
})
|
||||
.returning();
|
||||
generatedImageId = savedImage?.id || null;
|
||||
|
||||
const sourceAssets = sourceImageUrl
|
||||
? await getMediaAssetsByUserAndUrls({
|
||||
userId,
|
||||
urls: [sourceImageUrl],
|
||||
})
|
||||
: [];
|
||||
|
||||
const media = await upsertMediaAsset({
|
||||
userId,
|
||||
chatId,
|
||||
mediaType: "image",
|
||||
sourceType: "edited",
|
||||
status: "ready",
|
||||
originalUrl: uploadResult.original,
|
||||
previewUrl: uploadResult.original,
|
||||
thumbnailUrl: uploadResult.thumbnail,
|
||||
displayName: prompt.slice(0, 80),
|
||||
prompt,
|
||||
modelId: "resize-1.0",
|
||||
sourceMessageId: messageId,
|
||||
sourceTable: "GeneratedImage",
|
||||
sourceRowId: generatedImageId,
|
||||
primaryParentAssetId: sourceAssets[0]?.id ?? null,
|
||||
metadata: {
|
||||
imageSize: `${canvasWidth}x${canvasHeight}`,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
transparentBackground,
|
||||
backgroundColor: transparentBackground ? undefined : backgroundColor,
|
||||
},
|
||||
});
|
||||
mediaAssetId = media?.id || null;
|
||||
|
||||
if (media?.id && sourceAssets.length > 0) {
|
||||
for (const sourceAsset of sourceAssets) {
|
||||
await createMediaAssetRelation({
|
||||
userId,
|
||||
chatId,
|
||||
fromAssetId: sourceAsset.id,
|
||||
toAssetId: media.id,
|
||||
relationType: "edited_from",
|
||||
sourceMessageId: messageId,
|
||||
metadata: {
|
||||
prompt,
|
||||
tool: "resize",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[Resize Service] Failed to save result:", error);
|
||||
throw new ResizeServiceError(
|
||||
"尺寸调整成功但保存结果失败",
|
||||
500,
|
||||
error instanceof Error ? error.message : "未知错误",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
image_base64: imageBase64,
|
||||
thumbnail_url: uploadResult.thumbnail,
|
||||
original_url: uploadResult.original,
|
||||
canvas_width: canvasWidth,
|
||||
canvas_height: canvasHeight,
|
||||
mediaAssetId,
|
||||
generatedImageId,
|
||||
mimeType,
|
||||
};
|
||||
}
|
||||
@ -39,6 +39,8 @@ export type ExecuteRmbgJobResult = {
|
||||
original_url: string;
|
||||
processing_time: number;
|
||||
provider: RmbgProvider;
|
||||
mediaAssetId?: string | null;
|
||||
generatedImageId?: string | null;
|
||||
};
|
||||
|
||||
export class RmbgServiceError extends Error {
|
||||
@ -540,6 +542,7 @@ export async function executeRmbgJob(
|
||||
if (!skipGallerySave) {
|
||||
let saved = false;
|
||||
let saveError: unknown = null;
|
||||
let mediaAssetId: string | null = null;
|
||||
let savedImage:
|
||||
| {
|
||||
id: string;
|
||||
@ -606,6 +609,7 @@ export async function executeRmbgJob(
|
||||
imageSize: "2K",
|
||||
},
|
||||
});
|
||||
mediaAssetId = media?.id || null;
|
||||
|
||||
if (media) {
|
||||
for (const sourceAsset of sourceAssets) {
|
||||
@ -626,6 +630,16 @@ export async function executeRmbgJob(
|
||||
} catch (assetError) {
|
||||
console.error("[RMBG] ⚠️ MediaAsset 注册失败:", assetError);
|
||||
}
|
||||
|
||||
return {
|
||||
image_base64: resultBase64,
|
||||
thumbnail_url: thumbnailUrl,
|
||||
original_url: originalUrl,
|
||||
processing_time: processingTime,
|
||||
provider,
|
||||
mediaAssetId,
|
||||
generatedImageId: savedImage.id,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
console.log("[RMBG] Skip gallery save by request flag");
|
||||
@ -637,5 +651,7 @@ export async function executeRmbgJob(
|
||||
original_url: originalUrl,
|
||||
processing_time: processingTime,
|
||||
provider,
|
||||
mediaAssetId: null,
|
||||
generatedImageId: null,
|
||||
};
|
||||
}
|
||||
|
||||
@ -41,6 +41,39 @@ function isVideoGenerationRequest(text: string): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeCompactText(text: string): string {
|
||||
return text.toLowerCase().replace(/\s+/g, "").trim();
|
||||
}
|
||||
|
||||
function isExplicitImageGenerationRequest(text: string): boolean {
|
||||
const normalizedText = text.toLowerCase();
|
||||
return /重新生成|再来一张|生成新图|新建图片|创作一张|参考.*生成|按这个风格生成|以这张图为灵感生成|仿照.*生成|出一张/.test(
|
||||
normalizedText,
|
||||
);
|
||||
}
|
||||
|
||||
function isLikelyImageEditRequest(text: string): boolean {
|
||||
const normalizedText = text.toLowerCase();
|
||||
return /抠图|去背景|透明背景|去背|扣图|移除背景|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test(
|
||||
normalizedText,
|
||||
);
|
||||
}
|
||||
|
||||
function isLikelyVisionRequest(text: string): boolean {
|
||||
const normalizedText = text.toLowerCase();
|
||||
return /这是什么|是什么|图里|图片里|画面里|照片里|有什么|有啥|看起来像|像什么|帮我看|看看|看下|识别|分析|描述|介绍|解读|讲讲|解释|评价|评估|适合|优缺点|问题|构图|什么风格|风格是什么|材质|元素|内容|文案|品牌感|商业化|电商|卖点|哪里不好|哪里有问题|能看出/.test(
|
||||
normalizedText,
|
||||
);
|
||||
}
|
||||
|
||||
function isLikelyGeneralChatTurn(text: string): boolean {
|
||||
const compactText = normalizeCompactText(text);
|
||||
if (!compactText) return false;
|
||||
return /^(你好|您好|hello|hi|嗨|哈喽|在吗|谢谢|感谢|收到|好的|ok|okay|算了|不用了|明白了|了解了)$/.test(
|
||||
compactText,
|
||||
);
|
||||
}
|
||||
|
||||
function applyDirectExecutionPolicy(
|
||||
result: AnalysisResult,
|
||||
params: {
|
||||
@ -70,24 +103,72 @@ function applyDirectExecutionPolicy(
|
||||
params.hasImage || Boolean(params.imageFromHistory) || (params.imageUrls?.length || 0) > 0;
|
||||
if (!hasUsableImage) return result;
|
||||
|
||||
const compactText = params.text.replace(/\s+/g, "").trim();
|
||||
const baseResult = normalizeIntentForImageContext(result, params.text, hasUsableImage);
|
||||
const compactText = normalizeCompactText(params.text);
|
||||
const hasAnyRequestText = compactText.length > 0;
|
||||
const keepClarification = Boolean(result.needsClarification);
|
||||
const normalizedIntent = result.intent === "chat" || result.intent === "vision" ? "image_edit" : result.intent;
|
||||
const explicitEditRequest = hasAnyRequestText && isLikelyImageEditRequest(params.text);
|
||||
const explicitImageGenerationRequest = hasAnyRequestText && isExplicitImageGenerationRequest(params.text);
|
||||
const likelyVisionRequest = hasAnyRequestText && isLikelyVisionRequest(params.text);
|
||||
const likelyGeneralChatTurn = hasAnyRequestText && isLikelyGeneralChatTurn(params.text);
|
||||
|
||||
if (!hasAnyRequestText) {
|
||||
return {
|
||||
...baseResult,
|
||||
intent: "image_edit",
|
||||
subIntent: baseResult.subIntent || inferDefaultImageEditSubIntent(params.text),
|
||||
confidence: Math.max(baseResult.confidence, 0.72),
|
||||
needsClarification: keepClarification,
|
||||
clarifyingQuestions: keepClarification ? result.clarifyingQuestions || [] : [],
|
||||
conversationalReply:
|
||||
baseResult.conversationalReply || "收到图片，我先按保真优化策略直接执行生成。",
|
||||
};
|
||||
}
|
||||
|
||||
if (!explicitEditRequest && !explicitImageGenerationRequest) {
|
||||
if (likelyVisionRequest || baseResult.intent === "vision") {
|
||||
return {
|
||||
...baseResult,
|
||||
intent: "vision",
|
||||
subIntent: undefined,
|
||||
confidence: Math.max(baseResult.confidence, 0.78),
|
||||
needsClarification: false,
|
||||
clarifyingQuestions: [],
|
||||
conversationalReply:
|
||||
baseResult.conversationalReply || "收到，我先看看这张图，再直接告诉你我的判断。",
|
||||
};
|
||||
}
|
||||
|
||||
if (likelyGeneralChatTurn || baseResult.intent === "chat") {
|
||||
return {
|
||||
...baseResult,
|
||||
intent: "chat",
|
||||
subIntent: undefined,
|
||||
confidence: Math.max(baseResult.confidence, 0.72),
|
||||
needsClarification: keepClarification,
|
||||
clarifyingQuestions: keepClarification ? result.clarifyingQuestions || [] : [],
|
||||
conversationalReply:
|
||||
baseResult.conversationalReply || "收到，我们先围绕这张图继续聊。",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const normalizedIntent =
|
||||
explicitEditRequest && baseResult.intent !== "image_gen" ? "image_edit" : baseResult.intent;
|
||||
const normalizedSubIntent =
|
||||
normalizedIntent === "image_edit"
|
||||
? result.subIntent || inferDefaultImageEditSubIntent(params.text)
|
||||
: result.subIntent;
|
||||
? baseResult.subIntent || inferDefaultImageEditSubIntent(params.text)
|
||||
: baseResult.subIntent;
|
||||
|
||||
return {
|
||||
...result,
|
||||
...baseResult,
|
||||
intent: normalizedIntent,
|
||||
subIntent: normalizedSubIntent,
|
||||
confidence: Math.max(result.confidence, 0.72),
|
||||
confidence: Math.max(baseResult.confidence, 0.72),
|
||||
needsClarification: keepClarification,
|
||||
clarifyingQuestions: keepClarification ? result.clarifyingQuestions || [] : [],
|
||||
conversationalReply:
|
||||
result.conversationalReply ||
|
||||
baseResult.conversationalReply ||
|
||||
(hasAnyRequestText ? "收到，我会按你的描述直接执行生成。" : "收到图片，我先按保真优化策略直接执行生成。"),
|
||||
};
|
||||
}
|
||||
@ -102,10 +183,7 @@ function normalizeIntentForImageContext(
|
||||
}
|
||||
|
||||
const normalizedText = text.toLowerCase();
|
||||
const explicitGenerate =
|
||||
/重新生成|再来一张|生成新图|新建图片|创作一张|参考.*生成|按这个风格生成|以这张图为灵感生成/.test(
|
||||
normalizedText
|
||||
);
|
||||
const explicitGenerate = isExplicitImageGenerationRequest(normalizedText);
|
||||
const editSignal =
|
||||
/改|调整|优化|去掉|换成|改成|保留|继续|再.*一点|修|抠|替换|增强|清晰|重绘/.test(normalizedText);
|
||||
|
||||
@ -122,11 +200,11 @@ function normalizeIntentForImageContext(
|
||||
}
|
||||
|
||||
// 系统提示词 - 针对图片编辑场景优化，生成自然对话
|
||||
const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI图片编辑助手。用户会给你发送图片和编辑请求，你需要：
|
||||
const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI多模态Agent。用户可能会让你聊天、看图分析、修图、生图或生视频，你需要：
|
||||
1. 理解用户想要对图片做什么
|
||||
2. 用自然、友好的语气回复用户，告诉他你理解了他的需求并会帮他处理
|
||||
3. 同时输出结构化的意图分析结果
|
||||
4. 默认直接执行，不要反问用户；只有在完全无图且需求为空时才允许澄清
|
||||
4. 只有用户明确要求执行图片/视频任务时，才输出 image_edit / image_gen / video_gen；如果用户是在问图、聊图、要建议或分析，就输出 vision 或 chat；只有“只发图没说话”时，才允许默认按保真优化执行
|
||||
|
||||
## 意图分类
|
||||
|
||||
@ -204,6 +282,22 @@ const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI图片编辑助手。用
|
||||
"needsClarification": false
|
||||
}
|
||||
|
||||
用户发了图片并问："图片里是什么？"
|
||||
{
|
||||
"intent": "vision",
|
||||
"confidence": 0.92,
|
||||
"conversationalReply": "我先看一下这张图，然后直接告诉你内容和判断~",
|
||||
"needsClarification": false
|
||||
}
|
||||
|
||||
用户发了图片并问："这个 logo 适合做品牌吗？"
|
||||
{
|
||||
"intent": "chat",
|
||||
"confidence": 0.86,
|
||||
"conversationalReply": "可以，我先结合这张图聊聊它的品牌感和适用场景~",
|
||||
"needsClarification": false
|
||||
}
|
||||
|
||||
连续对话场景 - 用户说"换成蓝色"（系统自动使用历史图片）：
|
||||
{
|
||||
"intent": "image_edit",
|
||||
@ -2009,7 +2103,7 @@ export function fallbackKeywordAnalysis(
|
||||
}
|
||||
|
||||
// 风格迁移
|
||||
if (/风格|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格/.test(lowerText)) {
|
||||
if (/风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|转换为.*风格/.test(lowerText)) {
|
||||
const styleMatch = text.match(/(?:转换为|变成|改成|转成)(.+?)(?:风格|$)/);
|
||||
const style = styleMatch ? styleMatch[1].trim() : "新";
|
||||
return {
|
||||
@ -2063,7 +2157,23 @@ export function fallbackKeywordAnalysis(
|
||||
};
|
||||
}
|
||||
|
||||
// 有图但意图不明确
|
||||
if (isLikelyGeneralChatTurn(text)) {
|
||||
return {
|
||||
intent: "chat",
|
||||
confidence: 0.78,
|
||||
conversationalReply: "收到，我们先继续聊这张图~",
|
||||
};
|
||||
}
|
||||
|
||||
if (text.trim()) {
|
||||
return {
|
||||
intent: "vision",
|
||||
confidence: 0.72,
|
||||
conversationalReply: "收到，我先看看这张图，再根据你的问题继续分析~",
|
||||
};
|
||||
}
|
||||
|
||||
// 有图但没文本
|
||||
return {
|
||||
intent: "image_edit",
|
||||
subIntent: inferDefaultImageEditSubIntent(text),
|
||||
|
||||
@ -1,34 +1,220 @@
|
||||
import { getAllToolConfigs } from "@/lib/db/queries";
|
||||
|
||||
export type ToolRegistry = {
|
||||
getWeather: boolean;
|
||||
createDocument: boolean;
|
||||
updateDocument: boolean;
|
||||
requestSuggestions: boolean;
|
||||
generateVideo: boolean;
|
||||
};
|
||||
export type RuntimeToolName =
|
||||
| "getWeather"
|
||||
| "createDocument"
|
||||
| "updateDocument"
|
||||
| "requestSuggestions"
|
||||
| "generateVideo";
|
||||
|
||||
export type AgentProjectToolName =
|
||||
| RuntimeToolName
|
||||
| "respondDirect"
|
||||
| "analyzeImage"
|
||||
| "generateImage"
|
||||
| "editImage"
|
||||
| "removeBackground"
|
||||
| "enhanceImage"
|
||||
| "extendImage"
|
||||
| "resizeImage"
|
||||
| "boxCutout"
|
||||
| "publishToCanvas"
|
||||
| "useSelectedCanvasImage"
|
||||
| "useLatestResultImage"
|
||||
| "useOriginalUploadImage";
|
||||
|
||||
export type ToolRegistry = Record<RuntimeToolName, boolean>;
|
||||
|
||||
export interface AgentProjectToolDefinition {
|
||||
name: AgentProjectToolName;
|
||||
title: string;
|
||||
description: string;
|
||||
category: "conversation" | "image" | "video" | "document" | "utility" | "canvas";
|
||||
runtimeTool?: RuntimeToolName;
|
||||
requiresImage?: boolean;
|
||||
requiresCanvasSelection?: boolean;
|
||||
requiresHistoryImage?: boolean;
|
||||
requiresCurrentUpload?: boolean;
|
||||
producesMedia?: boolean;
|
||||
}
|
||||
|
||||
export interface AgentToolContext {
|
||||
hasImage?: boolean;
|
||||
hasCanvasSelection?: boolean;
|
||||
hasHistoryImage?: boolean;
|
||||
hasCurrentUpload?: boolean;
|
||||
}
|
||||
|
||||
const RUNTIME_TOOL_DEFINITIONS: Array<AgentProjectToolDefinition & { runtimeTool: RuntimeToolName }> = [
|
||||
{
|
||||
name: "getWeather",
|
||||
title: "天气查询",
|
||||
description: "查询天气并在对话中直接回答用户。",
|
||||
category: "utility",
|
||||
runtimeTool: "getWeather",
|
||||
},
|
||||
{
|
||||
name: "createDocument",
|
||||
title: "创建文档",
|
||||
description: "按用户要求创建文档、草稿或结构化内容。",
|
||||
category: "document",
|
||||
runtimeTool: "createDocument",
|
||||
},
|
||||
{
|
||||
name: "updateDocument",
|
||||
title: "更新文档",
|
||||
description: "继续修改已有文档、方案或草稿内容。",
|
||||
category: "document",
|
||||
runtimeTool: "updateDocument",
|
||||
},
|
||||
{
|
||||
name: "requestSuggestions",
|
||||
title: "内容建议",
|
||||
description: "生成补充建议、优化意见或下一步选项。",
|
||||
category: "document",
|
||||
runtimeTool: "requestSuggestions",
|
||||
},
|
||||
{
|
||||
name: "generateVideo",
|
||||
title: "视频生成",
|
||||
description: "根据文本或参考图生成视频。",
|
||||
category: "video",
|
||||
runtimeTool: "generateVideo",
|
||||
producesMedia: true,
|
||||
},
|
||||
];
|
||||
|
||||
const INTERNAL_AGENT_TOOLS: AgentProjectToolDefinition[] = [
|
||||
{
|
||||
name: "respondDirect",
|
||||
title: "直接回复",
|
||||
description: "仅对话、解释、建议或策略讨论，不执行生图生视频。",
|
||||
category: "conversation",
|
||||
},
|
||||
{
|
||||
name: "analyzeImage",
|
||||
title: "图片分析",
|
||||
description: "对当前图或历史图做识别、风格分析、质量评估、品牌/营销建议。",
|
||||
category: "image",
|
||||
requiresImage: true,
|
||||
},
|
||||
{
|
||||
name: "generateImage",
|
||||
title: "图片生成",
|
||||
description: "根据文本或参考风格生成新图片。",
|
||||
category: "image",
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "editImage",
|
||||
title: "图片编辑",
|
||||
description: "基于当前图进行抠图、换背景、增强、调色、尺寸调整、局部重绘等。",
|
||||
category: "image",
|
||||
requiresImage: true,
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "removeBackground",
|
||||
title: "抠图去背",
|
||||
description: "提取主体、去除背景，适合商品图、人像、元素裁切。",
|
||||
category: "image",
|
||||
requiresImage: true,
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "enhanceImage",
|
||||
title: "清晰增强",
|
||||
description: "提升清晰度、细节和质感，适合低清图修复与放大前预处理。",
|
||||
category: "image",
|
||||
requiresImage: true,
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "extendImage",
|
||||
title: "扩图补边",
|
||||
description: "向四周延展画面，适合改比例、补全背景或生成更大构图。",
|
||||
category: "image",
|
||||
requiresImage: true,
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "resizeImage",
|
||||
title: "尺寸调整",
|
||||
description: "调整画布尺寸、背景色与构图位置，适合电商主图、社媒比例适配。",
|
||||
category: "image",
|
||||
requiresImage: true,
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "boxCutout",
|
||||
title: "框选抠图",
|
||||
description: "对目标图中的局部区域做框选裁切，再提取被框选主体。",
|
||||
category: "image",
|
||||
requiresImage: true,
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "publishToCanvas",
|
||||
title: "发布到画布",
|
||||
description: "把当前结果图继续发送到无限画布或作为新的画布结果节点。",
|
||||
category: "canvas",
|
||||
producesMedia: false,
|
||||
},
|
||||
{
|
||||
name: "useSelectedCanvasImage",
|
||||
title: "使用选中图",
|
||||
description: "将当前画布里选中的图片作为本轮分析或编辑目标。",
|
||||
category: "canvas",
|
||||
requiresImage: true,
|
||||
requiresCanvasSelection: true,
|
||||
},
|
||||
{
|
||||
name: "useLatestResultImage",
|
||||
title: "使用最新结果图",
|
||||
description: "将上一轮生成结果作为连续对话和继续编辑的目标图。",
|
||||
category: "canvas",
|
||||
requiresImage: true,
|
||||
requiresHistoryImage: true,
|
||||
},
|
||||
{
|
||||
name: "useOriginalUploadImage",
|
||||
title: "使用原上传图",
|
||||
description: "将最近上传的原图作为分析、修图或对比基准。",
|
||||
category: "canvas",
|
||||
requiresImage: true,
|
||||
requiresCurrentUpload: true,
|
||||
},
|
||||
];
|
||||
|
||||
const DEFAULT_RUNTIME_TOOLS = RUNTIME_TOOL_DEFINITIONS.map((tool) => tool.runtimeTool);
|
||||
|
||||
function getAllAgentToolDefinitions(): AgentProjectToolDefinition[] {
|
||||
return [...INTERNAL_AGENT_TOOLS, ...RUNTIME_TOOL_DEFINITIONS];
|
||||
}
|
||||
|
||||
function matchesAgentToolContext(
|
||||
tool: AgentProjectToolDefinition,
|
||||
context?: AgentToolContext,
|
||||
) {
|
||||
if (!context) return true;
|
||||
if (tool.requiresImage && !context.hasImage) return false;
|
||||
if (tool.requiresCanvasSelection && !context.hasCanvasSelection) return false;
|
||||
if (tool.requiresHistoryImage && !context.hasHistoryImage) return false;
|
||||
if (tool.requiresCurrentUpload && !context.hasCurrentUpload) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取启用的工具列表
|
||||
* 获取启用的运行时工具列表
|
||||
* 从数据库读取工具配置，如果工具未配置则默认启用
|
||||
*/
|
||||
export async function getEnabledTools(): Promise<string[]> {
|
||||
export async function getEnabledTools(): Promise<RuntimeToolName[]> {
|
||||
try {
|
||||
const configs = await getAllToolConfigs();
|
||||
const enabledTools: string[] = [];
|
||||
const enabledTools: RuntimeToolName[] = [];
|
||||
|
||||
// 默认所有工具都启用
|
||||
const defaultTools = [
|
||||
"getWeather",
|
||||
"createDocument",
|
||||
"updateDocument",
|
||||
"requestSuggestions",
|
||||
"generateVideo",
|
||||
];
|
||||
|
||||
for (const toolName of defaultTools) {
|
||||
for (const toolName of DEFAULT_RUNTIME_TOOLS) {
|
||||
const config = configs.find((c) => c.toolName === toolName);
|
||||
// 如果未配置或配置为启用，则添加到启用列表
|
||||
if (!config || config.isEnabled) {
|
||||
enabledTools.push(toolName);
|
||||
}
|
||||
@ -37,14 +223,20 @@ export async function getEnabledTools(): Promise<string[]> {
|
||||
return enabledTools;
|
||||
} catch (error) {
|
||||
console.error("Failed to get enabled tools, using defaults:", error);
|
||||
// 出错时返回所有默认工具
|
||||
return [
|
||||
"getWeather",
|
||||
"createDocument",
|
||||
"updateDocument",
|
||||
"requestSuggestions",
|
||||
"generateVideo",
|
||||
];
|
||||
return [...DEFAULT_RUNTIME_TOOLS];
|
||||
}
|
||||
}
|
||||
|
||||
export async function getEnabledAgentToolDefinitions(): Promise<AgentProjectToolDefinition[]> {
|
||||
return getEnabledAgentToolDefinitionsForContext();
|
||||
}
|
||||
|
||||
export async function getEnabledAgentToolDefinitionsForContext(
|
||||
context?: AgentToolContext,
|
||||
): Promise<AgentProjectToolDefinition[]> {
|
||||
const runtimeTools = await getEnabledTools();
|
||||
return getAllAgentToolDefinitions().filter((tool) => {
|
||||
if (!tool.runtimeTool) return true;
|
||||
return runtimeTools.includes(tool.runtimeTool);
|
||||
}).filter((tool) => matchesAgentToolContext(tool, context));
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
Loading…
x
Reference in New Issue
Block a user