feat: route canvas agent local edits
This commit is contained in:
134
parent
96827c760c
commit
deacac6e8f
@ -8182,6 +8182,10 @@ export async function POST(request: Request) {
|
||||
defaultImageQualityParams,
|
||||
smartAutoExecution,
|
||||
smartAutoPlan,
|
||||
canvasAgentContext: requestCanvasAgentContext,
|
||||
targetNodeIds: (requestTargetResolution?.targetRefs || [])
|
||||
.map((ref) => ref.nodeId)
|
||||
.filter((nodeId): nodeId is string => Boolean(nodeId)),
|
||||
saveGeneratedImages,
|
||||
};
|
||||
const directImageEditResults = shouldBatchDirectImageEdit
|
||||
|
||||
@ -23,6 +23,7 @@ import {
|
||||
getAiModelAgentCodingModelId,
|
||||
isAiModelMultimodal,
|
||||
} from "@/lib/db/ai-model-flags";
|
||||
import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options";
|
||||
|
||||
interface HistoryMessage {
|
||||
role: "user" | "assistant";
|
||||
@ -82,6 +83,7 @@ export type AgentPreferredCapability =
|
||||
| "extend"
|
||||
| "resize"
|
||||
| "angle_variant"
|
||||
| "local_edit"
|
||||
| "box_cutout"
|
||||
| "generate_image"
|
||||
| "generate_video"
|
||||
@ -253,6 +255,7 @@ function inferPreferredCapability(
|
||||
if (firstTool === "extendImage") return "extend";
|
||||
if (firstTool === "resizeImage") return "resize";
|
||||
if (firstTool === "angleEditImage") return "angle_variant";
|
||||
if (firstTool === "localEditImage") return "local_edit";
|
||||
if (firstTool === "boxCutout") return "box_cutout";
|
||||
if (firstTool === "publishToCanvas" || firstTool === "publishResultToCanvas") return "publish_canvas";
|
||||
if (firstTool === "generateVideo") return "generate_video";
|
||||
@ -278,6 +281,9 @@ function inferPreferredCapability(
|
||||
case "angle_variant":
|
||||
case "generate_angle_variant":
|
||||
return "angle_variant";
|
||||
case "local_edit":
|
||||
case "inpaint_region":
|
||||
return "local_edit";
|
||||
default:
|
||||
return "generic_edit";
|
||||
}
|
||||
@ -501,14 +507,33 @@ function buildFallbackPlan(params: {
|
||||
const baseAction = params.analysis.needsClarification
|
||||
? "clarify"
|
||||
: mapIntentToAction(params.analysis.intent);
|
||||
const toolCalls = sanitizePlannedToolCalls([], params.availableTools, baseAction, {
|
||||
let toolCalls = sanitizePlannedToolCalls([], params.availableTools, baseAction, {
|
||||
targetSource: params.targetResolutionMode,
|
||||
});
|
||||
if (
|
||||
baseAction === "edit_image" &&
|
||||
hasDirectAgentLocalEditIntent(params.text) &&
|
||||
params.availableTools.some((tool) => tool.name === "localEditImage") &&
|
||||
!toolCalls.some((tool) => tool.name === "localEditImage")
|
||||
) {
|
||||
toolCalls = [
|
||||
...toolCalls.filter((tool) => tool.name !== "editTargetImage" && tool.name !== "editImage"),
|
||||
{
|
||||
name: "localEditImage",
|
||||
reason: "用户要求按局部区域或批注执行编辑。",
|
||||
},
|
||||
];
|
||||
}
|
||||
return {
|
||||
goal: createPlanGoalSummary(params.text, params.hasImage),
|
||||
nextAction: baseAction,
|
||||
intent: mapActionToIntent(baseAction, params.analysis.intent),
|
||||
subIntent: baseAction === "edit_image" ? params.analysis.subIntent || "enhance" : undefined,
|
||||
subIntent:
|
||||
baseAction === "edit_image"
|
||||
? hasDirectAgentLocalEditIntent(params.text)
|
||||
? "local_edit"
|
||||
: params.analysis.subIntent || "enhance"
|
||||
: undefined,
|
||||
targetSource: params.targetResolutionMode,
|
||||
preferredCapability: inferPreferredCapability(params.analysis, toolCalls),
|
||||
responseMode: defaultResponseMode(baseAction),
|
||||
@ -1409,6 +1434,7 @@ function sanitizeAgentPlan(
|
||||
"extend",
|
||||
"resize",
|
||||
"angle_variant",
|
||||
"local_edit",
|
||||
"box_cutout",
|
||||
"generate_image",
|
||||
"generate_video",
|
||||
@ -1590,7 +1616,7 @@ async function buildExecutionPlanWithAgent(
|
||||
5. 优先保持一次只选择一个主工具；toolCalls 只从提供的工具列表里选。
|
||||
6. shouldUseStructuredReply 只在 edit_image / generate_image / generate_video 时设为 true。
|
||||
7. targetSource 要忠实反映当前目标图来源：explicit_selection/current_upload/text_reference/working_memory/latest_result/clarification/none。
|
||||
8. preferredCapability 要尽量具体，例如 remove_background / remove_watermark / enhance / extend / resize / angle_variant / box_cutout / publish_canvas。
|
||||
8. preferredCapability 要尽量具体，例如 remove_background / remove_watermark / enhance / extend / resize / angle_variant / local_edit / box_cutout / publish_canvas。
|
||||
9. 必须优先使用新的强语义工具名：resolveTargetImage / analyzeTargetImage / editTargetImage / generateNewImage / publishResultToCanvas。
|
||||
10. 必须遵守当前专家路由约束；如果约束要求纯对话，则绝不能输出 generate_image / edit_image / generate_video。
|
||||
|
||||
@ -1601,7 +1627,7 @@ async function buildExecutionPlanWithAgent(
|
||||
"intent": "chat|vision|image_edit|image_gen|video_gen",
|
||||
"subIntent": "可选，仅 image_edit 时填写",
|
||||
"targetSource": "explicit_selection|current_upload|text_reference|working_memory|latest_result|clarification|none",
|
||||
"preferredCapability": "respond|analyze|generic_edit|remove_background|remove_watermark|enhance|extend|resize|angle_variant|box_cutout|generate_image|generate_video|publish_canvas",
|
||||
"preferredCapability": "respond|analyze|generic_edit|remove_background|remove_watermark|enhance|extend|resize|angle_variant|local_edit|box_cutout|generate_image|generate_video|publish_canvas",
|
||||
"responseMode": "direct_answer|tool_then_answer|tool_then_execute|clarify",
|
||||
"shouldUseHistoryImage": true,
|
||||
"shouldUseStructuredReply": false,
|
||||
|
||||
@ -71,6 +71,7 @@ export function buildImageEditSpecialistRoute(params: {
|
||||
"extendImage",
|
||||
"resizeImage",
|
||||
"angleEditImage",
|
||||
"localEditImage",
|
||||
"boxCutout",
|
||||
"useSelectedCanvasImage",
|
||||
"useLatestResultImage",
|
||||
|
||||
@ -17,6 +17,7 @@ export type DirectAgentToolCapability =
|
||||
| "resize"
|
||||
| "extend"
|
||||
| "angle_variant"
|
||||
| "local_edit"
|
||||
| "box_cutout"
|
||||
| "generate_image"
|
||||
| "generate_video";
|
||||
@ -30,6 +31,7 @@ export type DirectAgentToolName =
|
||||
| "resizeImage"
|
||||
| "extendImage"
|
||||
| "angleEditImage"
|
||||
| "localEditImage"
|
||||
| "boxCutout"
|
||||
| "generateImage"
|
||||
| "generateVideo";
|
||||
@ -102,6 +104,8 @@ function mapDirectToolNameToCapability(
|
||||
return "extend";
|
||||
case "angleEditImage":
|
||||
return "angle_variant";
|
||||
case "localEditImage":
|
||||
return "local_edit";
|
||||
case "boxCutout":
|
||||
return "box_cutout";
|
||||
case "generateImage":
|
||||
@ -210,6 +214,13 @@ export function resolveDirectAgentToolExecution(
|
||||
) {
|
||||
return "angleEditImage";
|
||||
}
|
||||
if (
|
||||
preferredCapability === "local_edit" ||
|
||||
routeSubIntent === "local_edit" ||
|
||||
routeSubIntent === "inpaint_region"
|
||||
) {
|
||||
return "localEditImage";
|
||||
}
|
||||
if (preferredCapability === "box_cutout" || routeSubIntent === "box_cutout") {
|
||||
return "boxCutout";
|
||||
}
|
||||
@ -265,15 +276,17 @@ export function resolveDirectAgentToolExecution(
|
||||
? "extendImage"
|
||||
: hasToolCall("angleEditImage")
|
||||
? "angleEditImage"
|
||||
: hasToolCall("boxCutout")
|
||||
? "boxCutout"
|
||||
: hasToolCall("editTargetImage") || hasToolCall("editImage")
|
||||
? preferredSpecificImageEditTool || "editImage"
|
||||
: hasToolCall("generateNewImage") || hasToolCall("generateImage")
|
||||
? "generateImage"
|
||||
: hasToolCall("generateVideo")
|
||||
? "generateVideo"
|
||||
: null) ||
|
||||
: hasToolCall("localEditImage")
|
||||
? "localEditImage"
|
||||
: hasToolCall("boxCutout")
|
||||
? "boxCutout"
|
||||
: hasToolCall("editTargetImage") || hasToolCall("editImage")
|
||||
? preferredSpecificImageEditTool || "editImage"
|
||||
: hasToolCall("generateNewImage") || hasToolCall("generateImage")
|
||||
? "generateImage"
|
||||
: hasToolCall("generateVideo")
|
||||
? "generateVideo"
|
||||
: null) ||
|
||||
(() => {
|
||||
if (
|
||||
execution?.preferredCapability === "analyze" ||
|
||||
@ -319,6 +332,13 @@ export function resolveDirectAgentToolExecution(
|
||||
) {
|
||||
return "angleEditImage";
|
||||
}
|
||||
if (
|
||||
execution?.preferredCapability === "local_edit" ||
|
||||
execution?.routeSubIntent === "local_edit" ||
|
||||
execution?.routeSubIntent === "inpaint_region"
|
||||
) {
|
||||
return "localEditImage";
|
||||
}
|
||||
if (
|
||||
execution?.preferredCapability === "box_cutout" ||
|
||||
execution?.routeSubIntent === "box_cutout"
|
||||
@ -402,6 +422,8 @@ export function isDirectAgentBatchImageEditCapability(
|
||||
case "extend":
|
||||
case "angle_variant":
|
||||
return true;
|
||||
case "local_edit":
|
||||
return false;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
@ -432,6 +454,7 @@ export function isDirectAgentImageEditToolName(
|
||||
| "resizeImage"
|
||||
| "extendImage"
|
||||
| "angleEditImage"
|
||||
| "localEditImage"
|
||||
| "boxCutout" {
|
||||
switch (toolName) {
|
||||
case "editImage":
|
||||
@ -441,6 +464,7 @@ export function isDirectAgentImageEditToolName(
|
||||
case "resizeImage":
|
||||
case "extendImage":
|
||||
case "angleEditImage":
|
||||
case "localEditImage":
|
||||
case "boxCutout":
|
||||
return true;
|
||||
default:
|
||||
@ -466,6 +490,8 @@ export function getDirectAgentToolLabel(capability: DirectAgentToolCapability):
|
||||
return "Agent工具：图像拓展";
|
||||
case "angle_variant":
|
||||
return "Agent工具：多视角";
|
||||
case "local_edit":
|
||||
return "Agent工具：局部编辑";
|
||||
case "box_cutout":
|
||||
return "Agent工具：框选抠图";
|
||||
case "generate_image":
|
||||
@ -513,6 +539,8 @@ export function getDirectAgentToolSuccessMessage(
|
||||
return "已完成图像拓展，结果如下：";
|
||||
case "angle_variant":
|
||||
return "已完成多视角生成，结果如下：";
|
||||
case "local_edit":
|
||||
return "已完成局部编辑，结果如下：";
|
||||
case "box_cutout":
|
||||
return "已完成框选抠图，结果如下：";
|
||||
case "generate_image":
|
||||
@ -544,6 +572,8 @@ export function getDirectAgentToolTargetMissingMessage(
|
||||
return "请先上传或选中一张图片，我才能继续帮你做图像拓展。";
|
||||
case "angle_variant":
|
||||
return "请先上传或选中一张图片，我才能继续帮你生成多视角变体。";
|
||||
case "local_edit":
|
||||
return "请先选中一张图片并框选局部编辑区域，我才能继续帮你做局部编辑。";
|
||||
case "box_cutout":
|
||||
return "请先选中一张图片并提供框选区域，我才能继续帮你做框选抠图。";
|
||||
case "generate_image":
|
||||
@ -584,6 +614,8 @@ export function getDirectAgentToolErrorMessage(
|
||||
return `图像拓展暂时失败：${detail}`;
|
||||
case "angle_variant":
|
||||
return `多视角生成暂时失败：${detail}`;
|
||||
case "local_edit":
|
||||
return `局部编辑暂时失败：${detail}`;
|
||||
case "box_cutout":
|
||||
return `框选抠图暂时失败：${detail}`;
|
||||
case "generate_image":
|
||||
|
||||
@ -35,6 +35,12 @@ import {
|
||||
inferAngleEditOptionsFromText,
|
||||
type DirectAgentAngleEditOptions,
|
||||
} from "@/lib/ai/runtime/angle-edit-options";
|
||||
import {
|
||||
DirectAgentLocalEditServiceError,
|
||||
resolveDirectAgentLocalEditRegions,
|
||||
type DirectAgentLocalEditRegion,
|
||||
} from "@/lib/ai/runtime/local-edit-options";
|
||||
import type { CanvasAgentContext } from "@/types/canvas/agent-protocol";
|
||||
|
||||
interface DirectAgentImageParamsSnapshot {
|
||||
negativePrompt?: string;
|
||||
@ -101,6 +107,8 @@ export interface ExecuteDirectAgentImageEditToolParams {
|
||||
defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null;
|
||||
smartAutoExecution?: AgentPreparedExecution;
|
||||
smartAutoPlan?: AgentExecutionPlan;
|
||||
canvasAgentContext?: CanvasAgentContext | null;
|
||||
targetNodeIds?: string[] | null;
|
||||
saveGeneratedImages: SaveGeneratedImages;
|
||||
}
|
||||
|
||||
@ -109,6 +117,10 @@ function toFiniteNumber(value: unknown): number | undefined {
|
||||
return Number.isFinite(normalized) ? normalized : undefined;
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function executeDirectAgentEnhanceJob(params: {
|
||||
requestUrl: string;
|
||||
requestHeaders: Headers;
|
||||
@ -221,12 +233,137 @@ async function executeDirectAgentAngleEditJob(params: {
|
||||
};
|
||||
}
|
||||
|
||||
async function safeParseJsonResponse(response: Response) {
|
||||
try {
|
||||
return await response.json();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function pollDirectAgentCanvasImageTaskResult(params: {
|
||||
requestUrl: string;
|
||||
requestHeaders: Headers;
|
||||
taskId: string;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const cookieHeader = params.requestHeaders.get("cookie") || "";
|
||||
const endpoint = new URL("/api/canvas/generate/image", params.requestUrl);
|
||||
endpoint.searchParams.set("taskId", params.taskId);
|
||||
const timeoutMs = Math.max(10_000, params.timeoutMs ?? 20 * 60 * 1000);
|
||||
const startedAt = Date.now();
|
||||
let pollAfterMs = 1500;
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
await sleep(pollAfterMs);
|
||||
|
||||
const response = await fetch(endpoint.toString(), {
|
||||
method: "GET",
|
||||
cache: "no-store",
|
||||
headers: cookieHeader ? { cookie: cookieHeader } : undefined,
|
||||
});
|
||||
const payload = await safeParseJsonResponse(response);
|
||||
if (!response.ok || payload?.success === false) {
|
||||
const detail = String(payload?.error || payload?.detail || "").trim();
|
||||
throw new DirectAgentLocalEditServiceError(
|
||||
detail || `查询局部编辑任务失败(${response.status})`,
|
||||
response.status || 500,
|
||||
);
|
||||
}
|
||||
|
||||
const task = payload?.task || {};
|
||||
const status = String(task.status || "").toLowerCase();
|
||||
if (status === "succeeded") {
|
||||
const result = task.result || {};
|
||||
const images = Array.isArray(result.images) ? result.images : [];
|
||||
const firstImage = images[0];
|
||||
const originalUrl = String(firstImage?.originalUrl || firstImage?.original_url || "").trim();
|
||||
const thumbnailUrl = String(
|
||||
firstImage?.thumbnailUrl ||
|
||||
firstImage?.thumbnail_url ||
|
||||
firstImage?.thumbnail ||
|
||||
originalUrl
|
||||
).trim();
|
||||
if (!originalUrl) {
|
||||
throw new DirectAgentLocalEditServiceError("局部编辑任务完成但未返回图片", 500);
|
||||
}
|
||||
return {
|
||||
originalUrl,
|
||||
thumbnailUrl: thumbnailUrl || originalUrl,
|
||||
imageSize: "2K",
|
||||
modelRef: String(result.modelRef || result.model || "").trim(),
|
||||
};
|
||||
}
|
||||
|
||||
if (status === "failed") {
|
||||
throw new DirectAgentLocalEditServiceError(
|
||||
String(task.error || "局部编辑任务失败"),
|
||||
500,
|
||||
);
|
||||
}
|
||||
|
||||
const estimatedWaitMs = Number(task.queueStatus?.estimatedWaitMs || 0);
|
||||
pollAfterMs = estimatedWaitMs > 0 ? Math.min(4000, Math.max(1200, estimatedWaitMs)) : 1500;
|
||||
}
|
||||
|
||||
throw new DirectAgentLocalEditServiceError("局部编辑任务处理超时，请稍后再试", 504);
|
||||
}
|
||||
|
||||
async function executeDirectAgentLocalEditJob(params: {
|
||||
requestUrl: string;
|
||||
requestHeaders: Headers;
|
||||
imageUrl: string;
|
||||
sourceNodeId?: string;
|
||||
regions: DirectAgentLocalEditRegion[];
|
||||
model?: string;
|
||||
imageSize?: string;
|
||||
aspectRatio?: string;
|
||||
}) {
|
||||
const endpoint = new URL("/api/canvas/local-edit", params.requestUrl).toString();
|
||||
const cookieHeader = params.requestHeaders.get("cookie") || "";
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(cookieHeader ? { cookie: cookieHeader } : {}),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
sourceImageUrl: params.imageUrl,
|
||||
sourceNodeId: params.sourceNodeId,
|
||||
regions: params.regions,
|
||||
operation: "local-edit",
|
||||
model: params.model,
|
||||
imageSize: params.imageSize || "2K",
|
||||
aspectRatio: params.aspectRatio || "auto",
|
||||
}),
|
||||
});
|
||||
const payload = await safeParseJsonResponse(response);
|
||||
if (!response.ok || payload?.success === false) {
|
||||
throw new DirectAgentLocalEditServiceError(
|
||||
String(payload?.error || payload?.detail || "局部编辑提交失败"),
|
||||
response.status || 500,
|
||||
);
|
||||
}
|
||||
|
||||
const taskId = String(payload?.taskId || "").trim();
|
||||
if (!taskId) {
|
||||
throw new DirectAgentLocalEditServiceError("局部编辑提交成功但未返回任务ID", 500);
|
||||
}
|
||||
|
||||
return pollDirectAgentCanvasImageTaskResult({
|
||||
requestUrl: params.requestUrl,
|
||||
requestHeaders: params.requestHeaders,
|
||||
taskId,
|
||||
});
|
||||
}
|
||||
|
||||
export function isDirectAgentImageEditServiceError(error: unknown) {
|
||||
return (
|
||||
error instanceof RmbgServiceError ||
|
||||
error instanceof ResizeServiceError ||
|
||||
error instanceof ExtendServiceError ||
|
||||
error instanceof BoxCutoutServiceError
|
||||
error instanceof BoxCutoutServiceError ||
|
||||
error instanceof DirectAgentLocalEditServiceError
|
||||
);
|
||||
}
|
||||
|
||||
@ -552,6 +689,75 @@ export async function executeDirectAgentImageEditTool(
|
||||
});
|
||||
}
|
||||
|
||||
if (params.capability === "local_edit") {
|
||||
const plannedArgs = findAgentToolCallArgs(
|
||||
"localEditImage",
|
||||
params.smartAutoExecution,
|
||||
params.smartAutoPlan,
|
||||
);
|
||||
const regionResolution = resolveDirectAgentLocalEditRegions({
|
||||
canvasAgentContext: params.canvasAgentContext,
|
||||
plannedArgs,
|
||||
requestTextContent: params.requestTextContent,
|
||||
sourceImageUrl: params.sourceImageUrl,
|
||||
targetNodeIds: params.targetNodeIds,
|
||||
});
|
||||
const aspectRatio =
|
||||
params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio || "auto";
|
||||
const imageSize =
|
||||
params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize || "2K";
|
||||
const result = await executeDirectAgentLocalEditJob({
|
||||
requestUrl: params.requestUrl,
|
||||
requestHeaders: params.requestHeaders,
|
||||
imageUrl: params.sourceImageUrl,
|
||||
sourceNodeId: regionResolution.sourceNodeId,
|
||||
regions: regionResolution.regions,
|
||||
model: params.finalModelId,
|
||||
imageSize,
|
||||
aspectRatio,
|
||||
});
|
||||
const editResultMarkdown = buildAgentToolImagesMarkdown([
|
||||
{
|
||||
originalUrl: result.originalUrl,
|
||||
thumbnailUrl: result.thumbnailUrl,
|
||||
imageSize,
|
||||
},
|
||||
]);
|
||||
|
||||
await params.saveGeneratedImages(
|
||||
editResultMarkdown,
|
||||
params.userId,
|
||||
params.chatId,
|
||||
params.messageId,
|
||||
params.requestTextContent || "局部编辑处理",
|
||||
result.modelRef || params.finalModelId,
|
||||
{
|
||||
sourceAssetIds: params.targetAssetIds || undefined,
|
||||
sourceImageUrls: [params.sourceImageUrl],
|
||||
relationType: "edited_from",
|
||||
sourceType: "canvas-local-edit",
|
||||
projectId: params.projectId || null,
|
||||
userPrompt: params.requestTextContent,
|
||||
optimizedPrompt:
|
||||
regionResolution.regions
|
||||
.map((region, index) => `区域 ${index + 1}：${region.prompt}`)
|
||||
.join("\n") || params.requestTextContent,
|
||||
requestedImageCount: 1,
|
||||
aspectRatio,
|
||||
imageSize,
|
||||
},
|
||||
);
|
||||
|
||||
return finalizeDirectAgentImageEditResult({
|
||||
userId: params.userId,
|
||||
capability: params.capability,
|
||||
originalUrl: result.originalUrl,
|
||||
thumbnailUrl: result.thumbnailUrl,
|
||||
imageSize,
|
||||
mediaAssetId: null,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.capability === "box_cutout") {
|
||||
const plannedArgs = findAgentToolCallArgs(
|
||||
"boxCutout",
|
||||
|
||||
@ -4,6 +4,7 @@ import type { ConversationActionResolution } from "@/lib/ai/conversation-action-
|
||||
import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types";
|
||||
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
|
||||
import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier";
|
||||
import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options";
|
||||
|
||||
export type IntentArbiterMode =
|
||||
| "respond"
|
||||
@ -23,6 +24,7 @@ export type IntentArbiterPreferredCapability =
|
||||
| "extend"
|
||||
| "resize"
|
||||
| "angle_variant"
|
||||
| "local_edit"
|
||||
| "box_cutout"
|
||||
| "generate_image"
|
||||
| "generate_video"
|
||||
@ -115,7 +117,7 @@ function isExplicitImageGenerationRequest(text: string): boolean {
|
||||
|
||||
function isLikelyImageEditRequest(text: string): boolean {
|
||||
const normalizedText = String(text || "").toLowerCase();
|
||||
return /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test(
|
||||
return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test(
|
||||
normalizedText,
|
||||
);
|
||||
}
|
||||
@ -126,6 +128,7 @@ function inferDefaultImageEditSubIntent(text: string): string {
|
||||
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background";
|
||||
if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background";
|
||||
if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant";
|
||||
if (hasDirectAgentLocalEditIntent(text)) return "local_edit";
|
||||
if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend";
|
||||
if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer";
|
||||
if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust";
|
||||
@ -174,6 +177,9 @@ function mapPreferredCapability(
|
||||
case "angle_variant":
|
||||
case "generate_angle_variant":
|
||||
return "angle_variant";
|
||||
case "local_edit":
|
||||
case "inpaint_region":
|
||||
return "local_edit";
|
||||
case "box_cutout":
|
||||
return "box_cutout";
|
||||
case "enhance":
|
||||
|
||||
398
lib/ai/runtime/local-edit-options.ts
Normal file
398
lib/ai/runtime/local-edit-options.ts
Normal file
@ -0,0 +1,398 @@
|
||||
import type {
|
||||
CanvasAgentAnnotationRef,
|
||||
CanvasAgentContext,
|
||||
CanvasAgentNodeRef,
|
||||
} from "@/types/canvas/agent-protocol";
|
||||
|
||||
export interface DirectAgentLocalEditRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface DirectAgentLocalEditRegion {
|
||||
id: string;
|
||||
color?: string;
|
||||
prompt: string;
|
||||
rect: DirectAgentLocalEditRect;
|
||||
targetRect?: DirectAgentLocalEditRect;
|
||||
referenceImages?: Array<{
|
||||
id?: string;
|
||||
url: string;
|
||||
name?: string;
|
||||
source?: "upload" | "canvas";
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface DirectAgentLocalEditRegionResolution {
|
||||
sourceNodeId?: string;
|
||||
regions: DirectAgentLocalEditRegion[];
|
||||
}
|
||||
|
||||
export class DirectAgentLocalEditServiceError extends Error {
|
||||
status: number;
|
||||
|
||||
constructor(message: string, status = 400) {
|
||||
super(message);
|
||||
this.name = "DirectAgentLocalEditServiceError";
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_DIRECT_AGENT_LOCAL_EDIT_REGIONS = 8;
|
||||
const DEFAULT_REGION_COLORS = [
|
||||
"#ff3b30",
|
||||
"#1677ff",
|
||||
"#00c853",
|
||||
"#ff9500",
|
||||
"#af52de",
|
||||
"#00bcd4",
|
||||
"#ff2d55",
|
||||
"#8d6e63",
|
||||
];
|
||||
|
||||
function trimText(value: unknown, maxLength: number) {
|
||||
return typeof value === "string" ? value.trim().slice(0, maxLength) : "";
|
||||
}
|
||||
|
||||
function normalizeCompactText(value: unknown) {
|
||||
return String(value || "").toLowerCase().replace(/\s+/g, "").trim();
|
||||
}
|
||||
|
||||
function toFiniteNumber(value: unknown): number | null {
|
||||
const numberValue = Number(value);
|
||||
return Number.isFinite(numberValue) ? numberValue : null;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.min(max, Math.max(min, value));
|
||||
}
|
||||
|
||||
function roundCoordinate(value: number) {
|
||||
return Math.round(value * 10_000) / 10_000;
|
||||
}
|
||||
|
||||
function normalizeHttpUrl(value: unknown) {
|
||||
const url = String(value || "").trim();
|
||||
return /^https?:\/\//i.test(url) ? url : "";
|
||||
}
|
||||
|
||||
function normalizeColor(value: unknown, fallback: string) {
|
||||
const raw = String(value || "").trim();
|
||||
const expanded = raw.replace(/^#([0-9a-f])([0-9a-f])([0-9a-f])$/i, "#$1$1$2$2$3$3");
|
||||
return /^#[0-9a-f]{6}$/i.test(expanded) ? expanded.toLowerCase() : fallback;
|
||||
}
|
||||
|
||||
export function hasDirectAgentLocalEditIntent(text: string) {
|
||||
const normalizedText = normalizeCompactText(text);
|
||||
if (!normalizedText) return false;
|
||||
|
||||
const annotationKeyword = "(?:批注|标注|箭头|框选|圈选|annotation|note)";
|
||||
if (
|
||||
new RegExp(`(?:添加|新增|创建|加|写|记录).{0,8}${annotationKeyword}`, "i").test(
|
||||
normalizedText
|
||||
)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
/局部编辑|局部重绘|局部修改|局部修图|局部处理|区域编辑|区域重绘|局部换|局部去除|选区/.test(
|
||||
normalizedText
|
||||
) ||
|
||||
/框选区域.{0,24}(?:修改|编辑|修图|改图|处理|调整|去掉|移除|换|生成|出图|执行|重绘)/.test(
|
||||
normalizedText
|
||||
) ||
|
||||
new RegExp(
|
||||
`(?:按|按照|根据|依据|依照|参考|基于).{0,18}${annotationKeyword}.{0,24}(?:修改|编辑|修图|改图|处理|调整|去掉|移除|换|生成|出图|执行|重绘)`,
|
||||
"i"
|
||||
).test(normalizedText) ||
|
||||
new RegExp(
|
||||
`${annotationKeyword}.{0,24}(?:修改|编辑|修图|改图|处理|调整|去掉|移除|换|生成|出图|执行|重绘)`,
|
||||
"i"
|
||||
).test(normalizedText)
|
||||
);
|
||||
}
|
||||
|
||||
export function normalizeDirectAgentLocalEditRect(
|
||||
value: unknown
|
||||
): DirectAgentLocalEditRect | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const raw = value as Record<string, unknown>;
|
||||
const x = toFiniteNumber(raw.x);
|
||||
const y = toFiniteNumber(raw.y);
|
||||
const width = toFiniteNumber(raw.width);
|
||||
const height = toFiniteNumber(raw.height);
|
||||
if (x === null || y === null || width === null || height === null) return null;
|
||||
|
||||
const left = clamp(Math.min(x, x + width), 0, 1);
|
||||
const top = clamp(Math.min(y, y + height), 0, 1);
|
||||
const right = clamp(Math.max(x, x + width), 0, 1);
|
||||
const bottom = clamp(Math.max(y, y + height), 0, 1);
|
||||
const normalizedWidth = clamp(right - left, 0, 1 - left);
|
||||
const normalizedHeight = clamp(bottom - top, 0, 1 - top);
|
||||
if (normalizedWidth <= 0 || normalizedHeight <= 0) return null;
|
||||
|
||||
return {
|
||||
x: roundCoordinate(left),
|
||||
y: roundCoordinate(top),
|
||||
width: roundCoordinate(normalizedWidth),
|
||||
height: roundCoordinate(normalizedHeight),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeReferenceImages(value: unknown): DirectAgentLocalEditRegion["referenceImages"] {
|
||||
if (!Array.isArray(value)) return undefined;
|
||||
const references = value
|
||||
.map((item, index) => {
|
||||
if (!item || typeof item !== "object") return null;
|
||||
const raw = item as Record<string, unknown>;
|
||||
const url = normalizeHttpUrl(raw.url);
|
||||
if (!url) return null;
|
||||
return {
|
||||
id: trimText(raw.id, 80) || `reference-${index + 1}`,
|
||||
url,
|
||||
...(trimText(raw.name, 80) ? { name: trimText(raw.name, 80) } : {}),
|
||||
source: raw.source === "canvas" ? ("canvas" as const) : ("upload" as const),
|
||||
};
|
||||
})
|
||||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||||
.slice(0, 3);
|
||||
return references.length > 0 ? references : undefined;
|
||||
}
|
||||
|
||||
function isGenericRegionPrompt(value: string) {
|
||||
const normalized = normalizeCompactText(value);
|
||||
return (
|
||||
!normalized ||
|
||||
/^局部编辑区域\d*$/.test(normalized) ||
|
||||
/^区域\d+$/.test(normalized) ||
|
||||
/^localeditregion\d*$/.test(normalized)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizePlannedRegion(
|
||||
value: unknown,
|
||||
index: number,
|
||||
fallbackPrompt: string
|
||||
): DirectAgentLocalEditRegion | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const raw = value as Record<string, unknown>;
|
||||
const rect = normalizeDirectAgentLocalEditRect(raw.rect || raw.cropRect || raw.bounds);
|
||||
if (!rect) return null;
|
||||
const prompt =
|
||||
trimText(raw.prompt, 300) ||
|
||||
trimText(raw.instruction, 300) ||
|
||||
trimText(raw.text, 300) ||
|
||||
fallbackPrompt;
|
||||
if (!prompt) return null;
|
||||
const targetRect = normalizeDirectAgentLocalEditRect(raw.targetRect);
|
||||
|
||||
return {
|
||||
id: trimText(raw.id, 80) || `region-${index + 1}`,
|
||||
color: normalizeColor(raw.color, DEFAULT_REGION_COLORS[index % DEFAULT_REGION_COLORS.length]),
|
||||
prompt,
|
||||
rect,
|
||||
...(targetRect ? { targetRect } : {}),
|
||||
...(normalizeReferenceImages(raw.referenceImages)
|
||||
? { referenceImages: normalizeReferenceImages(raw.referenceImages) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function getAllContextNodes(context?: CanvasAgentContext | null): CanvasAgentNodeRef[] {
|
||||
if (!context) return [];
|
||||
const nodes = [...(context.selectedNodes || []), ...(context.candidateNodes || [])];
|
||||
const seen = new Set<string>();
|
||||
return nodes.filter((node) => {
|
||||
if (!node?.nodeId || seen.has(node.nodeId)) return false;
|
||||
seen.add(node.nodeId);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function resolveTargetNodeIds(params: {
|
||||
canvasAgentContext?: CanvasAgentContext | null;
|
||||
sourceImageUrl?: string;
|
||||
targetNodeIds?: string[] | null;
|
||||
}) {
|
||||
const context = params.canvasAgentContext;
|
||||
const sourceUrl = normalizeHttpUrl(params.sourceImageUrl);
|
||||
const targetIds: string[] = [];
|
||||
const addTargetId = (value: unknown) => {
|
||||
const nodeId = trimText(value, 128);
|
||||
if (nodeId && !targetIds.includes(nodeId)) {
|
||||
targetIds.push(nodeId);
|
||||
}
|
||||
};
|
||||
|
||||
const nodes = getAllContextNodes(context);
|
||||
if (sourceUrl) {
|
||||
for (const node of nodes) {
|
||||
if (normalizeHttpUrl(node.url) === sourceUrl) {
|
||||
addTargetId(node.nodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const nodeId of params.targetNodeIds || []) addTargetId(nodeId);
|
||||
for (const nodeId of context?.selectedNodeIds || []) addTargetId(nodeId);
|
||||
for (const node of context?.selectedNodes || []) {
|
||||
if (node.isSelected) addTargetId(node.nodeId);
|
||||
}
|
||||
|
||||
return targetIds;
|
||||
}
|
||||
|
||||
export function resolveDirectAgentLocalEditSourceNodeId(params: {
|
||||
canvasAgentContext?: CanvasAgentContext | null;
|
||||
plannedArgs?: Record<string, unknown>;
|
||||
sourceImageUrl?: string;
|
||||
targetNodeIds?: string[] | null;
|
||||
}) {
|
||||
const plannedSourceNodeId = trimText(params.plannedArgs?.sourceNodeId, 128);
|
||||
if (plannedSourceNodeId) return plannedSourceNodeId;
|
||||
|
||||
const sourceUrl = normalizeHttpUrl(params.sourceImageUrl);
|
||||
if (sourceUrl) {
|
||||
const matchedNode = getAllContextNodes(params.canvasAgentContext).find(
|
||||
(node) => normalizeHttpUrl(node.url) === sourceUrl
|
||||
);
|
||||
if (matchedNode?.nodeId) return matchedNode.nodeId;
|
||||
}
|
||||
|
||||
const [firstTargetNodeId] = resolveTargetNodeIds(params);
|
||||
return firstTargetNodeId;
|
||||
}
|
||||
|
||||
function resolvePlannedRegions(params: {
|
||||
plannedArgs?: Record<string, unknown>;
|
||||
requestTextContent: string;
|
||||
}) {
|
||||
const plannedArgs = params.plannedArgs || {};
|
||||
const fallbackPrompt =
|
||||
trimText(plannedArgs.prompt, 300) ||
|
||||
trimText(plannedArgs.instruction, 300) ||
|
||||
trimText(plannedArgs.text, 300) ||
|
||||
trimText(params.requestTextContent, 300);
|
||||
const rawRegions = Array.isArray(plannedArgs.regions)
|
||||
? plannedArgs.regions
|
||||
: Array.isArray(plannedArgs.localEditRegions)
|
||||
? plannedArgs.localEditRegions
|
||||
: [];
|
||||
const regions = rawRegions
|
||||
.map((region, index) => normalizePlannedRegion(region, index, fallbackPrompt))
|
||||
.filter((region): region is DirectAgentLocalEditRegion => Boolean(region));
|
||||
|
||||
if (regions.length > 0) return regions.slice(0, MAX_DIRECT_AGENT_LOCAL_EDIT_REGIONS);
|
||||
|
||||
const singleRegion = normalizePlannedRegion(
|
||||
{
|
||||
id: plannedArgs.regionId || plannedArgs.id,
|
||||
color: plannedArgs.color,
|
||||
prompt: fallbackPrompt,
|
||||
rect: plannedArgs.rect || plannedArgs.cropRect || plannedArgs.bounds,
|
||||
targetRect: plannedArgs.targetRect,
|
||||
referenceImages: plannedArgs.referenceImages,
|
||||
},
|
||||
0,
|
||||
fallbackPrompt
|
||||
);
|
||||
return singleRegion ? [singleRegion] : [];
|
||||
}
|
||||
|
||||
function normalizeAnnotationRegion(params: {
|
||||
annotation: CanvasAgentAnnotationRef;
|
||||
index: number;
|
||||
requestTextContent: string;
|
||||
plannedPrompt: string;
|
||||
}): DirectAgentLocalEditRegion | null {
|
||||
const rect = normalizeDirectAgentLocalEditRect(params.annotation.bounds);
|
||||
if (!rect) return null;
|
||||
const annotationText = trimText(params.annotation.text, 300);
|
||||
const prompt =
|
||||
annotationText && !isGenericRegionPrompt(annotationText)
|
||||
? annotationText
|
||||
: params.plannedPrompt || trimText(params.requestTextContent, 300) || annotationText;
|
||||
if (!prompt) return null;
|
||||
|
||||
return {
|
||||
id: trimText(params.annotation.id, 80) || `annotation-region-${params.index + 1}`,
|
||||
color: normalizeColor(
|
||||
params.annotation.color,
|
||||
DEFAULT_REGION_COLORS[params.index % DEFAULT_REGION_COLORS.length]
|
||||
),
|
||||
prompt,
|
||||
rect,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveAnnotationRegions(params: {
|
||||
canvasAgentContext?: CanvasAgentContext | null;
|
||||
plannedArgs?: Record<string, unknown>;
|
||||
requestTextContent: string;
|
||||
sourceImageUrl?: string;
|
||||
targetNodeIds?: string[] | null;
|
||||
}) {
|
||||
const context = params.canvasAgentContext;
|
||||
if (!context?.annotations?.length) return [];
|
||||
const targetNodeIds = resolveTargetNodeIds(params);
|
||||
const targetNodeIdSet = new Set(targetNodeIds);
|
||||
const plannedPrompt =
|
||||
trimText(params.plannedArgs?.prompt, 300) ||
|
||||
trimText(params.plannedArgs?.instruction, 300) ||
|
||||
trimText(params.plannedArgs?.text, 300);
|
||||
|
||||
const annotations = context.annotations.filter((annotation) => {
|
||||
if (!annotation?.bounds) return false;
|
||||
if (targetNodeIdSet.size === 0) return true;
|
||||
return targetNodeIdSet.has(annotation.sourceNodeId);
|
||||
});
|
||||
|
||||
return annotations
|
||||
.map((annotation, index) =>
|
||||
normalizeAnnotationRegion({
|
||||
annotation,
|
||||
index,
|
||||
plannedPrompt,
|
||||
requestTextContent: params.requestTextContent,
|
||||
})
|
||||
)
|
||||
.filter((region): region is DirectAgentLocalEditRegion => Boolean(region))
|
||||
.slice(0, MAX_DIRECT_AGENT_LOCAL_EDIT_REGIONS);
|
||||
}
|
||||
|
||||
export function resolveDirectAgentLocalEditRegions(params: {
|
||||
canvasAgentContext?: CanvasAgentContext | null;
|
||||
plannedArgs?: Record<string, unknown>;
|
||||
requestTextContent: string;
|
||||
sourceImageUrl?: string;
|
||||
targetNodeIds?: string[] | null;
|
||||
}): DirectAgentLocalEditRegionResolution {
|
||||
const plannedRegions = resolvePlannedRegions({
|
||||
plannedArgs: params.plannedArgs,
|
||||
requestTextContent: params.requestTextContent,
|
||||
});
|
||||
const regions =
|
||||
plannedRegions.length > 0
|
||||
? plannedRegions
|
||||
: resolveAnnotationRegions({
|
||||
canvasAgentContext: params.canvasAgentContext,
|
||||
plannedArgs: params.plannedArgs,
|
||||
requestTextContent: params.requestTextContent,
|
||||
sourceImageUrl: params.sourceImageUrl,
|
||||
targetNodeIds: params.targetNodeIds,
|
||||
});
|
||||
|
||||
if (regions.length === 0) {
|
||||
throw new DirectAgentLocalEditServiceError(
|
||||
"请先在图片上框选局部编辑区域，或在批注里写明要修改的位置。"
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
sourceNodeId: resolveDirectAgentLocalEditSourceNodeId(params),
|
||||
regions,
|
||||
};
|
||||
}
|
||||
@ -111,6 +111,7 @@ export function routeAgentSkill(
|
||||
"extendImage",
|
||||
"resizeImage",
|
||||
"angleEditImage",
|
||||
"localEditImage",
|
||||
"boxCutout",
|
||||
"useSelectedCanvasImage",
|
||||
"useLatestResultImage",
|
||||
|
||||
@ -12,6 +12,7 @@ import {
|
||||
} from "@/lib/ai/model-compat";
|
||||
import sharp from "sharp";
|
||||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||
import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options";
|
||||
// 使用动态导入避免循环依赖问题
|
||||
// import { getLanguageModelById } from "./dynamic-provider";
|
||||
// import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
@ -35,6 +36,7 @@ function inferDefaultImageEditSubIntent(text: string): AnalysisResult["subIntent
|
||||
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background";
|
||||
if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background";
|
||||
if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant";
|
||||
if (hasDirectAgentLocalEditIntent(text)) return "local_edit";
|
||||
if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend";
|
||||
if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer";
|
||||
if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust";
|
||||
@ -139,7 +141,7 @@ export function isExplicitImageGenerationRequest(text: string): boolean {
|
||||
|
||||
function isLikelyImageEditRequest(text: string): boolean {
|
||||
const normalizedText = text.toLowerCase();
|
||||
return /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test(
|
||||
return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test(
|
||||
normalizedText,
|
||||
);
|
||||
}
|
||||
@ -390,6 +392,7 @@ const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI多模态Agent。用户
|
||||
- enhance: 变清晰、高清、锐化、降噪
|
||||
- extend: 扩展、拓展、延伸画布
|
||||
- angle_variant: 多视角、换角度、侧面、背面、俯视、仰视
|
||||
- local_edit: 按画布框选/批注/选区做局部编辑或局部重绘
|
||||
- inpaint: 去除某物、消除、擦掉、P掉、局部修改
|
||||
- style_transfer: 风格化、卡通化、动漫风、改风格
|
||||
- color_adjust: 调色、调整颜色、亮度、对比度
|
||||
@ -770,7 +773,7 @@ async function analyzeWithHardcodedQwen3VlFallback(params: {
|
||||
const prompt = [
|
||||
"你是图片需求分析助手。请结合图片与用户意图，输出严格 JSON，不要输出其他内容。",
|
||||
"intent 只能是 image_edit / vision / image_gen / video_gen / chat。",
|
||||
"subIntent 可选：remove_background / remove_watermark / replace_background / enhance / extend / angle_variant / inpaint / style_transfer / color_adjust / redo / resize。",
|
||||
"subIntent 可选：remove_background / remove_watermark / replace_background / enhance / extend / angle_variant / local_edit / inpaint / style_transfer / color_adjust / redo / resize。",
|
||||
"如果用户提到视频、首帧、尾帧、分镜，intent=video_gen。",
|
||||
"有图片且用户在修改图片时，优先 intent=image_edit。",
|
||||
"JSON 字段：intent, subIntent, confidence, conversationalReply, needsClarification, clarifyingQuestions。",
|
||||
|
||||
@ -2,6 +2,7 @@ import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
|
||||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||||
import { refinePlanWithImageEditSubagent } from "@/lib/ai/image-edit-subagent";
|
||||
import { refinePlanWithImageGenerationSubagent } from "@/lib/ai/image-generation-subagent";
|
||||
import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options";
|
||||
import { refinePlanWithStoryboardSubagent } from "@/lib/video-agent/storyboard-subagent";
|
||||
import type {
|
||||
AgentExecutionPlan,
|
||||
@ -231,6 +232,17 @@ function normalizeEditCapability(
|
||||
subIntent: "angle_variant",
|
||||
};
|
||||
}
|
||||
if (
|
||||
subIntent === "local_edit" ||
|
||||
subIntent === "inpaint_region" ||
|
||||
hasDirectAgentLocalEditIntent(text)
|
||||
) {
|
||||
return {
|
||||
capability: "local_edit",
|
||||
toolName: "localEditImage",
|
||||
subIntent: "local_edit",
|
||||
};
|
||||
}
|
||||
if (subIntent === "remove_watermark" || /去水印|水印|去logo|去\s*logo|移除logo|去除logo/.test(normalizedText)) {
|
||||
return {
|
||||
capability: "remove_watermark",
|
||||
|
||||
@ -24,6 +24,7 @@ export type AgentProjectToolName =
|
||||
| "extendImage"
|
||||
| "resizeImage"
|
||||
| "angleEditImage"
|
||||
| "localEditImage"
|
||||
| "boxCutout"
|
||||
| "publishToCanvas"
|
||||
| "useSelectedCanvasImage"
|
||||
@ -208,6 +209,15 @@ const INTERNAL_AGENT_TOOLS: AgentProjectToolDefinition[] = [
|
||||
requiresImage: true,
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "localEditImage",
|
||||
title: "局部编辑",
|
||||
description:
|
||||
"按画布框选区域或批注对目标图做局部重绘/局部修改，未选区域保持不变，结果作为新版本保留原图。",
|
||||
category: "image",
|
||||
requiresImage: true,
|
||||
producesMedia: true,
|
||||
},
|
||||
{
|
||||
name: "boxCutout",
|
||||
title: "框选抠图",
|
||||
|
||||
@ -131,6 +131,11 @@ import {
|
||||
shouldRunDirectAgentBatchImageEdit,
|
||||
} from "@/lib/ai/runtime/direct-agent-execution";
|
||||
import { inferAngleEditOptionsFromText } from "@/lib/ai/runtime/angle-edit-options";
|
||||
import {
|
||||
hasDirectAgentLocalEditIntent,
|
||||
normalizeDirectAgentLocalEditRect,
|
||||
resolveDirectAgentLocalEditRegions,
|
||||
} from "@/lib/ai/runtime/local-edit-options";
|
||||
import {
|
||||
applyCanvasOperationPlanToCanvasData,
|
||||
CanvasAgentMutationError,
|
||||
@ -161,6 +166,12 @@ import {
|
||||
buildDetailReviewAppendMarkdown,
|
||||
buildDetailReviewPlans,
|
||||
} from "@/components/canvas/chat-panel/canvas-detail-markdown";
|
||||
import {
|
||||
buildOpenClawDisplayMessage,
|
||||
buildOpenClawMessageWithSpreadsheetMarkdown,
|
||||
isOpenClawSpreadsheetFileName,
|
||||
rowsToOpenClawMarkdownTable,
|
||||
} from "@/lib/openclaw-chat/spreadsheet-markdown";
|
||||
import {
|
||||
DETAIL_BUNDLE_SLOT_INDEXES,
|
||||
DETAIL_GENERATION_MODE_OPTIONS,
|
||||
@ -200,6 +211,44 @@ import type { CanvasAgentContext } from "@/types/canvas/agent-protocol";
|
||||
import type { ChatMessage } from "@/lib/types";
|
||||
|
||||
test.describe("canvas chat pure helpers", () => {
|
||||
test("formats uploaded spreadsheet rows as safe markdown tables for OpenClaw", () => {
|
||||
const markdown = rowsToOpenClawMarkdownTable([
|
||||
["名称", "备注"],
|
||||
["A|B", "第一行\n第二行"],
|
||||
["空值列", ""],
|
||||
]);
|
||||
|
||||
expect(markdown).toContain("| 名称 | 备注 |");
|
||||
expect(markdown).toContain("A\\|B");
|
||||
expect(markdown).toContain("第一行<br>第二行");
|
||||
expect(isOpenClawSpreadsheetFileName("sample.xlsx")).toBe(true);
|
||||
expect(isOpenClawSpreadsheetFileName("sample.xls")).toBe(false);
|
||||
});
|
||||
|
||||
test("keeps OpenClaw spreadsheet display text compact while upstream message includes markdown", () => {
|
||||
const files = [
|
||||
{
|
||||
fileName: "sales.csv",
|
||||
rowCount: 12,
|
||||
sheetCount: 1,
|
||||
truncated: false,
|
||||
},
|
||||
];
|
||||
|
||||
const display = buildOpenClawDisplayMessage({ message: "", files });
|
||||
expect(display).toContain("请分析上传的表格内容");
|
||||
expect(display).toContain("sales.csv");
|
||||
expect(display).not.toContain("| SKU |");
|
||||
|
||||
const upstream = buildOpenClawMessageWithSpreadsheetMarkdown({
|
||||
message: "分析销量",
|
||||
spreadsheetMarkdown: "| SKU | 销量 |\n| --- | --- |\n| A01 | 20 |",
|
||||
});
|
||||
expect(upstream).toContain("分析销量");
|
||||
expect(upstream).toContain("以下是用户上传表格转换后的 Markdown 内容");
|
||||
expect(upstream).toContain("| SKU | 销量 |");
|
||||
});
|
||||
|
||||
test("infers explicit agent image counts from user text", () => {
|
||||
expect(inferAgentImageCountFromText("再出三版")).toBe(3);
|
||||
expect(inferAgentImageCountFromText("基于这张图再来 3 张")).toBe(3);
|
||||
@ -1150,6 +1199,110 @@ test.describe("canvas chat pure helpers", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("routes local edit requests through the dedicated direct image tool", () => {
|
||||
const result = resolveDirectAgentToolExecution({
|
||||
mode: "edit_image",
|
||||
routeIntent: "image_edit",
|
||||
routeSubIntent: "local_edit",
|
||||
targetSource: "explicit_selection",
|
||||
preferredCapability: "local_edit",
|
||||
shouldUseStructuredReply: true,
|
||||
shouldOptimizePrompt: true,
|
||||
shouldApplyImagePipeline: true,
|
||||
shouldApplyVideoPipeline: false,
|
||||
shouldStayConversational: false,
|
||||
shouldPublishToCanvas: false,
|
||||
toolCalls: [
|
||||
{
|
||||
name: "resolveTargetImage",
|
||||
args: { source: "selected_canvas" },
|
||||
},
|
||||
{
|
||||
name: "editTargetImage",
|
||||
reason: "planner only provided the generic edit tool",
|
||||
},
|
||||
],
|
||||
availableTools: ["resolveTargetImage", "editTargetImage", "localEditImage"],
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
toolName: "localEditImage",
|
||||
capability: "local_edit",
|
||||
sourcePreferences: ["selected_canvas"],
|
||||
});
|
||||
expect(getDirectAgentToolLabel("local_edit")).toBe("Agent工具：局部编辑");
|
||||
expect(getDirectAgentToolSuccessMessage("local_edit")).toBe(
|
||||
"已完成局部编辑，结果如下："
|
||||
);
|
||||
expect(isDirectAgentBatchImageEditCapability("local_edit")).toBe(false);
|
||||
});
|
||||
|
||||
test("resolves direct local edit regions from selected canvas annotations", () => {
|
||||
const context: CanvasAgentContext = {
|
||||
version: 1,
|
||||
source: "canvas",
|
||||
projectId: "project-1",
|
||||
selectedNodeIds: ["image-1"],
|
||||
selectedNodes: [
|
||||
{
|
||||
nodeId: "image-1",
|
||||
type: "image",
|
||||
url: "https://cdn.example.com/source.png",
|
||||
x: 0,
|
||||
y: 0,
|
||||
isSelected: true,
|
||||
},
|
||||
],
|
||||
candidateNodes: [],
|
||||
slots: [],
|
||||
annotations: [
|
||||
{
|
||||
id: "annotation-1",
|
||||
sourceNodeId: "image-1",
|
||||
type: "crop-box",
|
||||
text: "把杯子换成蓝色陶瓷杯",
|
||||
bounds: { x: 0.2, y: 0.3, width: 0.25, height: 0.2 },
|
||||
color: "#1677ff",
|
||||
},
|
||||
{
|
||||
id: "annotation-other",
|
||||
sourceNodeId: "image-2",
|
||||
type: "crop-box",
|
||||
text: "不应带入",
|
||||
bounds: { x: 0.1, y: 0.1, width: 0.1, height: 0.1 },
|
||||
},
|
||||
],
|
||||
focusPolicy: "selected",
|
||||
};
|
||||
|
||||
const result = resolveDirectAgentLocalEditRegions({
|
||||
canvasAgentContext: context,
|
||||
requestTextContent: "按批注修改",
|
||||
sourceImageUrl: "https://cdn.example.com/source.png",
|
||||
targetNodeIds: ["image-1"],
|
||||
});
|
||||
|
||||
expect(result.sourceNodeId).toBe("image-1");
|
||||
expect(result.regions).toEqual([
|
||||
{
|
||||
id: "annotation-1",
|
||||
color: "#1677ff",
|
||||
prompt: "把杯子换成蓝色陶瓷杯",
|
||||
rect: { x: 0.2, y: 0.3, width: 0.25, height: 0.2 },
|
||||
},
|
||||
]);
|
||||
expect(hasDirectAgentLocalEditIntent("按照批注修改这张图")).toBe(true);
|
||||
expect(hasDirectAgentLocalEditIntent("给这张图添加批注")).toBe(false);
|
||||
expect(
|
||||
normalizeDirectAgentLocalEditRect({
|
||||
x: 0.8,
|
||||
y: 0.8,
|
||||
width: 0.4,
|
||||
height: 0.3,
|
||||
})
|
||||
).toEqual({ x: 0.8, y: 0.8, width: 0.2, height: 0.2 });
|
||||
});
|
||||
|
||||
test("infers deterministic angle edit options from canvas agent text", () => {
|
||||
expect(inferAngleEditOptionsFromText("生成右侧面近景视角")).toMatchObject({
|
||||
horizontalAngle: 90,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user