diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index d78088a2f..628b04645 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -8182,6 +8182,10 @@ export async function POST(request: Request) { defaultImageQualityParams, smartAutoExecution, smartAutoPlan, + canvasAgentContext: requestCanvasAgentContext, + targetNodeIds: (requestTargetResolution?.targetRefs || []) + .map((ref) => ref.nodeId) + .filter((nodeId): nodeId is string => Boolean(nodeId)), saveGeneratedImages, }; const directImageEditResults = shouldBatchDirectImageEdit diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index 0284bf9b9..bf3b9fd64 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -23,6 +23,7 @@ import { getAiModelAgentCodingModelId, isAiModelMultimodal, } from "@/lib/db/ai-model-flags"; +import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options"; interface HistoryMessage { role: "user" | "assistant"; @@ -82,6 +83,7 @@ export type AgentPreferredCapability = | "extend" | "resize" | "angle_variant" + | "local_edit" | "box_cutout" | "generate_image" | "generate_video" @@ -253,6 +255,7 @@ function inferPreferredCapability( if (firstTool === "extendImage") return "extend"; if (firstTool === "resizeImage") return "resize"; if (firstTool === "angleEditImage") return "angle_variant"; + if (firstTool === "localEditImage") return "local_edit"; if (firstTool === "boxCutout") return "box_cutout"; if (firstTool === "publishToCanvas" || firstTool === "publishResultToCanvas") return "publish_canvas"; if (firstTool === "generateVideo") return "generate_video"; @@ -278,6 +281,9 @@ function inferPreferredCapability( case "angle_variant": case "generate_angle_variant": return "angle_variant"; + case "local_edit": + case "inpaint_region": + return "local_edit"; default: return "generic_edit"; } @@ -501,14 +507,33 @@ function buildFallbackPlan(params: { const baseAction = params.analysis.needsClarification ? "clarify" : mapIntentToAction(params.analysis.intent); - const toolCalls = sanitizePlannedToolCalls([], params.availableTools, baseAction, { + let toolCalls = sanitizePlannedToolCalls([], params.availableTools, baseAction, { targetSource: params.targetResolutionMode, }); + if ( + baseAction === "edit_image" && + hasDirectAgentLocalEditIntent(params.text) && + params.availableTools.some((tool) => tool.name === "localEditImage") && + !toolCalls.some((tool) => tool.name === "localEditImage") + ) { + toolCalls = [ + ...toolCalls.filter((tool) => tool.name !== "editTargetImage" && tool.name !== "editImage"), + { + name: "localEditImage", + reason: "用户要求按局部区域或批注执行编辑。", + }, + ]; + } return { goal: createPlanGoalSummary(params.text, params.hasImage), nextAction: baseAction, intent: mapActionToIntent(baseAction, params.analysis.intent), - subIntent: baseAction === "edit_image" ? params.analysis.subIntent || "enhance" : undefined, + subIntent: + baseAction === "edit_image" + ? hasDirectAgentLocalEditIntent(params.text) + ? "local_edit" + : params.analysis.subIntent || "enhance" + : undefined, targetSource: params.targetResolutionMode, preferredCapability: inferPreferredCapability(params.analysis, toolCalls), responseMode: defaultResponseMode(baseAction), @@ -1409,6 +1434,7 @@ function sanitizeAgentPlan( "extend", "resize", "angle_variant", + "local_edit", "box_cutout", "generate_image", "generate_video", @@ -1590,7 +1616,7 @@ async function buildExecutionPlanWithAgent( 5. 优先保持一次只选择一个主工具；toolCalls 只从提供的工具列表里选。 6. shouldUseStructuredReply 只在 edit_image / generate_image / generate_video 时设为 true。 7. targetSource 要忠实反映当前目标图来源：explicit_selection/current_upload/text_reference/working_memory/latest_result/clarification/none。 -8. preferredCapability 要尽量具体，例如 remove_background / remove_watermark / enhance / extend / resize / angle_variant / box_cutout / publish_canvas。 +8. preferredCapability 要尽量具体，例如 remove_background / remove_watermark / enhance / extend / resize / angle_variant / local_edit / box_cutout / publish_canvas。 9. 必须优先使用新的强语义工具名：resolveTargetImage / analyzeTargetImage / editTargetImage / generateNewImage / publishResultToCanvas。 10. 必须遵守当前专家路由约束；如果约束要求纯对话，则绝不能输出 generate_image / edit_image / generate_video。 @@ -1601,7 +1627,7 @@ async function buildExecutionPlanWithAgent( "intent": "chat|vision|image_edit|image_gen|video_gen", "subIntent": "可选，仅 image_edit 时填写", "targetSource": "explicit_selection|current_upload|text_reference|working_memory|latest_result|clarification|none", - "preferredCapability": "respond|analyze|generic_edit|remove_background|remove_watermark|enhance|extend|resize|angle_variant|box_cutout|generate_image|generate_video|publish_canvas", + "preferredCapability": "respond|analyze|generic_edit|remove_background|remove_watermark|enhance|extend|resize|angle_variant|local_edit|box_cutout|generate_image|generate_video|publish_canvas", "responseMode": "direct_answer|tool_then_answer|tool_then_execute|clarify", "shouldUseHistoryImage": true, "shouldUseStructuredReply": false, diff --git a/lib/ai/image-edit-subagent.ts b/lib/ai/image-edit-subagent.ts index 56de2bcae..fe236e50b 100644 --- a/lib/ai/image-edit-subagent.ts +++ b/lib/ai/image-edit-subagent.ts @@ -71,6 +71,7 @@ export function buildImageEditSpecialistRoute(params: { "extendImage", "resizeImage", "angleEditImage", + "localEditImage", "boxCutout", "useSelectedCanvasImage", "useLatestResultImage", diff --git a/lib/ai/runtime/direct-agent-execution.ts b/lib/ai/runtime/direct-agent-execution.ts index c15e8c9ac..bd507cb55 100644 --- a/lib/ai/runtime/direct-agent-execution.ts +++ b/lib/ai/runtime/direct-agent-execution.ts @@ -17,6 +17,7 @@ export type DirectAgentToolCapability = | "resize" | "extend" | "angle_variant" + | "local_edit" | "box_cutout" | "generate_image" | "generate_video"; @@ -30,6 +31,7 @@ export type DirectAgentToolName = | "resizeImage" | "extendImage" | "angleEditImage" + | "localEditImage" | "boxCutout" | "generateImage" | "generateVideo"; @@ -102,6 +104,8 @@ function mapDirectToolNameToCapability( return "extend"; case "angleEditImage": return "angle_variant"; + case "localEditImage": + return "local_edit"; case "boxCutout": return "box_cutout"; case "generateImage": @@ -210,6 +214,13 @@ export function resolveDirectAgentToolExecution( ) { return "angleEditImage"; } + if ( + preferredCapability === "local_edit" || + routeSubIntent === "local_edit" || + routeSubIntent === "inpaint_region" + ) { + return "localEditImage"; + } if (preferredCapability === "box_cutout" || routeSubIntent === "box_cutout") { return "boxCutout"; } @@ -265,15 +276,17 @@ export function resolveDirectAgentToolExecution( ? "extendImage" : hasToolCall("angleEditImage") ? "angleEditImage" - : hasToolCall("boxCutout") - ? "boxCutout" - : hasToolCall("editTargetImage") || hasToolCall("editImage") - ? preferredSpecificImageEditTool || "editImage" - : hasToolCall("generateNewImage") || hasToolCall("generateImage") - ? "generateImage" - : hasToolCall("generateVideo") - ? "generateVideo" - : null) || + : hasToolCall("localEditImage") + ? "localEditImage" + : hasToolCall("boxCutout") + ? "boxCutout" + : hasToolCall("editTargetImage") || hasToolCall("editImage") + ? preferredSpecificImageEditTool || "editImage" + : hasToolCall("generateNewImage") || hasToolCall("generateImage") + ? "generateImage" + : hasToolCall("generateVideo") + ? "generateVideo" + : null) || (() => { if ( execution?.preferredCapability === "analyze" || @@ -319,6 +332,13 @@ export function resolveDirectAgentToolExecution( ) { return "angleEditImage"; } + if ( + execution?.preferredCapability === "local_edit" || + execution?.routeSubIntent === "local_edit" || + execution?.routeSubIntent === "inpaint_region" + ) { + return "localEditImage"; + } if ( execution?.preferredCapability === "box_cutout" || execution?.routeSubIntent === "box_cutout" @@ -402,6 +422,8 @@ export function isDirectAgentBatchImageEditCapability( case "extend": case "angle_variant": return true; + case "local_edit": + return false; default: return false; } @@ -432,6 +454,7 @@ export function isDirectAgentImageEditToolName( | "resizeImage" | "extendImage" | "angleEditImage" + | "localEditImage" | "boxCutout" { switch (toolName) { case "editImage": @@ -441,6 +464,7 @@ export function isDirectAgentImageEditToolName( case "resizeImage": case "extendImage": case "angleEditImage": + case "localEditImage": case "boxCutout": return true; default: @@ -466,6 +490,8 @@ export function getDirectAgentToolLabel(capability: DirectAgentToolCapability): return "Agent工具：图像拓展"; case "angle_variant": return "Agent工具：多视角"; + case "local_edit": + return "Agent工具：局部编辑"; case "box_cutout": return "Agent工具：框选抠图"; case "generate_image": @@ -513,6 +539,8 @@ export function getDirectAgentToolSuccessMessage( return "已完成图像拓展，结果如下："; case "angle_variant": return "已完成多视角生成，结果如下："; + case "local_edit": + return "已完成局部编辑，结果如下："; case "box_cutout": return "已完成框选抠图，结果如下："; case "generate_image": @@ -544,6 +572,8 @@ export function getDirectAgentToolTargetMissingMessage( return "请先上传或选中一张图片，我才能继续帮你做图像拓展。"; case "angle_variant": return "请先上传或选中一张图片，我才能继续帮你生成多视角变体。"; + case "local_edit": + return "请先选中一张图片并框选局部编辑区域，我才能继续帮你做局部编辑。"; case "box_cutout": return "请先选中一张图片并提供框选区域，我才能继续帮你做框选抠图。"; case "generate_image": @@ -584,6 +614,8 @@ export function getDirectAgentToolErrorMessage( return `图像拓展暂时失败：${detail}`; case "angle_variant": return `多视角生成暂时失败：${detail}`; + case "local_edit": + return `局部编辑暂时失败：${detail}`; case "box_cutout": return `框选抠图暂时失败：${detail}`; case "generate_image": diff --git a/lib/ai/runtime/direct-agent-image-edit-executor.ts b/lib/ai/runtime/direct-agent-image-edit-executor.ts index 5141cfab0..3ac2c787e 100644 --- a/lib/ai/runtime/direct-agent-image-edit-executor.ts +++ b/lib/ai/runtime/direct-agent-image-edit-executor.ts @@ -35,6 +35,12 @@ import { inferAngleEditOptionsFromText, type DirectAgentAngleEditOptions, } from "@/lib/ai/runtime/angle-edit-options"; +import { + DirectAgentLocalEditServiceError, + resolveDirectAgentLocalEditRegions, + type DirectAgentLocalEditRegion, +} from "@/lib/ai/runtime/local-edit-options"; +import type { CanvasAgentContext } from "@/types/canvas/agent-protocol"; interface DirectAgentImageParamsSnapshot { negativePrompt?: string; @@ -101,6 +107,8 @@ export interface ExecuteDirectAgentImageEditToolParams { defaultImageQualityParams?: DirectAgentImageParamsSnapshot | null; smartAutoExecution?: AgentPreparedExecution; smartAutoPlan?: AgentExecutionPlan; + canvasAgentContext?: CanvasAgentContext | null; + targetNodeIds?: string[] | null; saveGeneratedImages: SaveGeneratedImages; } @@ -109,6 +117,10 @@ function toFiniteNumber(value: unknown): number | undefined { return Number.isFinite(normalized) ? normalized : undefined; } +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + async function executeDirectAgentEnhanceJob(params: { requestUrl: string; requestHeaders: Headers; @@ -221,12 +233,137 @@ async function executeDirectAgentAngleEditJob(params: { }; } +async function safeParseJsonResponse(response: Response) { + try { + return await response.json(); + } catch { + return null; + } +} + +async function pollDirectAgentCanvasImageTaskResult(params: { + requestUrl: string; + requestHeaders: Headers; + taskId: string; + timeoutMs?: number; +}) { + const cookieHeader = params.requestHeaders.get("cookie") || ""; + const endpoint = new URL("/api/canvas/generate/image", params.requestUrl); + endpoint.searchParams.set("taskId", params.taskId); + const timeoutMs = Math.max(10_000, params.timeoutMs ?? 20 * 60 * 1000); + const startedAt = Date.now(); + let pollAfterMs = 1500; + + while (Date.now() - startedAt < timeoutMs) { + await sleep(pollAfterMs); + + const response = await fetch(endpoint.toString(), { + method: "GET", + cache: "no-store", + headers: cookieHeader ? { cookie: cookieHeader } : undefined, + }); + const payload = await safeParseJsonResponse(response); + if (!response.ok || payload?.success === false) { + const detail = String(payload?.error || payload?.detail || "").trim(); + throw new DirectAgentLocalEditServiceError( + detail || `查询局部编辑任务失败(${response.status})`, + response.status || 500, + ); + } + + const task = payload?.task || {}; + const status = String(task.status || "").toLowerCase(); + if (status === "succeeded") { + const result = task.result || {}; + const images = Array.isArray(result.images) ? result.images : []; + const firstImage = images[0]; + const originalUrl = String(firstImage?.originalUrl || firstImage?.original_url || "").trim(); + const thumbnailUrl = String( + firstImage?.thumbnailUrl || + firstImage?.thumbnail_url || + firstImage?.thumbnail || + originalUrl + ).trim(); + if (!originalUrl) { + throw new DirectAgentLocalEditServiceError("局部编辑任务完成但未返回图片", 500); + } + return { + originalUrl, + thumbnailUrl: thumbnailUrl || originalUrl, + imageSize: "2K", + modelRef: String(result.modelRef || result.model || "").trim(), + }; + } + + if (status === "failed") { + throw new DirectAgentLocalEditServiceError( + String(task.error || "局部编辑任务失败"), + 500, + ); + } + + const estimatedWaitMs = Number(task.queueStatus?.estimatedWaitMs || 0); + pollAfterMs = estimatedWaitMs > 0 ? Math.min(4000, Math.max(1200, estimatedWaitMs)) : 1500; + } + + throw new DirectAgentLocalEditServiceError("局部编辑任务处理超时，请稍后再试", 504); +} + +async function executeDirectAgentLocalEditJob(params: { + requestUrl: string; + requestHeaders: Headers; + imageUrl: string; + sourceNodeId?: string; + regions: DirectAgentLocalEditRegion[]; + model?: string; + imageSize?: string; + aspectRatio?: string; +}) { + const endpoint = new URL("/api/canvas/local-edit", params.requestUrl).toString(); + const cookieHeader = params.requestHeaders.get("cookie") || ""; + const response = await fetch(endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(cookieHeader ? { cookie: cookieHeader } : {}), + }, + body: JSON.stringify({ + sourceImageUrl: params.imageUrl, + sourceNodeId: params.sourceNodeId, + regions: params.regions, + operation: "local-edit", + model: params.model, + imageSize: params.imageSize || "2K", + aspectRatio: params.aspectRatio || "auto", + }), + }); + const payload = await safeParseJsonResponse(response); + if (!response.ok || payload?.success === false) { + throw new DirectAgentLocalEditServiceError( + String(payload?.error || payload?.detail || "局部编辑提交失败"), + response.status || 500, + ); + } + + const taskId = String(payload?.taskId || "").trim(); + if (!taskId) { + throw new DirectAgentLocalEditServiceError("局部编辑提交成功但未返回任务ID", 500); + } + + return pollDirectAgentCanvasImageTaskResult({ + requestUrl: params.requestUrl, + requestHeaders: params.requestHeaders, + taskId, + }); +} + export function isDirectAgentImageEditServiceError(error: unknown) { return ( error instanceof RmbgServiceError || error instanceof ResizeServiceError || error instanceof ExtendServiceError || - error instanceof BoxCutoutServiceError + error instanceof BoxCutoutServiceError || + error instanceof DirectAgentLocalEditServiceError ); } @@ -552,6 +689,75 @@ export async function executeDirectAgentImageEditTool( }); } + if (params.capability === "local_edit") { + const plannedArgs = findAgentToolCallArgs( + "localEditImage", + params.smartAutoExecution, + params.smartAutoPlan, + ); + const regionResolution = resolveDirectAgentLocalEditRegions({ + canvasAgentContext: params.canvasAgentContext, + plannedArgs, + requestTextContent: params.requestTextContent, + sourceImageUrl: params.sourceImageUrl, + targetNodeIds: params.targetNodeIds, + }); + const aspectRatio = + params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio || "auto"; + const imageSize = + params.imageParams?.imageSize || params.defaultImageQualityParams?.imageSize || "2K"; + const result = await executeDirectAgentLocalEditJob({ + requestUrl: params.requestUrl, + requestHeaders: params.requestHeaders, + imageUrl: params.sourceImageUrl, + sourceNodeId: regionResolution.sourceNodeId, + regions: regionResolution.regions, + model: params.finalModelId, + imageSize, + aspectRatio, + }); + const editResultMarkdown = buildAgentToolImagesMarkdown([ + { + originalUrl: result.originalUrl, + thumbnailUrl: result.thumbnailUrl, + imageSize, + }, + ]); + + await params.saveGeneratedImages( + editResultMarkdown, + params.userId, + params.chatId, + params.messageId, + params.requestTextContent || "局部编辑处理", + result.modelRef || params.finalModelId, + { + sourceAssetIds: params.targetAssetIds || undefined, + sourceImageUrls: [params.sourceImageUrl], + relationType: "edited_from", + sourceType: "canvas-local-edit", + projectId: params.projectId || null, + userPrompt: params.requestTextContent, + optimizedPrompt: + regionResolution.regions + .map((region, index) => `区域 ${index + 1}：${region.prompt}`) + .join("\n") || params.requestTextContent, + requestedImageCount: 1, + aspectRatio, + imageSize, + }, + ); + + return finalizeDirectAgentImageEditResult({ + userId: params.userId, + capability: params.capability, + originalUrl: result.originalUrl, + thumbnailUrl: result.thumbnailUrl, + imageSize, + mediaAssetId: null, + }); + } + if (params.capability === "box_cutout") { const plannedArgs = findAgentToolCallArgs( "boxCutout", diff --git a/lib/ai/runtime/intent-arbiter.ts b/lib/ai/runtime/intent-arbiter.ts index d7200b719..cfd463d3b 100644 --- a/lib/ai/runtime/intent-arbiter.ts +++ b/lib/ai/runtime/intent-arbiter.ts @@ -4,6 +4,7 @@ import type { ConversationActionResolution } from "@/lib/ai/conversation-action- import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; +import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options"; export type IntentArbiterMode = | "respond" @@ -23,6 +24,7 @@ export type IntentArbiterPreferredCapability = | "extend" | "resize" | "angle_variant" + | "local_edit" | "box_cutout" | "generate_image" | "generate_video" @@ -115,7 +117,7 @@ function isExplicitImageGenerationRequest(text: string): boolean { function isLikelyImageEditRequest(text: string): boolean { const normalizedText = String(text || "").toLowerCase(); - return /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( + return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( normalizedText, ); } @@ -126,6 +128,7 @@ function inferDefaultImageEditSubIntent(text: string): string { if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant"; + if (hasDirectAgentLocalEditIntent(text)) return "local_edit"; if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend"; if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer"; if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust"; @@ -174,6 +177,9 @@ function mapPreferredCapability( case "angle_variant": case "generate_angle_variant": return "angle_variant"; + case "local_edit": + case "inpaint_region": + return "local_edit"; case "box_cutout": return "box_cutout"; case "enhance": diff --git a/lib/ai/runtime/local-edit-options.ts b/lib/ai/runtime/local-edit-options.ts new file mode 100644 index 000000000..283bc142f --- /dev/null +++ b/lib/ai/runtime/local-edit-options.ts @@ -0,0 +1,398 @@ +import type { + CanvasAgentAnnotationRef, + CanvasAgentContext, + CanvasAgentNodeRef, +} from "@/types/canvas/agent-protocol"; + +export interface DirectAgentLocalEditRect { + x: number; + y: number; + width: number; + height: number; +} + +export interface DirectAgentLocalEditRegion { + id: string; + color?: string; + prompt: string; + rect: DirectAgentLocalEditRect; + targetRect?: DirectAgentLocalEditRect; + referenceImages?: Array<{ + id?: string; + url: string; + name?: string; + source?: "upload" | "canvas"; + }>; +} + +export interface DirectAgentLocalEditRegionResolution { + sourceNodeId?: string; + regions: DirectAgentLocalEditRegion[]; +} + +export class DirectAgentLocalEditServiceError extends Error { + status: number; + + constructor(message: string, status = 400) { + super(message); + this.name = "DirectAgentLocalEditServiceError"; + this.status = status; + } +} + +const MAX_DIRECT_AGENT_LOCAL_EDIT_REGIONS = 8; +const DEFAULT_REGION_COLORS = [ + "#ff3b30", + "#1677ff", + "#00c853", + "#ff9500", + "#af52de", + "#00bcd4", + "#ff2d55", + "#8d6e63", +]; + +function trimText(value: unknown, maxLength: number) { + return typeof value === "string" ? value.trim().slice(0, maxLength) : ""; +} + +function normalizeCompactText(value: unknown) { + return String(value || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +function toFiniteNumber(value: unknown): number | null { + const numberValue = Number(value); + return Number.isFinite(numberValue) ? numberValue : null; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function roundCoordinate(value: number) { + return Math.round(value * 10_000) / 10_000; +} + +function normalizeHttpUrl(value: unknown) { + const url = String(value || "").trim(); + return /^https?:\/\//i.test(url) ? url : ""; +} + +function normalizeColor(value: unknown, fallback: string) { + const raw = String(value || "").trim(); + const expanded = raw.replace(/^#([0-9a-f])([0-9a-f])([0-9a-f])$/i, "#$1$1$2$2$3$3"); + return /^#[0-9a-f]{6}$/i.test(expanded) ? expanded.toLowerCase() : fallback; +} + +export function hasDirectAgentLocalEditIntent(text: string) { + const normalizedText = normalizeCompactText(text); + if (!normalizedText) return false; + + const annotationKeyword = "(?:批注|标注|箭头|框选|圈选|annotation|note)"; + if ( + new RegExp(`(?:添加|新增|创建|加|写|记录).{0,8}${annotationKeyword}`, "i").test( + normalizedText + ) + ) { + return false; + } + + return ( + /局部编辑|局部重绘|局部修改|局部修图|局部处理|区域编辑|区域重绘|局部换|局部去除|选区/.test( + normalizedText + ) || + /框选区域.{0,24}(?:修改|编辑|修图|改图|处理|调整|去掉|移除|换|生成|出图|执行|重绘)/.test( + normalizedText + ) || + new RegExp( + `(?:按|按照|根据|依据|依照|参考|基于).{0,18}${annotationKeyword}.{0,24}(?:修改|编辑|修图|改图|处理|调整|去掉|移除|换|生成|出图|执行|重绘)`, + "i" + ).test(normalizedText) || + new RegExp( + `${annotationKeyword}.{0,24}(?:修改|编辑|修图|改图|处理|调整|去掉|移除|换|生成|出图|执行|重绘)`, + "i" + ).test(normalizedText) + ); +} + +export function normalizeDirectAgentLocalEditRect( + value: unknown +): DirectAgentLocalEditRect | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const raw = value as Record; + const x = toFiniteNumber(raw.x); + const y = toFiniteNumber(raw.y); + const width = toFiniteNumber(raw.width); + const height = toFiniteNumber(raw.height); + if (x === null || y === null || width === null || height === null) return null; + + const left = clamp(Math.min(x, x + width), 0, 1); + const top = clamp(Math.min(y, y + height), 0, 1); + const right = clamp(Math.max(x, x + width), 0, 1); + const bottom = clamp(Math.max(y, y + height), 0, 1); + const normalizedWidth = clamp(right - left, 0, 1 - left); + const normalizedHeight = clamp(bottom - top, 0, 1 - top); + if (normalizedWidth <= 0 || normalizedHeight <= 0) return null; + + return { + x: roundCoordinate(left), + y: roundCoordinate(top), + width: roundCoordinate(normalizedWidth), + height: roundCoordinate(normalizedHeight), + }; +} + +function normalizeReferenceImages(value: unknown): DirectAgentLocalEditRegion["referenceImages"] { + if (!Array.isArray(value)) return undefined; + const references = value + .map((item, index) => { + if (!item || typeof item !== "object") return null; + const raw = item as Record; + const url = normalizeHttpUrl(raw.url); + if (!url) return null; + return { + id: trimText(raw.id, 80) || `reference-${index + 1}`, + url, + ...(trimText(raw.name, 80) ? { name: trimText(raw.name, 80) } : {}), + source: raw.source === "canvas" ? ("canvas" as const) : ("upload" as const), + }; + }) + .filter((item): item is NonNullable => Boolean(item)) + .slice(0, 3); + return references.length > 0 ? references : undefined; +} + +function isGenericRegionPrompt(value: string) { + const normalized = normalizeCompactText(value); + return ( + !normalized || + /^局部编辑区域\d*$/.test(normalized) || + /^区域\d+$/.test(normalized) || + /^localeditregion\d*$/.test(normalized) + ); +} + +function normalizePlannedRegion( + value: unknown, + index: number, + fallbackPrompt: string +): DirectAgentLocalEditRegion | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const raw = value as Record; + const rect = normalizeDirectAgentLocalEditRect(raw.rect || raw.cropRect || raw.bounds); + if (!rect) return null; + const prompt = + trimText(raw.prompt, 300) || + trimText(raw.instruction, 300) || + trimText(raw.text, 300) || + fallbackPrompt; + if (!prompt) return null; + const targetRect = normalizeDirectAgentLocalEditRect(raw.targetRect); + + return { + id: trimText(raw.id, 80) || `region-${index + 1}`, + color: normalizeColor(raw.color, DEFAULT_REGION_COLORS[index % DEFAULT_REGION_COLORS.length]), + prompt, + rect, + ...(targetRect ? { targetRect } : {}), + ...(normalizeReferenceImages(raw.referenceImages) + ? { referenceImages: normalizeReferenceImages(raw.referenceImages) } + : {}), + }; +} + +function getAllContextNodes(context?: CanvasAgentContext | null): CanvasAgentNodeRef[] { + if (!context) return []; + const nodes = [...(context.selectedNodes || []), ...(context.candidateNodes || [])]; + const seen = new Set(); + return nodes.filter((node) => { + if (!node?.nodeId || seen.has(node.nodeId)) return false; + seen.add(node.nodeId); + return true; + }); +} + +function resolveTargetNodeIds(params: { + canvasAgentContext?: CanvasAgentContext | null; + sourceImageUrl?: string; + targetNodeIds?: string[] | null; +}) { + const context = params.canvasAgentContext; + const sourceUrl = normalizeHttpUrl(params.sourceImageUrl); + const targetIds: string[] = []; + const addTargetId = (value: unknown) => { + const nodeId = trimText(value, 128); + if (nodeId && !targetIds.includes(nodeId)) { + targetIds.push(nodeId); + } + }; + + const nodes = getAllContextNodes(context); + if (sourceUrl) { + for (const node of nodes) { + if (normalizeHttpUrl(node.url) === sourceUrl) { + addTargetId(node.nodeId); + } + } + } + + for (const nodeId of params.targetNodeIds || []) addTargetId(nodeId); + for (const nodeId of context?.selectedNodeIds || []) addTargetId(nodeId); + for (const node of context?.selectedNodes || []) { + if (node.isSelected) addTargetId(node.nodeId); + } + + return targetIds; +} + +export function resolveDirectAgentLocalEditSourceNodeId(params: { + canvasAgentContext?: CanvasAgentContext | null; + plannedArgs?: Record; + sourceImageUrl?: string; + targetNodeIds?: string[] | null; +}) { + const plannedSourceNodeId = trimText(params.plannedArgs?.sourceNodeId, 128); + if (plannedSourceNodeId) return plannedSourceNodeId; + + const sourceUrl = normalizeHttpUrl(params.sourceImageUrl); + if (sourceUrl) { + const matchedNode = getAllContextNodes(params.canvasAgentContext).find( + (node) => normalizeHttpUrl(node.url) === sourceUrl + ); + if (matchedNode?.nodeId) return matchedNode.nodeId; + } + + const [firstTargetNodeId] = resolveTargetNodeIds(params); + return firstTargetNodeId; +} + +function resolvePlannedRegions(params: { + plannedArgs?: Record; + requestTextContent: string; +}) { + const plannedArgs = params.plannedArgs || {}; + const fallbackPrompt = + trimText(plannedArgs.prompt, 300) || + trimText(plannedArgs.instruction, 300) || + trimText(plannedArgs.text, 300) || + trimText(params.requestTextContent, 300); + const rawRegions = Array.isArray(plannedArgs.regions) + ? plannedArgs.regions + : Array.isArray(plannedArgs.localEditRegions) + ? plannedArgs.localEditRegions + : []; + const regions = rawRegions + .map((region, index) => normalizePlannedRegion(region, index, fallbackPrompt)) + .filter((region): region is DirectAgentLocalEditRegion => Boolean(region)); + + if (regions.length > 0) return regions.slice(0, MAX_DIRECT_AGENT_LOCAL_EDIT_REGIONS); + + const singleRegion = normalizePlannedRegion( + { + id: plannedArgs.regionId || plannedArgs.id, + color: plannedArgs.color, + prompt: fallbackPrompt, + rect: plannedArgs.rect || plannedArgs.cropRect || plannedArgs.bounds, + targetRect: plannedArgs.targetRect, + referenceImages: plannedArgs.referenceImages, + }, + 0, + fallbackPrompt + ); + return singleRegion ? [singleRegion] : []; +} + +function normalizeAnnotationRegion(params: { + annotation: CanvasAgentAnnotationRef; + index: number; + requestTextContent: string; + plannedPrompt: string; +}): DirectAgentLocalEditRegion | null { + const rect = normalizeDirectAgentLocalEditRect(params.annotation.bounds); + if (!rect) return null; + const annotationText = trimText(params.annotation.text, 300); + const prompt = + annotationText && !isGenericRegionPrompt(annotationText) + ? annotationText + : params.plannedPrompt || trimText(params.requestTextContent, 300) || annotationText; + if (!prompt) return null; + + return { + id: trimText(params.annotation.id, 80) || `annotation-region-${params.index + 1}`, + color: normalizeColor( + params.annotation.color, + DEFAULT_REGION_COLORS[params.index % DEFAULT_REGION_COLORS.length] + ), + prompt, + rect, + }; +} + +function resolveAnnotationRegions(params: { + canvasAgentContext?: CanvasAgentContext | null; + plannedArgs?: Record; + requestTextContent: string; + sourceImageUrl?: string; + targetNodeIds?: string[] | null; +}) { + const context = params.canvasAgentContext; + if (!context?.annotations?.length) return []; + const targetNodeIds = resolveTargetNodeIds(params); + const targetNodeIdSet = new Set(targetNodeIds); + const plannedPrompt = + trimText(params.plannedArgs?.prompt, 300) || + trimText(params.plannedArgs?.instruction, 300) || + trimText(params.plannedArgs?.text, 300); + + const annotations = context.annotations.filter((annotation) => { + if (!annotation?.bounds) return false; + if (targetNodeIdSet.size === 0) return true; + return targetNodeIdSet.has(annotation.sourceNodeId); + }); + + return annotations + .map((annotation, index) => + normalizeAnnotationRegion({ + annotation, + index, + plannedPrompt, + requestTextContent: params.requestTextContent, + }) + ) + .filter((region): region is DirectAgentLocalEditRegion => Boolean(region)) + .slice(0, MAX_DIRECT_AGENT_LOCAL_EDIT_REGIONS); +} + +export function resolveDirectAgentLocalEditRegions(params: { + canvasAgentContext?: CanvasAgentContext | null; + plannedArgs?: Record; + requestTextContent: string; + sourceImageUrl?: string; + targetNodeIds?: string[] | null; +}): DirectAgentLocalEditRegionResolution { + const plannedRegions = resolvePlannedRegions({ + plannedArgs: params.plannedArgs, + requestTextContent: params.requestTextContent, + }); + const regions = + plannedRegions.length > 0 + ? plannedRegions + : resolveAnnotationRegions({ + canvasAgentContext: params.canvasAgentContext, + plannedArgs: params.plannedArgs, + requestTextContent: params.requestTextContent, + sourceImageUrl: params.sourceImageUrl, + targetNodeIds: params.targetNodeIds, + }); + + if (regions.length === 0) { + throw new DirectAgentLocalEditServiceError( + "请先在图片上框选局部编辑区域，或在批注里写明要修改的位置。" + ); + } + + return { + sourceNodeId: resolveDirectAgentLocalEditSourceNodeId(params), + regions, + }; +} diff --git a/lib/ai/skill-router.ts b/lib/ai/skill-router.ts index 377e8451d..6c37f9012 100644 --- a/lib/ai/skill-router.ts +++ b/lib/ai/skill-router.ts @@ -111,6 +111,7 @@ export function routeAgentSkill( "extendImage", "resizeImage", "angleEditImage", + "localEditImage", "boxCutout", "useSelectedCanvasImage", "useLatestResultImage", diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 3de90b42e..3bde4b6ee 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -12,6 +12,7 @@ import { } from "@/lib/ai/model-compat"; import sharp from "sharp"; import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; +import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options"; // 使用动态导入避免循环依赖问题 // import { getLanguageModelById } from "./dynamic-provider"; // import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; @@ -35,6 +36,7 @@ function inferDefaultImageEditSubIntent(text: string): AnalysisResult["subIntent if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant"; + if (hasDirectAgentLocalEditIntent(text)) return "local_edit"; if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend"; if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer"; if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust"; @@ -139,7 +141,7 @@ export function isExplicitImageGenerationRequest(text: string): boolean { function isLikelyImageEditRequest(text: string): boolean { const normalizedText = text.toLowerCase(); - return /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( + return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( normalizedText, ); } @@ -390,6 +392,7 @@ const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI多模态Agent。用户 - enhance: 变清晰、高清、锐化、降噪 - extend: 扩展、拓展、延伸画布 - angle_variant: 多视角、换角度、侧面、背面、俯视、仰视 +- local_edit: 按画布框选/批注/选区做局部编辑或局部重绘 - inpaint: 去除某物、消除、擦掉、P掉、局部修改 - style_transfer: 风格化、卡通化、动漫风、改风格 - color_adjust: 调色、调整颜色、亮度、对比度 @@ -770,7 +773,7 @@ async function analyzeWithHardcodedQwen3VlFallback(params: { const prompt = [ "你是图片需求分析助手。请结合图片与用户意图，输出严格 JSON，不要输出其他内容。", "intent 只能是 image_edit / vision / image_gen / video_gen / chat。", - "subIntent 可选：remove_background / remove_watermark / replace_background / enhance / extend / angle_variant / inpaint / style_transfer / color_adjust / redo / resize。", + "subIntent 可选：remove_background / remove_watermark / replace_background / enhance / extend / angle_variant / local_edit / inpaint / style_transfer / color_adjust / redo / resize。", "如果用户提到视频、首帧、尾帧、分镜，intent=video_gen。", "有图片且用户在修改图片时，优先 intent=image_edit。", "JSON 字段：intent, subIntent, confidence, conversationalReply, needsClarification, clarifyingQuestions。", diff --git a/lib/ai/specialists/index.ts b/lib/ai/specialists/index.ts index 8cf533493..f91793f61 100644 --- a/lib/ai/specialists/index.ts +++ b/lib/ai/specialists/index.ts @@ -2,6 +2,7 @@ import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; import { refinePlanWithImageEditSubagent } from "@/lib/ai/image-edit-subagent"; import { refinePlanWithImageGenerationSubagent } from "@/lib/ai/image-generation-subagent"; +import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options"; import { refinePlanWithStoryboardSubagent } from "@/lib/video-agent/storyboard-subagent"; import type { AgentExecutionPlan, @@ -231,6 +232,17 @@ function normalizeEditCapability( subIntent: "angle_variant", }; } + if ( + subIntent === "local_edit" || + subIntent === "inpaint_region" || + hasDirectAgentLocalEditIntent(text) + ) { + return { + capability: "local_edit", + toolName: "localEditImage", + subIntent: "local_edit", + }; + } if (subIntent === "remove_watermark" || /去水印|水印|去logo|去\s*logo|移除logo|去除logo/.test(normalizedText)) { return { capability: "remove_watermark", diff --git a/lib/ai/tools-registry.ts b/lib/ai/tools-registry.ts index c6d041c9c..6f961d4d0 100644 --- a/lib/ai/tools-registry.ts +++ b/lib/ai/tools-registry.ts @@ -24,6 +24,7 @@ export type AgentProjectToolName = | "extendImage" | "resizeImage" | "angleEditImage" + | "localEditImage" | "boxCutout" | "publishToCanvas" | "useSelectedCanvasImage" @@ -208,6 +209,15 @@ const INTERNAL_AGENT_TOOLS: AgentProjectToolDefinition[] = [ requiresImage: true, producesMedia: true, }, + { + name: "localEditImage", + title: "局部编辑", + description: + "按画布框选区域或批注对目标图做局部重绘/局部修改，未选区域保持不变，结果作为新版本保留原图。", + category: "image", + requiresImage: true, + producesMedia: true, + }, { name: "boxCutout", title: "框选抠图", diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 083e58c48..291c27296 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -131,6 +131,11 @@ import { shouldRunDirectAgentBatchImageEdit, } from "@/lib/ai/runtime/direct-agent-execution"; import { inferAngleEditOptionsFromText } from "@/lib/ai/runtime/angle-edit-options"; +import { + hasDirectAgentLocalEditIntent, + normalizeDirectAgentLocalEditRect, + resolveDirectAgentLocalEditRegions, +} from "@/lib/ai/runtime/local-edit-options"; import { applyCanvasOperationPlanToCanvasData, CanvasAgentMutationError, @@ -161,6 +166,12 @@ import { buildDetailReviewAppendMarkdown, buildDetailReviewPlans, } from "@/components/canvas/chat-panel/canvas-detail-markdown"; +import { + buildOpenClawDisplayMessage, + buildOpenClawMessageWithSpreadsheetMarkdown, + isOpenClawSpreadsheetFileName, + rowsToOpenClawMarkdownTable, +} from "@/lib/openclaw-chat/spreadsheet-markdown"; import { DETAIL_BUNDLE_SLOT_INDEXES, DETAIL_GENERATION_MODE_OPTIONS, @@ -200,6 +211,44 @@ import type { CanvasAgentContext } from "@/types/canvas/agent-protocol"; import type { ChatMessage } from "@/lib/types"; test.describe("canvas chat pure helpers", () => { + test("formats uploaded spreadsheet rows as safe markdown tables for OpenClaw", () => { + const markdown = rowsToOpenClawMarkdownTable([ + ["名称", "备注"], + ["A|B", "第一行\n第二行"], + ["空值列", ""], + ]); + + expect(markdown).toContain("| 名称 | 备注 |"); + expect(markdown).toContain("A\\|B"); + expect(markdown).toContain("第一行
第二行"); + expect(isOpenClawSpreadsheetFileName("sample.xlsx")).toBe(true); + expect(isOpenClawSpreadsheetFileName("sample.xls")).toBe(false); + }); + + test("keeps OpenClaw spreadsheet display text compact while upstream message includes markdown", () => { + const files = [ + { + fileName: "sales.csv", + rowCount: 12, + sheetCount: 1, + truncated: false, + }, + ]; + + const display = buildOpenClawDisplayMessage({ message: "", files }); + expect(display).toContain("请分析上传的表格内容"); + expect(display).toContain("sales.csv"); + expect(display).not.toContain("| SKU |"); + + const upstream = buildOpenClawMessageWithSpreadsheetMarkdown({ + message: "分析销量", + spreadsheetMarkdown: "| SKU | 销量 |\n| --- | --- |\n| A01 | 20 |", + }); + expect(upstream).toContain("分析销量"); + expect(upstream).toContain("以下是用户上传表格转换后的 Markdown 内容"); + expect(upstream).toContain("| SKU | 销量 |"); + }); + test("infers explicit agent image counts from user text", () => { expect(inferAgentImageCountFromText("再出三版")).toBe(3); expect(inferAgentImageCountFromText("基于这张图再来 3 张")).toBe(3); @@ -1150,6 +1199,110 @@ test.describe("canvas chat pure helpers", () => { ); }); + test("routes local edit requests through the dedicated direct image tool", () => { + const result = resolveDirectAgentToolExecution({ + mode: "edit_image", + routeIntent: "image_edit", + routeSubIntent: "local_edit", + targetSource: "explicit_selection", + preferredCapability: "local_edit", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [ + { + name: "resolveTargetImage", + args: { source: "selected_canvas" }, + }, + { + name: "editTargetImage", + reason: "planner only provided the generic edit tool", + }, + ], + availableTools: ["resolveTargetImage", "editTargetImage", "localEditImage"], + }); + + expect(result).toMatchObject({ + toolName: "localEditImage", + capability: "local_edit", + sourcePreferences: ["selected_canvas"], + }); + expect(getDirectAgentToolLabel("local_edit")).toBe("Agent工具：局部编辑"); + expect(getDirectAgentToolSuccessMessage("local_edit")).toBe( + "已完成局部编辑，结果如下：" + ); + expect(isDirectAgentBatchImageEditCapability("local_edit")).toBe(false); + }); + + test("resolves direct local edit regions from selected canvas annotations", () => { + const context: CanvasAgentContext = { + version: 1, + source: "canvas", + projectId: "project-1", + selectedNodeIds: ["image-1"], + selectedNodes: [ + { + nodeId: "image-1", + type: "image", + url: "https://cdn.example.com/source.png", + x: 0, + y: 0, + isSelected: true, + }, + ], + candidateNodes: [], + slots: [], + annotations: [ + { + id: "annotation-1", + sourceNodeId: "image-1", + type: "crop-box", + text: "把杯子换成蓝色陶瓷杯", + bounds: { x: 0.2, y: 0.3, width: 0.25, height: 0.2 }, + color: "#1677ff", + }, + { + id: "annotation-other", + sourceNodeId: "image-2", + type: "crop-box", + text: "不应带入", + bounds: { x: 0.1, y: 0.1, width: 0.1, height: 0.1 }, + }, + ], + focusPolicy: "selected", + }; + + const result = resolveDirectAgentLocalEditRegions({ + canvasAgentContext: context, + requestTextContent: "按批注修改", + sourceImageUrl: "https://cdn.example.com/source.png", + targetNodeIds: ["image-1"], + }); + + expect(result.sourceNodeId).toBe("image-1"); + expect(result.regions).toEqual([ + { + id: "annotation-1", + color: "#1677ff", + prompt: "把杯子换成蓝色陶瓷杯", + rect: { x: 0.2, y: 0.3, width: 0.25, height: 0.2 }, + }, + ]); + expect(hasDirectAgentLocalEditIntent("按照批注修改这张图")).toBe(true); + expect(hasDirectAgentLocalEditIntent("给这张图添加批注")).toBe(false); + expect( + normalizeDirectAgentLocalEditRect({ + x: 0.8, + y: 0.8, + width: 0.4, + height: 0.3, + }) + ).toEqual({ x: 0.8, y: 0.8, width: 0.2, height: 0.2 }); + }); + test("infers deterministic angle edit options from canvas agent text", () => { expect(inferAngleEditOptionsFromText("生成右侧面近景视角")).toMatchObject({ horizontalAngle: 90,