diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index f3ac860b3..d78088a2f 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -198,6 +198,7 @@ import type { FalImageEditTaskMetadata } from "@/lib/ai/runtime/fal-image-edit-t import { saveAssistantTextOnlyMessage } from "@/lib/ai/runtime/assistant-message-persistence"; import { resolveCanvasAgentClarificationGate } from "@/lib/ai/runtime/middlewares/canvas-agent-clarification-gate"; import { + executeDirectAgentBatchImageEditTool, executeDirectAgentImageEditTool, isDirectAgentImageEditServiceError, } from "@/lib/ai/runtime/direct-agent-image-edit-executor"; @@ -208,6 +209,7 @@ import { } from "@/lib/ai/runtime/direct-agent-media-executors"; import { buildDirectAgentAnalysisAssistantText, + buildDirectAgentBatchImageEditAssistantText, buildDirectAgentImageEditAssistantText, buildDirectAgentImageGenerationAssistantText, buildDirectAgentVideoGenerationAssistantText, @@ -242,6 +244,7 @@ import { getDirectAgentToolTargetMissingMessage, isDirectAgentImageEditToolName, resolveDirectAgentToolContext, + shouldRunDirectAgentBatchImageEdit, } from "@/lib/ai/runtime/direct-agent-execution"; import { pruneTaskBoundaryContext, @@ -8155,7 +8158,12 @@ export async function POST(request: Request) { operationLabel: `agent-${directAgentToolCapability}`, resultType: "image", }); - const directImageEditResult = await executeDirectAgentImageEditTool({ + const shouldBatchDirectImageEdit = shouldRunDirectAgentBatchImageEdit({ + capability: directAgentToolCapability, + sourceImageUrls: directAgentToolTargetImageUrls, + userText: requestTextContent, + }); + const directImageEditParams = { capability: directAgentToolCapability, requestUrl: request.url, requestHeaders: request.headers, @@ -8175,18 +8183,48 @@ export async function POST(request: Request) { smartAutoExecution, smartAutoPlan, saveGeneratedImages, - }); - const resultOriginalUrl = directImageEditResult.originalUrl; - const resultThumbnailUrl = directImageEditResult.thumbnailUrl; - const resultImageSize = directImageEditResult.imageSize; - const resultAssetId = directImageEditResult.mediaAssetId; + }; + const directImageEditResults = shouldBatchDirectImageEdit + ? await executeDirectAgentBatchImageEditTool(directImageEditParams) + : [ + { + ...(await executeDirectAgentImageEditTool(directImageEditParams)), + sourceImageUrl: directAgentToolTargetImageUrl, + sourceIndex: 0, + }, + ]; + const firstImageEditResult = directImageEditResults[0]; + const resultUrls = directImageEditResults + .map((result) => result.originalUrl) + .filter((url): url is string => Boolean(url)); + const resultAssetIds = directImageEditResults + .map((result) => result.mediaAssetId) + .filter((assetId): assetId is string => Boolean(assetId)); + const resultOriginalUrl = firstImageEditResult?.originalUrl || ""; + const resultThumbnailUrl = firstImageEditResult?.thumbnailUrl || null; + const resultImageSize = firstImageEditResult?.imageSize || "2K"; + const resultAssetId = firstImageEditResult?.mediaAssetId || null; + const assetIdsByUrl = directImageEditResults.reduce< + Record + >((acc, result) => { + if (result.originalUrl) { + acc[result.originalUrl] = result.mediaAssetId || undefined; + } + return acc; + }, {}); - const assistantText = buildDirectAgentImageEditAssistantText({ - capability: directAgentToolCapability, - originalUrl: resultOriginalUrl, - thumbnailUrl: resultThumbnailUrl, - imageSize: resultImageSize, - }); + const assistantText = + directImageEditResults.length > 1 + ? buildDirectAgentBatchImageEditAssistantText({ + capability: directAgentToolCapability, + results: directImageEditResults, + }) + : buildDirectAgentImageEditAssistantText({ + capability: directAgentToolCapability, + originalUrl: resultOriginalUrl, + thumbnailUrl: resultThumbnailUrl, + imageSize: resultImageSize, + }); await saveAssistantTextOnlyMessage({ chatId: id, @@ -8201,7 +8239,9 @@ export async function POST(request: Request) { capability: directAgentToolCapability, currentSubIntent: smartAutoExecution?.routeSubIntent || directAgentToolCapability, resultAssetId, + resultAssetIds: resultAssetIds.length > 0 ? resultAssetIds : null, resultOriginalUrl, + resultOriginalUrls: resultUrls.length > 0 ? resultUrls : null, resultImageSize, sourceImageUrls: directAgentToolTargetImageUrls, messageId: assistantMessageId, @@ -8225,13 +8265,11 @@ export async function POST(request: Request) { assistantMessageId, operationLabel: `agent-${directAgentToolCapability}`, resultType: "image", - resultUrls: resultOriginalUrl ? [resultOriginalUrl] : [], + resultUrls, prompt: requestTextContent || directToolLabel, mode: "edit", sourceImageUrls: directAgentToolTargetImageUrls, - assetIdsByUrl: resultOriginalUrl - ? { [resultOriginalUrl]: resultAssetId || undefined } - : undefined, + assetIdsByUrl: resultUrls.length > 0 ? assetIdsByUrl : undefined, model: finalModelId, targetSlotId: createdTargetSlotId, }); @@ -8239,7 +8277,7 @@ export async function POST(request: Request) { writer, { messageId: assistantMessageId, - imageUrls: resultOriginalUrl ? [resultOriginalUrl] : [], + imageUrls: resultUrls, prompt: requestTextContent || directToolLabel, sourceImageUrls: directAgentToolTargetImageUrls, mediaItemIds: canvasWriteback.mediaItemIds, diff --git a/lib/ai/runtime/direct-agent-execution.ts b/lib/ai/runtime/direct-agent-execution.ts index 7dec89889..5f5ad2bb5 100644 --- a/lib/ai/runtime/direct-agent-execution.ts +++ b/lib/ai/runtime/direct-agent-execution.ts @@ -360,6 +360,47 @@ export function resolveDirectAgentToolContext(params: { }; } +const DIRECT_AGENT_BATCH_IMAGE_EDIT_INTENT_REGEX = + /(全部|都|一起|批量|这几张|这些图|所有|统一处理|统一改|都改|全部改|一起改|一起处理|每张|每一张|分别|各自|同时|一并|各(?:做|出|处理|来)?)/i; + +export function hasExplicitDirectAgentBatchImageEditIntent(userText: string) { + const normalizedText = String(userText || "").replace(/\s+/g, ""); + return Boolean( + normalizedText && + DIRECT_AGENT_BATCH_IMAGE_EDIT_INTENT_REGEX.test(normalizedText), + ); +} + +export function isDirectAgentBatchImageEditCapability( + capability: DirectAgentToolCapability, +) { + switch (capability) { + case "remove_background": + case "remove_watermark": + case "enhance": + case "resize": + case "extend": + return true; + default: + return false; + } +} + +export function shouldRunDirectAgentBatchImageEdit(params: { + capability: DirectAgentToolCapability; + sourceImageUrls: string[]; + userText: string; +}) { + const validSourceCount = params.sourceImageUrls.filter((url) => + Boolean(String(url || "").trim()), + ).length; + return ( + validSourceCount > 1 && + isDirectAgentBatchImageEditCapability(params.capability) && + hasExplicitDirectAgentBatchImageEditIntent(params.userText) + ); +} + export function isDirectAgentImageEditToolName( toolName?: DirectAgentToolName | null, ): toolName is diff --git a/lib/ai/runtime/direct-agent-image-edit-executor.ts b/lib/ai/runtime/direct-agent-image-edit-executor.ts index 966cf6b77..9f3a0d6ba 100644 --- a/lib/ai/runtime/direct-agent-image-edit-executor.ts +++ b/lib/ai/runtime/direct-agent-image-edit-executor.ts @@ -71,6 +71,12 @@ export interface DirectAgentImageEditExecutionResult { mediaAssetId: string | null; } +export interface DirectAgentBatchImageEditExecutionResult + extends DirectAgentImageEditExecutionResult { + sourceImageUrl: string; + sourceIndex: number; +} + export interface ExecuteDirectAgentImageEditToolParams { capability: DirectAgentToolCapability; requestUrl: string; @@ -497,3 +503,35 @@ export async function executeDirectAgentImageEditTool( throw new Error(`不支持的图片编辑工具能力:${params.capability}`); } + +export async function executeDirectAgentBatchImageEditTool( + params: ExecuteDirectAgentImageEditToolParams, +): Promise { + const sourceImageUrls = params.sourceImageUrls + .map((url) => String(url || "").trim()) + .filter(Boolean); + const results: DirectAgentBatchImageEditExecutionResult[] = []; + + for (const [sourceIndex, sourceImageUrl] of sourceImageUrls.entries()) { + const targetAssetIds = params.targetAssetIds?.length + ? [params.targetAssetIds[sourceIndex] || params.targetAssetIds[0]].filter(Boolean) + : params.targetAssetIds; + const result = await executeDirectAgentImageEditTool({ + ...params, + targetAssetIds, + sourceImageUrl, + sourceImageUrls: [sourceImageUrl], + }); + results.push({ + ...result, + sourceImageUrl, + sourceIndex, + }); + } + + if (results.length <= 0) { + throw new Error("批量图片编辑缺少可处理的源图"); + } + + return results; +} diff --git a/lib/ai/runtime/direct-agent-response-builders.ts b/lib/ai/runtime/direct-agent-response-builders.ts index 5f8af5fbc..c1b349c86 100644 --- a/lib/ai/runtime/direct-agent-response-builders.ts +++ b/lib/ai/runtime/direct-agent-response-builders.ts @@ -1,5 +1,6 @@ import { buildAgentToolImageMarkdown, + buildAgentToolImagesMarkdown, buildAgentToolVideoMarkdown, prependImageModelBadge, } from "@/lib/ai/runtime/agent-media-markdown"; @@ -76,3 +77,22 @@ export function buildDirectAgentImageEditAssistantText(params: { getDirectAgentToolLabel(params.capability), ); } + +export function buildDirectAgentBatchImageEditAssistantText(params: { + capability: DirectAgentToolCapability; + results: Array<{ + originalUrl: string; + thumbnailUrl?: string | null; + imageSize?: string; + }>; +}) { + return prependImageModelBadge( + [ + getDirectAgentToolSuccessMessage(params.capability), + buildAgentToolImagesMarkdown(params.results), + ] + .filter(Boolean) + .join("\n\n"), + getDirectAgentToolLabel(params.capability), + ); +} diff --git a/lib/ai/runtime/direct-agent-working-state.ts b/lib/ai/runtime/direct-agent-working-state.ts index e1ab143c9..b75c67762 100644 --- a/lib/ai/runtime/direct-agent-working-state.ts +++ b/lib/ai/runtime/direct-agent-working-state.ts @@ -43,7 +43,9 @@ export async function safePersistDirectAgentImageEditTurn(params: { capability: DirectAgentToolCapability; currentSubIntent?: string | null; resultAssetId?: string | null; + resultAssetIds?: string[] | null; resultOriginalUrl?: string | null; + resultOriginalUrls?: string[] | null; resultImageSize?: string | null; sourceImageUrls: string[]; messageId: string; @@ -51,8 +53,12 @@ export async function safePersistDirectAgentImageEditTurn(params: { optimizedPrompt?: string | null; }) { try { - const resultAssetIds = params.resultAssetId ? [params.resultAssetId] : null; - const resultImageUrls = params.resultOriginalUrl ? [params.resultOriginalUrl] : null; + const resultAssetIds = + params.resultAssetIds?.filter(Boolean) || + (params.resultAssetId ? [params.resultAssetId] : null); + const resultImageUrls = + params.resultOriginalUrls?.filter(Boolean) || + (params.resultOriginalUrl ? [params.resultOriginalUrl] : null); await persistImageWorkflowTurn({ userId: params.userId, chatId: params.chatId, diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 5832d8a1c..d7a36e9f3 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -125,7 +125,10 @@ import { import { getDirectAgentToolLabel, getDirectAgentToolSuccessMessage, + hasExplicitDirectAgentBatchImageEditIntent, + isDirectAgentBatchImageEditCapability, resolveDirectAgentToolExecution, + shouldRunDirectAgentBatchImageEdit, } from "@/lib/ai/runtime/direct-agent-execution"; import { applyCanvasOperationPlanToCanvasData, @@ -1057,6 +1060,48 @@ test.describe("canvas chat pure helpers", () => { ); }); + test("only runs direct image edit batches for explicit safe batch requests", () => { + expect( + shouldRunDirectAgentBatchImageEdit({ + capability: "remove_watermark", + sourceImageUrls: [ + "https://cdn.example.com/a.png", + "https://cdn.example.com/b.png", + ], + userText: "把这几张都去水印", + }) + ).toBe(true); + expect( + shouldRunDirectAgentBatchImageEdit({ + capability: "remove_background", + sourceImageUrls: [ + "https://cdn.example.com/a.png", + "https://cdn.example.com/b.png", + ], + userText: "每张分别抠图", + }) + ).toBe(true); + expect( + shouldRunDirectAgentBatchImageEdit({ + capability: "remove_watermark", + sourceImageUrls: ["https://cdn.example.com/a.png"], + userText: "全部去水印", + }) + ).toBe(false); + expect( + shouldRunDirectAgentBatchImageEdit({ + capability: "generic_edit", + sourceImageUrls: [ + "https://cdn.example.com/a.png", + "https://cdn.example.com/b.png", + ], + userText: "把这几张都改成黄昏", + }) + ).toBe(false); + expect(hasExplicitDirectAgentBatchImageEditIntent("只处理第一张")).toBe(false); + expect(isDirectAgentBatchImageEditCapability("box_cutout")).toBe(false); + }); + test("plans watermark removal results as non-destructive canvas revisions", () => { const result = buildCanvasAgentDirectWritebackPlan({ projectId: plannerContext.projectId || "", @@ -1201,6 +1246,71 @@ test.describe("canvas chat pure helpers", () => { ).not.toContain("annotation-other"); }); + test("plans direct batch image edits beside each matched source", () => { + const context: CanvasAgentContext = { + ...plannerContext, + selectedNodeIds: ["image-a", "image-b"], + selectedNodes: [ + { + nodeId: "image-a", + type: "image", + title: "A", + x: 0, + y: 0, + width: 200, + height: 160, + isSelected: true, + }, + { + nodeId: "image-b", + type: "image", + title: "B", + x: 260, + y: 0, + width: 200, + height: 160, + isSelected: true, + }, + ], + }; + + const result = buildCanvasAgentDirectWritebackPlan({ + projectId: plannerContext.projectId || "", + chatId: "chat-1", + assistantMessageId: "assistant-batch-edit", + operationLabel: "agent-remove_watermark", + resultType: "image", + resultUrls: [ + "https://cdn.example.com/a-clean.png", + "https://cdn.example.com/b-clean.png", + ], + prompt: "这两张都去水印", + requestTextContent: "这两张都去水印", + mode: "edit", + context, + targetRefs: [{ nodeId: "image-a" }, { nodeId: "image-b" }], + sourceImageUrls: [ + "https://cdn.example.com/a.png", + "https://cdn.example.com/b.png", + ], + }); + + expect(result).toMatchObject({ + operation: "append_revision", + shouldFillSlot: false, + plan: { + operation: "append_revision", + anchorNodeId: "image-a", + placementStrategy: "beside_each_source", + sourceNodeIds: ["image-a", "image-b"], + }, + }); + expect(result?.mediaItems.map((item) => item.sourceNodeIds)).toEqual([ + ["image-a"], + ["image-b"], + ]); + }); + test("plans direct batch generations beside each matched source", () => { const context: CanvasAgentContext = { ...plannerContext,