lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: batch direct canvas image edits

Browse Source
This commit is contained in:
134 2026-06-23 23:42:45 +08:00
parent 0afa404b19
commit 0c95829b1e
6 changed files with 272 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
app/(chat)/api/chat/route.ts
View File

@ -198,6 +198,7 @@ import type { FalImageEditTaskMetadata } from "@/lib/ai/runtime/fal-image-edit-t
import { saveAssistantTextOnlyMessage } from "@/lib/ai/runtime/assistant-message-persistence";
import { resolveCanvasAgentClarificationGate } from "@/lib/ai/runtime/middlewares/canvas-agent-clarification-gate";
import {
executeDirectAgentBatchImageEditTool,
executeDirectAgentImageEditTool,
isDirectAgentImageEditServiceError,
} from "@/lib/ai/runtime/direct-agent-image-edit-executor";
@ -208,6 +209,7 @@ import {
} from "@/lib/ai/runtime/direct-agent-media-executors";
import {
buildDirectAgentAnalysisAssistantText,
buildDirectAgentBatchImageEditAssistantText,
buildDirectAgentImageEditAssistantText,
buildDirectAgentImageGenerationAssistantText,
buildDirectAgentVideoGenerationAssistantText,
@ -242,6 +244,7 @@ import {
getDirectAgentToolTargetMissingMessage,
isDirectAgentImageEditToolName,
resolveDirectAgentToolContext,
shouldRunDirectAgentBatchImageEdit,
} from "@/lib/ai/runtime/direct-agent-execution";
import {
pruneTaskBoundaryContext,
@ -8155,7 +8158,12 @@ export async function POST(request: Request) {
operationLabel: `agent-${directAgentToolCapability}`,
resultType: "image",
});
const directImageEditResult = await executeDirectAgentImageEditTool({
const shouldBatchDirectImageEdit = shouldRunDirectAgentBatchImageEdit({
capability: directAgentToolCapability,
sourceImageUrls: directAgentToolTargetImageUrls,
userText: requestTextContent,
});
const directImageEditParams = {
capability: directAgentToolCapability,
requestUrl: request.url,
requestHeaders: request.headers,
@ -8175,18 +8183,48 @@ export async function POST(request: Request) {
smartAutoExecution,
smartAutoPlan,
saveGeneratedImages,
});
const resultOriginalUrl = directImageEditResult.originalUrl;
const resultThumbnailUrl = directImageEditResult.thumbnailUrl;
const resultImageSize = directImageEditResult.imageSize;
const resultAssetId = directImageEditResult.mediaAssetId;
};
const directImageEditResults = shouldBatchDirectImageEdit
? await executeDirectAgentBatchImageEditTool(directImageEditParams)
: [
{
...(await executeDirectAgentImageEditTool(directImageEditParams)),
sourceImageUrl: directAgentToolTargetImageUrl,
sourceIndex: 0,
},
];
const firstImageEditResult = directImageEditResults[0];
const resultUrls = directImageEditResults
.map((result) => result.originalUrl)
.filter((url): url is string => Boolean(url));
const resultAssetIds = directImageEditResults
.map((result) => result.mediaAssetId)
.filter((assetId): assetId is string => Boolean(assetId));
const resultOriginalUrl = firstImageEditResult?.originalUrl || "";
const resultThumbnailUrl = firstImageEditResult?.thumbnailUrl || null;
const resultImageSize = firstImageEditResult?.imageSize || "2K";
const resultAssetId = firstImageEditResult?.mediaAssetId || null;
const assetIdsByUrl = directImageEditResults.reduce<
Record<string, string | undefined>
>((acc, result) => {
if (result.originalUrl) {
acc[result.originalUrl] = result.mediaAssetId || undefined;
}
return acc;
}, {});
const assistantText = buildDirectAgentImageEditAssistantText({
capability: directAgentToolCapability,
originalUrl: resultOriginalUrl,
thumbnailUrl: resultThumbnailUrl,
imageSize: resultImageSize,
});
const assistantText =
directImageEditResults.length > 1
? buildDirectAgentBatchImageEditAssistantText({
capability: directAgentToolCapability,
results: directImageEditResults,
})
: buildDirectAgentImageEditAssistantText({
capability: directAgentToolCapability,
originalUrl: resultOriginalUrl,
thumbnailUrl: resultThumbnailUrl,
imageSize: resultImageSize,
});
await saveAssistantTextOnlyMessage({
chatId: id,
@ -8201,7 +8239,9 @@ export async function POST(request: Request) {
capability: directAgentToolCapability,
currentSubIntent: smartAutoExecution?.routeSubIntent || directAgentToolCapability,
resultAssetId,
resultAssetIds: resultAssetIds.length > 0 ? resultAssetIds : null,
resultOriginalUrl,
resultOriginalUrls: resultUrls.length > 0 ? resultUrls : null,
resultImageSize,
sourceImageUrls: directAgentToolTargetImageUrls,
messageId: assistantMessageId,
@ -8225,13 +8265,11 @@ export async function POST(request: Request) {
assistantMessageId,
operationLabel: `agent-${directAgentToolCapability}`,
resultType: "image",
resultUrls: resultOriginalUrl ? [resultOriginalUrl] : [],
resultUrls,
prompt: requestTextContent || directToolLabel,
mode: "edit",
sourceImageUrls: directAgentToolTargetImageUrls,
assetIdsByUrl: resultOriginalUrl
? { [resultOriginalUrl]: resultAssetId || undefined }
: undefined,
assetIdsByUrl: resultUrls.length > 0 ? assetIdsByUrl : undefined,
model: finalModelId,
targetSlotId: createdTargetSlotId,
});
@ -8239,7 +8277,7 @@ export async function POST(request: Request) {
writer,
{
messageId: assistantMessageId,
imageUrls: resultOriginalUrl ? [resultOriginalUrl] : [],
imageUrls: resultUrls,
prompt: requestTextContent || directToolLabel,
sourceImageUrls: directAgentToolTargetImageUrls,
mediaItemIds: canvasWriteback.mediaItemIds,

41
lib/ai/runtime/direct-agent-execution.ts
View File

@ -360,6 +360,47 @@ export function resolveDirectAgentToolContext(params: {
};
}
const DIRECT_AGENT_BATCH_IMAGE_EDIT_INTENT_REGEX =
/(全部|都|一起|批量|这几张|这些图|所有|统一处理|统一改|都改|全部改|一起改|一起处理|每张|每一张|分别|各自|同时|一并|各(?:做|出|处理|来)?)/i;
export function hasExplicitDirectAgentBatchImageEditIntent(userText: string) {
const normalizedText = String(userText || "").replace(/\s+/g, "");
return Boolean(
normalizedText &&
DIRECT_AGENT_BATCH_IMAGE_EDIT_INTENT_REGEX.test(normalizedText),
);
}
export function isDirectAgentBatchImageEditCapability(
capability: DirectAgentToolCapability,
) {
switch (capability) {
case "remove_background":
case "remove_watermark":
case "enhance":
case "resize":
case "extend":
return true;
default:
return false;
}
}
export function shouldRunDirectAgentBatchImageEdit(params: {
capability: DirectAgentToolCapability;
sourceImageUrls: string[];
userText: string;
}) {
const validSourceCount = params.sourceImageUrls.filter((url) =>
Boolean(String(url || "").trim()),
).length;
return (
validSourceCount > 1 &&
isDirectAgentBatchImageEditCapability(params.capability) &&
hasExplicitDirectAgentBatchImageEditIntent(params.userText)
);
}
export function isDirectAgentImageEditToolName(
toolName?: DirectAgentToolName | null,
): toolName is

38
lib/ai/runtime/direct-agent-image-edit-executor.ts
View File

@ -71,6 +71,12 @@ export interface DirectAgentImageEditExecutionResult {
mediaAssetId: string | null;
}
export interface DirectAgentBatchImageEditExecutionResult
extends DirectAgentImageEditExecutionResult {
sourceImageUrl: string;
sourceIndex: number;
}
export interface ExecuteDirectAgentImageEditToolParams {
capability: DirectAgentToolCapability;
requestUrl: string;
@ -497,3 +503,35 @@ export async function executeDirectAgentImageEditTool(
throw new Error(`不支持的图片编辑工具能力：${params.capability}`);
}
export async function executeDirectAgentBatchImageEditTool(
params: ExecuteDirectAgentImageEditToolParams,
): Promise<DirectAgentBatchImageEditExecutionResult[]> {
const sourceImageUrls = params.sourceImageUrls
.map((url) => String(url || "").trim())
.filter(Boolean);
const results: DirectAgentBatchImageEditExecutionResult[] = [];
for (const [sourceIndex, sourceImageUrl] of sourceImageUrls.entries()) {
const targetAssetIds = params.targetAssetIds?.length
? [params.targetAssetIds[sourceIndex] || params.targetAssetIds[0]].filter(Boolean)
: params.targetAssetIds;
const result = await executeDirectAgentImageEditTool({
...params,
targetAssetIds,
sourceImageUrl,
sourceImageUrls: [sourceImageUrl],
});
results.push({
...result,
sourceImageUrl,
sourceIndex,
});
}
if (results.length <= 0) {
throw new Error("批量图片编辑缺少可处理的源图");
}
return results;
}

20
lib/ai/runtime/direct-agent-response-builders.ts
View File

@ -1,5 +1,6 @@
import {
buildAgentToolImageMarkdown,
buildAgentToolImagesMarkdown,
buildAgentToolVideoMarkdown,
prependImageModelBadge,
} from "@/lib/ai/runtime/agent-media-markdown";
@ -76,3 +77,22 @@ export function buildDirectAgentImageEditAssistantText(params: {
getDirectAgentToolLabel(params.capability),
);
}
export function buildDirectAgentBatchImageEditAssistantText(params: {
capability: DirectAgentToolCapability;
results: Array<{
originalUrl: string;
thumbnailUrl?: string | null;
imageSize?: string;
}>;
}) {
return prependImageModelBadge(
[
getDirectAgentToolSuccessMessage(params.capability),
buildAgentToolImagesMarkdown(params.results),
]
.filter(Boolean)
.join("\n\n"),
getDirectAgentToolLabel(params.capability),
);
}

10
lib/ai/runtime/direct-agent-working-state.ts
View File

@ -43,7 +43,9 @@ export async function safePersistDirectAgentImageEditTurn(params: {
capability: DirectAgentToolCapability;
currentSubIntent?: string | null;
resultAssetId?: string | null;
resultAssetIds?: string[] | null;
resultOriginalUrl?: string | null;
resultOriginalUrls?: string[] | null;
resultImageSize?: string | null;
sourceImageUrls: string[];
messageId: string;
@ -51,8 +53,12 @@ export async function safePersistDirectAgentImageEditTurn(params: {
optimizedPrompt?: string | null;
}) {
try {
const resultAssetIds = params.resultAssetId ? [params.resultAssetId] : null;
const resultImageUrls = params.resultOriginalUrl ? [params.resultOriginalUrl] : null;
const resultAssetIds =
params.resultAssetIds?.filter(Boolean) ||
(params.resultAssetId ? [params.resultAssetId] : null);
const resultImageUrls =
params.resultOriginalUrls?.filter(Boolean) ||
(params.resultOriginalUrl ? [params.resultOriginalUrl] : null);
await persistImageWorkflowTurn({
userId: params.userId,
chatId: params.chatId,

110
tests/routes/canvas-chat-helpers.test.ts
View File

@ -125,7 +125,10 @@ import {
import {
getDirectAgentToolLabel,
getDirectAgentToolSuccessMessage,
hasExplicitDirectAgentBatchImageEditIntent,
isDirectAgentBatchImageEditCapability,
resolveDirectAgentToolExecution,
shouldRunDirectAgentBatchImageEdit,
} from "@/lib/ai/runtime/direct-agent-execution";
import {
applyCanvasOperationPlanToCanvasData,
@ -1057,6 +1060,48 @@ test.describe("canvas chat pure helpers", () => {
);
});
test("only runs direct image edit batches for explicit safe batch requests", () => {
expect(
shouldRunDirectAgentBatchImageEdit({
capability: "remove_watermark",
sourceImageUrls: [
"https://cdn.example.com/a.png",
"https://cdn.example.com/b.png",
],
userText: "把这几张都去水印",
})
).toBe(true);
expect(
shouldRunDirectAgentBatchImageEdit({
capability: "remove_background",
sourceImageUrls: [
"https://cdn.example.com/a.png",
"https://cdn.example.com/b.png",
],
userText: "每张分别抠图",
})
).toBe(true);
expect(
shouldRunDirectAgentBatchImageEdit({
capability: "remove_watermark",
sourceImageUrls: ["https://cdn.example.com/a.png"],
userText: "全部去水印",
})
).toBe(false);
expect(
shouldRunDirectAgentBatchImageEdit({
capability: "generic_edit",
sourceImageUrls: [
"https://cdn.example.com/a.png",
"https://cdn.example.com/b.png",
],
userText: "把这几张都改成黄昏",
})
).toBe(false);
expect(hasExplicitDirectAgentBatchImageEditIntent("只处理第一张")).toBe(false);
expect(isDirectAgentBatchImageEditCapability("box_cutout")).toBe(false);
});
test("plans watermark removal results as non-destructive canvas revisions", () => {
const result = buildCanvasAgentDirectWritebackPlan({
projectId: plannerContext.projectId || "",
@ -1201,6 +1246,71 @@ test.describe("canvas chat pure helpers", () => {
).not.toContain("annotation-other");
});
test("plans direct batch image edits beside each matched source", () => {
const context: CanvasAgentContext = {
...plannerContext,
selectedNodeIds: ["image-a", "image-b"],
selectedNodes: [
{
nodeId: "image-a",
type: "image",
title: "A",
x: 0,
y: 0,
width: 200,
height: 160,
isSelected: true,
},
{
nodeId: "image-b",
type: "image",
title: "B",
x: 260,
y: 0,
width: 200,
height: 160,
isSelected: true,
},
],
};
const result = buildCanvasAgentDirectWritebackPlan({
projectId: plannerContext.projectId || "",
chatId: "chat-1",
assistantMessageId: "assistant-batch-edit",
operationLabel: "agent-remove_watermark",
resultType: "image",
resultUrls: [
"https://cdn.example.com/a-clean.png",
"https://cdn.example.com/b-clean.png",
],
prompt: "这两张都去水印",
requestTextContent: "这两张都去水印",
mode: "edit",
context,
targetRefs: [{ nodeId: "image-a" }, { nodeId: "image-b" }],
sourceImageUrls: [
"https://cdn.example.com/a.png",
"https://cdn.example.com/b.png",
],
});
expect(result).toMatchObject({
operation: "append_revision",
shouldFillSlot: false,
plan: {
operation: "append_revision",
anchorNodeId: "image-a",
placementStrategy: "beside_each_source",
sourceNodeIds: ["image-a", "image-b"],
},
});
expect(result?.mediaItems.map((item) => item.sourceNodeIds)).toEqual([
["image-a"],
["image-b"],
]);
});
test("plans direct batch generations beside each matched source", () => {
const context: CanvasAgentContext = {
...plannerContext,