ai-chatbot/lib/ai/runtime/direct-agent-execution.ts
2026-06-24 00:17:03 +08:00

670 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import type {
AgentExecutionPlan,
AgentPreparedExecution,
AgentPlannedToolCall,
} from "@/lib/ai/auto-agent-graph";
import type {
ResolvedChatMediaInput,
ResolvedTargetAssets,
} from "@/lib/ai/context-memory/types";
export type DirectAgentToolCapability =
| "analyze"
| "generic_edit"
| "remove_background"
| "remove_watermark"
| "enhance"
| "resize"
| "extend"
| "angle_variant"
| "local_edit"
| "box_cutout"
| "generate_image"
| "generate_video";
export type DirectAgentToolName =
| "analyzeImage"
| "editImage"
| "removeBackground"
| "removeWatermark"
| "enhanceImage"
| "resizeImage"
| "extendImage"
| "angleEditImage"
| "localEditImage"
| "boxCutout"
| "generateImage"
| "generateVideo";
type DirectAgentSourceToolName =
| "useSelectedCanvasImage"
| "useLatestResultImage"
| "useOriginalUploadImage";
export type DirectAgentSourcePreference =
| "selected_canvas"
| "latest_result"
| "original_upload"
| "auto";
export interface ResolvedDirectAgentToolExecution {
toolName: DirectAgentToolName;
capability: DirectAgentToolCapability;
sourcePreferences: DirectAgentSourcePreference[];
publishToCanvas: boolean;
}
export interface ResolvedDirectAgentToolContext {
execution: ResolvedDirectAgentToolExecution;
targetImageUrls: string[];
targetImageUrl: string | null;
}
export interface DirectAgentWorkingStateSnapshot {
lastResolvedTargetImageUrls?: string[] | null;
lastGeneratedImageUrls?: string[] | null;
lastUploadedImageUrls?: string[] | null;
}
function collectAgentToolCalls(
execution?: AgentPreparedExecution,
plan?: AgentExecutionPlan,
): AgentPlannedToolCall[] {
const merged = [...(execution?.toolCalls || []), ...(plan?.toolCalls || [])];
const seen = new Set<string>();
const result: AgentPlannedToolCall[] = [];
for (const tool of merged) {
const name = String(tool?.name || "").trim();
if (!name || seen.has(name)) continue;
seen.add(name);
result.push(tool);
}
return result;
}
function mapDirectToolNameToCapability(
toolName: DirectAgentToolName,
): DirectAgentToolCapability {
switch (toolName) {
case "analyzeImage":
return "analyze";
case "editImage":
return "generic_edit";
case "removeBackground":
return "remove_background";
case "removeWatermark":
return "remove_watermark";
case "enhanceImage":
return "enhance";
case "resizeImage":
return "resize";
case "extendImage":
return "extend";
case "angleEditImage":
return "angle_variant";
case "localEditImage":
return "local_edit";
case "boxCutout":
return "box_cutout";
case "generateImage":
return "generate_image";
case "generateVideo":
return "generate_video";
default:
return "generate_image";
}
}
function mapLegacySourceToolToPreference(
toolName: DirectAgentSourceToolName,
): DirectAgentSourcePreference {
switch (toolName) {
case "useSelectedCanvasImage":
return "selected_canvas";
case "useLatestResultImage":
return "latest_result";
case "useOriginalUploadImage":
return "original_upload";
default:
return "auto";
}
}
function normalizeSourcePreference(value: unknown): DirectAgentSourcePreference | null {
const normalized = String(value || "").trim().toLowerCase();
switch (normalized) {
case "selected_canvas":
case "selected":
case "canvas_selection":
case "explicit_selection":
return "selected_canvas";
case "latest_result":
case "latest":
case "working_memory":
case "history":
return "latest_result";
case "original_upload":
case "current_upload":
case "upload":
case "current":
return "original_upload";
case "auto":
return "auto";
default:
return null;
}
}
function mapTargetSourceToSourcePreference(
targetSource: AgentExecutionPlan["targetSource"] | undefined,
): DirectAgentSourcePreference | null {
switch (targetSource) {
case "explicit_selection":
return "selected_canvas";
case "latest_result":
case "working_memory":
return "latest_result";
case "current_upload":
return "original_upload";
case "text_reference":
return "auto";
default:
return null;
}
}
export function resolveDirectAgentToolExecution(
execution?: AgentPreparedExecution,
plan?: AgentExecutionPlan,
): ResolvedDirectAgentToolExecution | null {
const mergedToolCalls = collectAgentToolCalls(execution, plan);
const toolNames = mergedToolCalls.map((tool) => String(tool.name || ""));
const toolNameSet = new Set(toolNames);
const hasToolCall = (toolName: string) => toolNameSet.has(toolName);
const preferredSpecificImageEditTool = (() => {
const preferredCapability = execution?.preferredCapability || plan?.preferredCapability;
const routeSubIntent = execution?.routeSubIntent || plan?.subIntent;
if (
preferredCapability === "remove_background" ||
routeSubIntent === "remove_background"
) {
return "removeBackground";
}
if (
preferredCapability === "remove_watermark" ||
routeSubIntent === "remove_watermark"
) {
return "removeWatermark";
}
if (preferredCapability === "enhance" || routeSubIntent === "enhance") {
return "enhanceImage";
}
if (preferredCapability === "resize" || routeSubIntent === "resize") {
return "resizeImage";
}
if (preferredCapability === "extend" || routeSubIntent === "extend") {
return "extendImage";
}
if (
preferredCapability === "angle_variant" ||
routeSubIntent === "angle_variant" ||
routeSubIntent === "generate_angle_variant"
) {
return "angleEditImage";
}
if (
preferredCapability === "local_edit" ||
routeSubIntent === "local_edit" ||
routeSubIntent === "inpaint_region"
) {
return "localEditImage";
}
if (preferredCapability === "box_cutout" || routeSubIntent === "box_cutout") {
return "boxCutout";
}
return null;
})();
const sourceTools = toolNames.filter((toolName): toolName is DirectAgentSourceToolName =>
toolName === "useSelectedCanvasImage" ||
toolName === "useLatestResultImage" ||
toolName === "useOriginalUploadImage",
);
const sourcePreferences: DirectAgentSourcePreference[] = [];
const appendSourcePreference = (value: DirectAgentSourcePreference | null) => {
if (!value || sourcePreferences.includes(value)) return;
sourcePreferences.push(value);
};
for (const sourceTool of sourceTools) {
appendSourcePreference(mapLegacySourceToolToPreference(sourceTool));
}
for (const tool of mergedToolCalls) {
if (tool.name !== "resolveTargetImage") continue;
const sourceArg = normalizeSourcePreference(tool.args?.source ?? tool.args?.sourcePreference);
appendSourcePreference(sourceArg);
const sourceList = Array.isArray(tool.args?.sourcePreferences)
? tool.args?.sourcePreferences
: Array.isArray(tool.args?.sources)
? tool.args?.sources
: [];
for (const item of sourceList) {
appendSourcePreference(normalizeSourcePreference(item));
}
}
if (sourcePreferences.length === 0) {
appendSourcePreference(
mapTargetSourceToSourcePreference(execution?.targetSource || plan?.targetSource),
);
}
const directToolName =
(hasToolCall("analyzeTargetImage") || hasToolCall("analyzeImage")
? "analyzeImage"
: hasToolCall("removeBackground")
? "removeBackground"
: hasToolCall("removeWatermark")
? "removeWatermark"
: hasToolCall("enhanceImage")
? "enhanceImage"
: hasToolCall("resizeImage")
? "resizeImage"
: hasToolCall("extendImage")
? "extendImage"
: hasToolCall("angleEditImage")
? "angleEditImage"
: hasToolCall("localEditImage")
? "localEditImage"
: hasToolCall("boxCutout")
? "boxCutout"
: hasToolCall("editTargetImage") || hasToolCall("editImage")
? preferredSpecificImageEditTool || "editImage"
: hasToolCall("generateNewImage") || hasToolCall("generateImage")
? "generateImage"
: hasToolCall("generateVideo")
? "generateVideo"
: null) ||
(() => {
if (
execution?.preferredCapability === "analyze" ||
execution?.routeIntent === "vision" ||
execution?.mode === "analyze_image"
) {
return "analyzeImage";
}
if (
execution?.preferredCapability === "remove_background" ||
execution?.routeSubIntent === "remove_background"
) {
return "removeBackground";
}
if (
execution?.preferredCapability === "remove_watermark" ||
execution?.routeSubIntent === "remove_watermark"
) {
return "removeWatermark";
}
if (
execution?.preferredCapability === "enhance" ||
execution?.routeSubIntent === "enhance"
) {
return "enhanceImage";
}
if (
execution?.preferredCapability === "resize" ||
execution?.routeSubIntent === "resize"
) {
return "resizeImage";
}
if (
execution?.preferredCapability === "extend" ||
execution?.routeSubIntent === "extend"
) {
return "extendImage";
}
if (
execution?.preferredCapability === "angle_variant" ||
execution?.routeSubIntent === "angle_variant" ||
execution?.routeSubIntent === "generate_angle_variant"
) {
return "angleEditImage";
}
if (
execution?.preferredCapability === "local_edit" ||
execution?.routeSubIntent === "local_edit" ||
execution?.routeSubIntent === "inpaint_region"
) {
return "localEditImage";
}
if (
execution?.preferredCapability === "box_cutout" ||
execution?.routeSubIntent === "box_cutout"
) {
return "boxCutout";
}
if (
execution?.preferredCapability === "generic_edit" ||
execution?.routeIntent === "image_edit" ||
execution?.mode === "edit_image"
) {
return "editImage";
}
if (execution?.mode === "generate_image") {
return "generateImage";
}
if (execution?.mode === "generate_video") {
return "generateVideo";
}
return null;
})();
if (!directToolName) {
return null;
}
return {
toolName: directToolName,
capability: mapDirectToolNameToCapability(directToolName),
sourcePreferences,
publishToCanvas:
execution?.shouldPublishToCanvas === true ||
toolNames.includes("publishToCanvas") ||
toolNames.includes("publishResultToCanvas"),
};
}
export function resolveDirectAgentToolContext(params: {
execution?: AgentPreparedExecution;
plan?: AgentExecutionPlan;
requestResolvedMediaInput?: ResolvedChatMediaInput | null;
requestTargetResolution?: ResolvedTargetAssets | null;
requestWorkingState?: DirectAgentWorkingStateSnapshot | null;
}): ResolvedDirectAgentToolContext | null {
const execution = resolveDirectAgentToolExecution(params.execution, params.plan);
if (!execution) return null;
const targetImageUrls = pickAgentSourceImageUrls({
sourcePreferences: execution.sourcePreferences,
requestResolvedMediaInput: params.requestResolvedMediaInput,
requestTargetResolution: params.requestTargetResolution,
requestWorkingState: params.requestWorkingState,
});
return {
execution,
targetImageUrls,
targetImageUrl: targetImageUrls[0] || null,
};
}
const DIRECT_AGENT_BATCH_IMAGE_EDIT_INTENT_REGEX =
/(全部|都|一起|批量|这几张|这些图|所有|统一处理|统一改|都改|全部改|一起改|一起处理|每张|每一张|分别|各自|同时|一并|各(?:做|出|处理|来)?)/i;
export function hasExplicitDirectAgentBatchImageEditIntent(userText: string) {
const normalizedText = String(userText || "").replace(/\s+/g, "");
return Boolean(
normalizedText &&
DIRECT_AGENT_BATCH_IMAGE_EDIT_INTENT_REGEX.test(normalizedText),
);
}
export function isDirectAgentBatchImageEditCapability(
capability: DirectAgentToolCapability,
) {
switch (capability) {
case "remove_background":
case "remove_watermark":
case "enhance":
case "resize":
case "extend":
case "angle_variant":
return true;
case "local_edit":
return false;
default:
return false;
}
}
export function shouldRunDirectAgentBatchImageEdit(params: {
capability: DirectAgentToolCapability;
sourceImageUrls: string[];
userText: string;
}) {
const validSourceCount = params.sourceImageUrls.filter((url) =>
Boolean(String(url || "").trim()),
).length;
return (
validSourceCount > 1 &&
isDirectAgentBatchImageEditCapability(params.capability) &&
hasExplicitDirectAgentBatchImageEditIntent(params.userText)
);
}
export function isDirectAgentImageEditToolName(
toolName?: DirectAgentToolName | null,
): toolName is
| "editImage"
| "enhanceImage"
| "removeBackground"
| "removeWatermark"
| "resizeImage"
| "extendImage"
| "angleEditImage"
| "localEditImage"
| "boxCutout" {
switch (toolName) {
case "editImage":
case "enhanceImage":
case "removeBackground":
case "removeWatermark":
case "resizeImage":
case "extendImage":
case "angleEditImage":
case "localEditImage":
case "boxCutout":
return true;
default:
return false;
}
}
export function getDirectAgentToolLabel(capability: DirectAgentToolCapability): string {
switch (capability) {
case "analyze":
return "Agent工具:图片分析";
case "generic_edit":
return "Agent工具:图片编辑";
case "remove_background":
return "Agent工具:抠图";
case "remove_watermark":
return "Agent工具:去水印";
case "enhance":
return "Agent工具:图像增强";
case "resize":
return "Agent工具:尺寸调整";
case "extend":
return "Agent工具:图像拓展";
case "angle_variant":
return "Agent工具:多视角";
case "local_edit":
return "Agent工具:局部编辑";
case "box_cutout":
return "Agent工具:框选抠图";
case "generate_image":
return "Agent工具:图片生成";
case "generate_video":
return "Agent工具:视频生成";
default:
return "Agent工具";
}
}
export function findAgentToolCallArgs(
toolName: AgentPlannedToolCall["name"],
execution?: AgentPreparedExecution,
plan?: AgentExecutionPlan,
): Record<string, unknown> | undefined {
const matchedTool =
execution?.toolCalls?.find((tool) => tool.name === toolName) ||
plan?.toolCalls?.find((tool) => tool.name === toolName);
if (!matchedTool?.args || typeof matchedTool.args !== "object") {
return undefined;
}
return matchedTool.args;
}
export function getDirectAgentToolSuccessMessage(
capability: DirectAgentToolCapability,
): string {
switch (capability) {
case "analyze":
return "已完成当前目标图分析,结果如下:";
case "generic_edit":
return "已完成图片编辑,结果如下:";
case "remove_background":
return "已完成抠图处理,结果如下:";
case "remove_watermark":
return "已完成去水印处理,结果如下:";
case "enhance":
return "已完成图像增强,结果如下:";
case "resize":
return "已完成尺寸调整,结果如下:";
case "extend":
return "已完成图像拓展,结果如下:";
case "angle_variant":
return "已完成多视角生成,结果如下:";
case "local_edit":
return "已完成局部编辑,结果如下:";
case "box_cutout":
return "已完成框选抠图,结果如下:";
case "generate_image":
return "已完成图片生成,结果如下:";
case "generate_video":
return "已完成视频生成,结果如下:";
default:
return "已完成处理,结果如下:";
}
}
export function getDirectAgentToolTargetMissingMessage(
capability: DirectAgentToolCapability,
): string {
switch (capability) {
case "analyze":
return "请先上传或选中一张图片,我才能继续帮你分析。";
case "generic_edit":
return "请先上传或选中一张图片,我才能继续帮你修改。";
case "remove_background":
return "请先上传或选中一张图片,我才能继续帮你执行抠图。";
case "remove_watermark":
return "请先上传或选中一张图片,我才能继续帮你去水印。";
case "enhance":
return "请先上传或选中一张图片,我才能继续帮你做图像增强。";
case "resize":
return "请先上传或选中一张图片,我才能继续帮你调整尺寸。";
case "extend":
return "请先上传或选中一张图片,我才能继续帮你做图像拓展。";
case "angle_variant":
return "请先上传或选中一张图片,我才能继续帮你生成多视角变体。";
case "local_edit":
return "请先选中一张图片并框选局部编辑区域,我才能继续帮你做局部编辑。";
case "box_cutout":
return "请先选中一张图片并提供框选区域,我才能继续帮你做框选抠图。";
case "generate_image":
return "请先告诉我你想生成什么,或者提供参考图。";
case "generate_video":
return "请先告诉我你想生成什么视频,必要时再提供参考图。";
default:
return "请先提供要处理的图片。";
}
}
export function getDirectAgentToolErrorMessage(
capability: DirectAgentToolCapability,
error: unknown,
): string {
const errorMessage =
error instanceof Error ? String(error.message || "").trim() : String(error || "").trim();
const errorDetail =
error && typeof error === "object" && "detail" in error
? String((error as { detail?: unknown }).detail || "").trim()
: "";
const detail = errorDetail || errorMessage || "请稍后重试。";
switch (capability) {
case "analyze":
return `图片分析暂时失败:${detail}`;
case "generic_edit":
return `图片编辑暂时失败:${detail}`;
case "remove_background":
return `抠图暂时失败:${detail}`;
case "remove_watermark":
return `去水印暂时失败:${detail}`;
case "enhance":
return `图像增强暂时失败:${detail}`;
case "resize":
return `尺寸调整暂时失败:${detail}`;
case "extend":
return `图像拓展暂时失败:${detail}`;
case "angle_variant":
return `多视角生成暂时失败:${detail}`;
case "local_edit":
return `局部编辑暂时失败:${detail}`;
case "box_cutout":
return `框选抠图暂时失败:${detail}`;
case "generate_image":
return `图片生成暂时失败:${detail}`;
case "generate_video":
return `视频生成暂时失败:${detail}`;
default:
return `处理暂时失败:${detail}`;
}
}
export function pickAgentSourceImageUrls(params: {
sourcePreferences: DirectAgentSourcePreference[];
requestResolvedMediaInput?: ResolvedChatMediaInput | null;
requestTargetResolution?: ResolvedTargetAssets | null;
requestWorkingState?: DirectAgentWorkingStateSnapshot | null;
}): string[] {
const preferredBuckets: string[][] = [];
const sourcePreferences = params.sourcePreferences || [];
if (sourcePreferences.includes("selected_canvas")) {
preferredBuckets.push(params.requestResolvedMediaInput?.selectedImageUrls || []);
}
if (sourcePreferences.includes("latest_result")) {
preferredBuckets.push(params.requestWorkingState?.lastGeneratedImageUrls || []);
preferredBuckets.push(params.requestResolvedMediaInput?.historyAssistantImageUrls || []);
}
if (sourcePreferences.includes("original_upload")) {
preferredBuckets.push(params.requestResolvedMediaInput?.currentMessageImageUrls || []);
preferredBuckets.push(params.requestWorkingState?.lastUploadedImageUrls || []);
preferredBuckets.push(params.requestResolvedMediaInput?.historyUserImageUrls || []);
}
preferredBuckets.push(
params.requestTargetResolution?.targetImageUrls || [],
params.requestResolvedMediaInput?.selectedImageUrls || [],
params.requestResolvedMediaInput?.currentMessageImageUrls || [],
params.requestWorkingState?.lastResolvedTargetImageUrls || [],
params.requestWorkingState?.lastGeneratedImageUrls || [],
params.requestResolvedMediaInput?.historyAssistantImageUrls || [],
params.requestResolvedMediaInput?.historyUserImageUrls || [],
);
return Array.from(
new Set(
preferredBuckets.flatMap((bucket) =>
(bucket || []).map((item) => String(item || "").trim()).filter(Boolean),
),
),
);
}