515 lines
16 KiB
TypeScript
515 lines
16 KiB
TypeScript
import type {
|
||
AgentExecutionPlan,
|
||
AgentPreparedExecution,
|
||
AgentPlannedToolCall,
|
||
} from "@/lib/ai/auto-agent-graph";
|
||
import type {
|
||
ResolvedChatMediaInput,
|
||
ResolvedTargetAssets,
|
||
} from "@/lib/ai/context-memory/types";
|
||
|
||
export type DirectAgentToolCapability =
|
||
| "analyze"
|
||
| "generic_edit"
|
||
| "remove_background"
|
||
| "enhance"
|
||
| "resize"
|
||
| "extend"
|
||
| "box_cutout"
|
||
| "generate_image"
|
||
| "generate_video";
|
||
|
||
export type DirectAgentToolName =
|
||
| "analyzeImage"
|
||
| "editImage"
|
||
| "removeBackground"
|
||
| "enhanceImage"
|
||
| "resizeImage"
|
||
| "extendImage"
|
||
| "boxCutout"
|
||
| "generateImage"
|
||
| "generateVideo";
|
||
|
||
type DirectAgentSourceToolName =
|
||
| "useSelectedCanvasImage"
|
||
| "useLatestResultImage"
|
||
| "useOriginalUploadImage";
|
||
|
||
export type DirectAgentSourcePreference =
|
||
| "selected_canvas"
|
||
| "latest_result"
|
||
| "original_upload"
|
||
| "auto";
|
||
|
||
export interface ResolvedDirectAgentToolExecution {
|
||
toolName: DirectAgentToolName;
|
||
capability: DirectAgentToolCapability;
|
||
sourcePreferences: DirectAgentSourcePreference[];
|
||
publishToCanvas: boolean;
|
||
}
|
||
|
||
export interface ResolvedDirectAgentToolContext {
|
||
execution: ResolvedDirectAgentToolExecution;
|
||
targetImageUrls: string[];
|
||
targetImageUrl: string | null;
|
||
}
|
||
|
||
export interface DirectAgentWorkingStateSnapshot {
|
||
lastResolvedTargetImageUrls?: string[] | null;
|
||
lastGeneratedImageUrls?: string[] | null;
|
||
lastUploadedImageUrls?: string[] | null;
|
||
}
|
||
|
||
function collectAgentToolCalls(
|
||
execution?: AgentPreparedExecution,
|
||
plan?: AgentExecutionPlan,
|
||
): AgentPlannedToolCall[] {
|
||
const merged = [...(execution?.toolCalls || []), ...(plan?.toolCalls || [])];
|
||
const seen = new Set<string>();
|
||
const result: AgentPlannedToolCall[] = [];
|
||
|
||
for (const tool of merged) {
|
||
const name = String(tool?.name || "").trim();
|
||
if (!name || seen.has(name)) continue;
|
||
seen.add(name);
|
||
result.push(tool);
|
||
}
|
||
|
||
return result;
|
||
}
|
||
|
||
function mapDirectToolNameToCapability(
|
||
toolName: DirectAgentToolName,
|
||
): DirectAgentToolCapability {
|
||
switch (toolName) {
|
||
case "analyzeImage":
|
||
return "analyze";
|
||
case "editImage":
|
||
return "generic_edit";
|
||
case "removeBackground":
|
||
return "remove_background";
|
||
case "enhanceImage":
|
||
return "enhance";
|
||
case "resizeImage":
|
||
return "resize";
|
||
case "extendImage":
|
||
return "extend";
|
||
case "boxCutout":
|
||
return "box_cutout";
|
||
case "generateImage":
|
||
return "generate_image";
|
||
case "generateVideo":
|
||
return "generate_video";
|
||
default:
|
||
return "generate_image";
|
||
}
|
||
}
|
||
|
||
function mapLegacySourceToolToPreference(
|
||
toolName: DirectAgentSourceToolName,
|
||
): DirectAgentSourcePreference {
|
||
switch (toolName) {
|
||
case "useSelectedCanvasImage":
|
||
return "selected_canvas";
|
||
case "useLatestResultImage":
|
||
return "latest_result";
|
||
case "useOriginalUploadImage":
|
||
return "original_upload";
|
||
default:
|
||
return "auto";
|
||
}
|
||
}
|
||
|
||
function normalizeSourcePreference(value: unknown): DirectAgentSourcePreference | null {
|
||
const normalized = String(value || "").trim().toLowerCase();
|
||
switch (normalized) {
|
||
case "selected_canvas":
|
||
case "selected":
|
||
case "canvas_selection":
|
||
case "explicit_selection":
|
||
return "selected_canvas";
|
||
case "latest_result":
|
||
case "latest":
|
||
case "working_memory":
|
||
case "history":
|
||
return "latest_result";
|
||
case "original_upload":
|
||
case "current_upload":
|
||
case "upload":
|
||
case "current":
|
||
return "original_upload";
|
||
case "auto":
|
||
return "auto";
|
||
default:
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function mapTargetSourceToSourcePreference(
|
||
targetSource: AgentExecutionPlan["targetSource"] | undefined,
|
||
): DirectAgentSourcePreference | null {
|
||
switch (targetSource) {
|
||
case "explicit_selection":
|
||
return "selected_canvas";
|
||
case "latest_result":
|
||
case "working_memory":
|
||
return "latest_result";
|
||
case "current_upload":
|
||
return "original_upload";
|
||
case "text_reference":
|
||
return "auto";
|
||
default:
|
||
return null;
|
||
}
|
||
}
|
||
|
||
export function resolveDirectAgentToolExecution(
|
||
execution?: AgentPreparedExecution,
|
||
plan?: AgentExecutionPlan,
|
||
): ResolvedDirectAgentToolExecution | null {
|
||
const mergedToolCalls = collectAgentToolCalls(execution, plan);
|
||
const toolNames = mergedToolCalls.map((tool) => String(tool.name || ""));
|
||
const toolNameSet = new Set(toolNames);
|
||
const hasToolCall = (toolName: string) => toolNameSet.has(toolName);
|
||
const sourceTools = toolNames.filter((toolName): toolName is DirectAgentSourceToolName =>
|
||
toolName === "useSelectedCanvasImage" ||
|
||
toolName === "useLatestResultImage" ||
|
||
toolName === "useOriginalUploadImage",
|
||
);
|
||
const sourcePreferences: DirectAgentSourcePreference[] = [];
|
||
const appendSourcePreference = (value: DirectAgentSourcePreference | null) => {
|
||
if (!value || sourcePreferences.includes(value)) return;
|
||
sourcePreferences.push(value);
|
||
};
|
||
|
||
for (const sourceTool of sourceTools) {
|
||
appendSourcePreference(mapLegacySourceToolToPreference(sourceTool));
|
||
}
|
||
|
||
for (const tool of mergedToolCalls) {
|
||
if (tool.name !== "resolveTargetImage") continue;
|
||
const sourceArg = normalizeSourcePreference(tool.args?.source ?? tool.args?.sourcePreference);
|
||
appendSourcePreference(sourceArg);
|
||
const sourceList = Array.isArray(tool.args?.sourcePreferences)
|
||
? tool.args?.sourcePreferences
|
||
: Array.isArray(tool.args?.sources)
|
||
? tool.args?.sources
|
||
: [];
|
||
for (const item of sourceList) {
|
||
appendSourcePreference(normalizeSourcePreference(item));
|
||
}
|
||
}
|
||
|
||
if (sourcePreferences.length === 0) {
|
||
appendSourcePreference(
|
||
mapTargetSourceToSourcePreference(execution?.targetSource || plan?.targetSource),
|
||
);
|
||
}
|
||
|
||
const directToolName =
|
||
(hasToolCall("analyzeTargetImage") || hasToolCall("analyzeImage")
|
||
? "analyzeImage"
|
||
: hasToolCall("removeBackground")
|
||
? "removeBackground"
|
||
: hasToolCall("enhanceImage")
|
||
? "enhanceImage"
|
||
: hasToolCall("resizeImage")
|
||
? "resizeImage"
|
||
: hasToolCall("extendImage")
|
||
? "extendImage"
|
||
: hasToolCall("boxCutout")
|
||
? "boxCutout"
|
||
: hasToolCall("editTargetImage") || hasToolCall("editImage")
|
||
? "editImage"
|
||
: hasToolCall("generateNewImage") || hasToolCall("generateImage")
|
||
? "generateImage"
|
||
: hasToolCall("generateVideo")
|
||
? "generateVideo"
|
||
: null) ||
|
||
(() => {
|
||
if (
|
||
execution?.preferredCapability === "analyze" ||
|
||
execution?.routeIntent === "vision" ||
|
||
execution?.mode === "analyze_image"
|
||
) {
|
||
return "analyzeImage";
|
||
}
|
||
if (
|
||
execution?.preferredCapability === "remove_background" ||
|
||
execution?.routeSubIntent === "remove_background"
|
||
) {
|
||
return "removeBackground";
|
||
}
|
||
if (
|
||
execution?.preferredCapability === "enhance" ||
|
||
execution?.routeSubIntent === "enhance"
|
||
) {
|
||
return "enhanceImage";
|
||
}
|
||
if (
|
||
execution?.preferredCapability === "resize" ||
|
||
execution?.routeSubIntent === "resize"
|
||
) {
|
||
return "resizeImage";
|
||
}
|
||
if (
|
||
execution?.preferredCapability === "extend" ||
|
||
execution?.routeSubIntent === "extend"
|
||
) {
|
||
return "extendImage";
|
||
}
|
||
if (
|
||
execution?.preferredCapability === "box_cutout" ||
|
||
execution?.routeSubIntent === "box_cutout"
|
||
) {
|
||
return "boxCutout";
|
||
}
|
||
if (
|
||
execution?.preferredCapability === "generic_edit" ||
|
||
execution?.routeIntent === "image_edit" ||
|
||
execution?.mode === "edit_image"
|
||
) {
|
||
return "editImage";
|
||
}
|
||
if (execution?.mode === "generate_image") {
|
||
return "generateImage";
|
||
}
|
||
if (execution?.mode === "generate_video") {
|
||
return "generateVideo";
|
||
}
|
||
return null;
|
||
})();
|
||
|
||
if (!directToolName) {
|
||
return null;
|
||
}
|
||
|
||
return {
|
||
toolName: directToolName,
|
||
capability: mapDirectToolNameToCapability(directToolName),
|
||
sourcePreferences,
|
||
publishToCanvas:
|
||
execution?.shouldPublishToCanvas === true ||
|
||
toolNames.includes("publishToCanvas") ||
|
||
toolNames.includes("publishResultToCanvas"),
|
||
};
|
||
}
|
||
|
||
export function resolveDirectAgentToolContext(params: {
|
||
execution?: AgentPreparedExecution;
|
||
plan?: AgentExecutionPlan;
|
||
requestResolvedMediaInput?: ResolvedChatMediaInput | null;
|
||
requestTargetResolution?: ResolvedTargetAssets | null;
|
||
requestWorkingState?: DirectAgentWorkingStateSnapshot | null;
|
||
}): ResolvedDirectAgentToolContext | null {
|
||
const execution = resolveDirectAgentToolExecution(params.execution, params.plan);
|
||
if (!execution) return null;
|
||
|
||
const targetImageUrls = pickAgentSourceImageUrls({
|
||
sourcePreferences: execution.sourcePreferences,
|
||
requestResolvedMediaInput: params.requestResolvedMediaInput,
|
||
requestTargetResolution: params.requestTargetResolution,
|
||
requestWorkingState: params.requestWorkingState,
|
||
});
|
||
|
||
return {
|
||
execution,
|
||
targetImageUrls,
|
||
targetImageUrl: targetImageUrls[0] || null,
|
||
};
|
||
}
|
||
|
||
export function isDirectAgentImageEditToolName(
|
||
toolName?: DirectAgentToolName | null,
|
||
): toolName is
|
||
| "editImage"
|
||
| "enhanceImage"
|
||
| "removeBackground"
|
||
| "resizeImage"
|
||
| "extendImage"
|
||
| "boxCutout" {
|
||
switch (toolName) {
|
||
case "editImage":
|
||
case "enhanceImage":
|
||
case "removeBackground":
|
||
case "resizeImage":
|
||
case "extendImage":
|
||
case "boxCutout":
|
||
return true;
|
||
default:
|
||
return false;
|
||
}
|
||
}
|
||
|
||
export function getDirectAgentToolLabel(capability: DirectAgentToolCapability): string {
|
||
switch (capability) {
|
||
case "analyze":
|
||
return "Agent工具：图片分析";
|
||
case "generic_edit":
|
||
return "Agent工具：图片编辑";
|
||
case "remove_background":
|
||
return "Agent工具：抠图";
|
||
case "enhance":
|
||
return "Agent工具：图像增强";
|
||
case "resize":
|
||
return "Agent工具：尺寸调整";
|
||
case "extend":
|
||
return "Agent工具：图像拓展";
|
||
case "box_cutout":
|
||
return "Agent工具：框选抠图";
|
||
case "generate_image":
|
||
return "Agent工具：图片生成";
|
||
case "generate_video":
|
||
return "Agent工具：视频生成";
|
||
default:
|
||
return "Agent工具";
|
||
}
|
||
}
|
||
|
||
export function findAgentToolCallArgs(
|
||
toolName: AgentPlannedToolCall["name"],
|
||
execution?: AgentPreparedExecution,
|
||
plan?: AgentExecutionPlan,
|
||
): Record<string, unknown> | undefined {
|
||
const matchedTool =
|
||
execution?.toolCalls?.find((tool) => tool.name === toolName) ||
|
||
plan?.toolCalls?.find((tool) => tool.name === toolName);
|
||
|
||
if (!matchedTool?.args || typeof matchedTool.args !== "object") {
|
||
return undefined;
|
||
}
|
||
|
||
return matchedTool.args;
|
||
}
|
||
|
||
export function getDirectAgentToolSuccessMessage(
|
||
capability: DirectAgentToolCapability,
|
||
): string {
|
||
switch (capability) {
|
||
case "analyze":
|
||
return "已完成当前目标图分析，结果如下：";
|
||
case "generic_edit":
|
||
return "已完成图片编辑，结果如下：";
|
||
case "remove_background":
|
||
return "已完成抠图处理，结果如下：";
|
||
case "enhance":
|
||
return "已完成图像增强，结果如下：";
|
||
case "resize":
|
||
return "已完成尺寸调整，结果如下：";
|
||
case "extend":
|
||
return "已完成图像拓展，结果如下：";
|
||
case "box_cutout":
|
||
return "已完成框选抠图，结果如下：";
|
||
case "generate_image":
|
||
return "已完成图片生成，结果如下：";
|
||
case "generate_video":
|
||
return "已完成视频生成，结果如下：";
|
||
default:
|
||
return "已完成处理，结果如下：";
|
||
}
|
||
}
|
||
|
||
export function getDirectAgentToolTargetMissingMessage(
|
||
capability: DirectAgentToolCapability,
|
||
): string {
|
||
switch (capability) {
|
||
case "analyze":
|
||
return "请先上传或选中一张图片，我才能继续帮你分析。";
|
||
case "generic_edit":
|
||
return "请先上传或选中一张图片，我才能继续帮你修改。";
|
||
case "remove_background":
|
||
return "请先上传或选中一张图片，我才能继续帮你执行抠图。";
|
||
case "enhance":
|
||
return "请先上传或选中一张图片，我才能继续帮你做图像增强。";
|
||
case "resize":
|
||
return "请先上传或选中一张图片，我才能继续帮你调整尺寸。";
|
||
case "extend":
|
||
return "请先上传或选中一张图片，我才能继续帮你做图像拓展。";
|
||
case "box_cutout":
|
||
return "请先选中一张图片并提供框选区域，我才能继续帮你做框选抠图。";
|
||
case "generate_image":
|
||
return "请先告诉我你想生成什么，或者提供参考图。";
|
||
case "generate_video":
|
||
return "请先告诉我你想生成什么视频，必要时再提供参考图。";
|
||
default:
|
||
return "请先提供要处理的图片。";
|
||
}
|
||
}
|
||
|
||
export function getDirectAgentToolErrorMessage(
|
||
capability: DirectAgentToolCapability,
|
||
error: unknown,
|
||
): string {
|
||
const errorMessage =
|
||
error instanceof Error ? String(error.message || "").trim() : String(error || "").trim();
|
||
const errorDetail =
|
||
error && typeof error === "object" && "detail" in error
|
||
? String((error as { detail?: unknown }).detail || "").trim()
|
||
: "";
|
||
const detail = errorDetail || errorMessage || "请稍后重试。";
|
||
|
||
switch (capability) {
|
||
case "analyze":
|
||
return `图片分析暂时失败：${detail}`;
|
||
case "generic_edit":
|
||
return `图片编辑暂时失败：${detail}`;
|
||
case "remove_background":
|
||
return `抠图暂时失败：${detail}`;
|
||
case "enhance":
|
||
return `图像增强暂时失败：${detail}`;
|
||
case "resize":
|
||
return `尺寸调整暂时失败：${detail}`;
|
||
case "extend":
|
||
return `图像拓展暂时失败：${detail}`;
|
||
case "box_cutout":
|
||
return `框选抠图暂时失败：${detail}`;
|
||
case "generate_image":
|
||
return `图片生成暂时失败：${detail}`;
|
||
case "generate_video":
|
||
return `视频生成暂时失败：${detail}`;
|
||
default:
|
||
return `处理暂时失败：${detail}`;
|
||
}
|
||
}
|
||
|
||
export function pickAgentSourceImageUrls(params: {
|
||
sourcePreferences: DirectAgentSourcePreference[];
|
||
requestResolvedMediaInput?: ResolvedChatMediaInput | null;
|
||
requestTargetResolution?: ResolvedTargetAssets | null;
|
||
requestWorkingState?: DirectAgentWorkingStateSnapshot | null;
|
||
}): string[] {
|
||
const preferredBuckets: string[][] = [];
|
||
const sourcePreferences = params.sourcePreferences || [];
|
||
|
||
if (sourcePreferences.includes("selected_canvas")) {
|
||
preferredBuckets.push(params.requestResolvedMediaInput?.selectedImageUrls || []);
|
||
}
|
||
if (sourcePreferences.includes("latest_result")) {
|
||
preferredBuckets.push(params.requestWorkingState?.lastGeneratedImageUrls || []);
|
||
preferredBuckets.push(params.requestResolvedMediaInput?.historyAssistantImageUrls || []);
|
||
}
|
||
if (sourcePreferences.includes("original_upload")) {
|
||
preferredBuckets.push(params.requestResolvedMediaInput?.currentMessageImageUrls || []);
|
||
preferredBuckets.push(params.requestWorkingState?.lastUploadedImageUrls || []);
|
||
preferredBuckets.push(params.requestResolvedMediaInput?.historyUserImageUrls || []);
|
||
}
|
||
|
||
preferredBuckets.push(
|
||
params.requestTargetResolution?.targetImageUrls || [],
|
||
params.requestResolvedMediaInput?.selectedImageUrls || [],
|
||
params.requestResolvedMediaInput?.currentMessageImageUrls || [],
|
||
params.requestWorkingState?.lastResolvedTargetImageUrls || [],
|
||
params.requestWorkingState?.lastGeneratedImageUrls || [],
|
||
params.requestResolvedMediaInput?.historyAssistantImageUrls || [],
|
||
params.requestResolvedMediaInput?.historyUserImageUrls || [],
|
||
);
|
||
|
||
return Array.from(
|
||
new Set(
|
||
preferredBuckets.flatMap((bucket) =>
|
||
(bucket || []).map((item) => String(item || "").trim()).filter(Boolean),
|
||
),
|
||
),
|
||
);
|
||
}
|