2504 lines
85 KiB
TypeScript
2504 lines
85 KiB
TypeScript
import { Annotation, StateGraph, START, END } from "@langchain/langgraph";
|
||
import { generateText } from "ai";
|
||
import sharp from "sharp";
|
||
import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat";
|
||
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
|
||
import type { RouteResult } from "@/lib/ai/model-router";
|
||
import type { AgentSkillRouteResult } from "@/lib/ai/skill-router";
|
||
import type { CanvasAgentContext } from "@/types/canvas/agent-protocol";
|
||
import {
|
||
getEnabledAgentToolDefinitionsForContext,
|
||
type AgentProjectToolDefinition,
|
||
type AgentProjectToolName,
|
||
} from "@/lib/ai/tools-registry";
|
||
import {
|
||
runHumanConfirmationGate,
|
||
type HumanConfirmationGateDecision,
|
||
} from "@/lib/ai/runtime/middlewares/human-confirmation-gate";
|
||
import {
|
||
normalizeAgentThreadState,
|
||
type AgentThreadState,
|
||
} from "@/lib/ai/runtime/agent-thread-state";
|
||
import {
|
||
buildAgentPlanningBriefMarkdown,
|
||
isAgentPlanningBriefMarkdown,
|
||
} from "@/lib/ai/runtime/agent-planning-brief";
|
||
import {
|
||
getAiModelAgentCodingModelId,
|
||
isAiModelMultimodal,
|
||
} from "@/lib/db/ai-model-flags";
|
||
import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options";
|
||
|
||
interface HistoryMessage {
|
||
role: "user" | "assistant";
|
||
content: string;
|
||
}
|
||
|
||
export interface AgentRecommendedImageParams {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
negativePrompt?: string;
|
||
promptExtend?: boolean;
|
||
}
|
||
|
||
export interface AgentRecommendedVideoParams {
|
||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||
resolution?: "480p" | "720p" | "1080p";
|
||
ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive";
|
||
duration?: number;
|
||
generateAudio?: boolean;
|
||
cameraMove?: string;
|
||
}
|
||
|
||
export interface AgentVideoStoryboardShot {
|
||
shot: number;
|
||
title: string;
|
||
duration: number;
|
||
description: string;
|
||
prompt: string;
|
||
camera?: string;
|
||
}
|
||
|
||
export interface AgentVideoPlan {
|
||
script: string;
|
||
storyboard: AgentVideoStoryboardShot[];
|
||
finalPrompt: string;
|
||
negativePrompt?: string;
|
||
recommendedVideoParams?: AgentRecommendedVideoParams;
|
||
}
|
||
|
||
export type AgentTargetResolutionMode =
|
||
| "explicit_selection"
|
||
| "current_upload"
|
||
| "text_reference"
|
||
| "working_memory"
|
||
| "latest_result"
|
||
| "clarification"
|
||
| "none";
|
||
|
||
export type AgentPreferredCapability =
|
||
| "respond"
|
||
| "analyze"
|
||
| "generic_edit"
|
||
| "remove_background"
|
||
| "remove_watermark"
|
||
| "enhance"
|
||
| "extend"
|
||
| "resize"
|
||
| "angle_variant"
|
||
| "local_edit"
|
||
| "box_cutout"
|
||
| "generate_image"
|
||
| "generate_video"
|
||
| "publish_canvas";
|
||
|
||
export type AgentPlanNextAction =
|
||
| "respond"
|
||
| "clarify"
|
||
| "analyze_image"
|
||
| "generate_image"
|
||
| "edit_image"
|
||
| "generate_video";
|
||
|
||
export type AgentPlanResponseMode =
|
||
| "direct_answer"
|
||
| "tool_then_answer"
|
||
| "tool_then_execute"
|
||
| "clarify";
|
||
|
||
export interface AgentPlannedToolCall {
|
||
name: AgentProjectToolName;
|
||
reason?: string;
|
||
args?: Record<string, unknown>;
|
||
}
|
||
|
||
export interface AgentExecutionPlan {
|
||
goal: string;
|
||
nextAction: AgentPlanNextAction;
|
||
intent: AnalysisResult["intent"];
|
||
subIntent?: string;
|
||
targetSource: AgentTargetResolutionMode;
|
||
preferredCapability: AgentPreferredCapability;
|
||
responseMode: AgentPlanResponseMode;
|
||
shouldUseHistoryImage: boolean;
|
||
shouldUseStructuredReply: boolean;
|
||
confidence: number;
|
||
toolCalls: AgentPlannedToolCall[];
|
||
}
|
||
|
||
export interface AgentPreparedExecution {
|
||
mode: AgentPlanNextAction;
|
||
routeIntent: AnalysisResult["intent"];
|
||
routeSubIntent?: string;
|
||
targetSource: AgentTargetResolutionMode;
|
||
preferredCapability: AgentPreferredCapability;
|
||
shouldUseStructuredReply: boolean;
|
||
shouldOptimizePrompt: boolean;
|
||
shouldApplyImagePipeline: boolean;
|
||
shouldApplyVideoPipeline: boolean;
|
||
shouldStayConversational: boolean;
|
||
shouldPublishToCanvas: boolean;
|
||
toolCalls: AgentPlannedToolCall[];
|
||
availableTools: AgentProjectToolName[];
|
||
}
|
||
|
||
export interface AutoAgentInput {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
assetIds?: string[];
|
||
imageFromHistory?: boolean;
|
||
targetResolutionMode?: AgentTargetResolutionMode;
|
||
mediaSource?: "current_message" | "canvas_selection" | "history" | "none";
|
||
hasCanvasSelection?: boolean;
|
||
selectedImageCount?: number;
|
||
canvasAgentContext?: CanvasAgentContext | null;
|
||
selectedSlotCount?: number;
|
||
hasSelectedAgentSlot?: boolean;
|
||
historyImageCount?: number;
|
||
currentUploadImageCount?: number;
|
||
historyMessages?: HistoryMessage[];
|
||
context?: string;
|
||
preferredModelId?: string;
|
||
threadState?: AgentThreadState | null;
|
||
}
|
||
|
||
export interface AutoAgentResult {
|
||
analysis?: AnalysisResult;
|
||
plan?: AgentExecutionPlan;
|
||
execution?: AgentPreparedExecution;
|
||
friendlyMessage?: string;
|
||
optimizedPrompt?: string;
|
||
recommendedImageParams?: AgentRecommendedImageParams;
|
||
videoPlan?: AgentVideoPlan;
|
||
route?: RouteResult;
|
||
threadState?: AgentThreadState | null;
|
||
workflowDecision?: HumanConfirmationGateDecision;
|
||
}
|
||
|
||
const AutoAgentState = Annotation.Root({
|
||
text: Annotation<string>,
|
||
hasImage: Annotation<boolean>,
|
||
imageUrls: Annotation<string[]>,
|
||
assetIds: Annotation<string[]>,
|
||
imageFromHistory: Annotation<boolean>,
|
||
targetResolutionMode: Annotation<AgentTargetResolutionMode>,
|
||
mediaSource: Annotation<"current_message" | "canvas_selection" | "history" | "none">,
|
||
hasCanvasSelection: Annotation<boolean>,
|
||
selectedImageCount: Annotation<number>,
|
||
canvasAgentContext: Annotation<CanvasAgentContext | null>,
|
||
selectedSlotCount: Annotation<number>,
|
||
hasSelectedAgentSlot: Annotation<boolean>,
|
||
historyImageCount: Annotation<number>,
|
||
currentUploadImageCount: Annotation<number>,
|
||
historyMessages: Annotation<HistoryMessage[]>,
|
||
context: Annotation<string | null>,
|
||
preferredModelId: Annotation<string | null>,
|
||
threadState: Annotation<AgentThreadState | null>,
|
||
analysis: Annotation<AnalysisResult | null>,
|
||
availableTools: Annotation<AgentProjectToolDefinition[]>,
|
||
plan: Annotation<AgentExecutionPlan | null>,
|
||
execution: Annotation<AgentPreparedExecution | null>,
|
||
friendlyMessage: Annotation<string | null>,
|
||
optimizedPrompt: Annotation<string | null>,
|
||
recommendedImageParams: Annotation<AgentRecommendedImageParams | null>,
|
||
videoPlan: Annotation<AgentVideoPlan | null>,
|
||
route: Annotation<RouteResult | null>,
|
||
workflowDecision: Annotation<HumanConfirmationGateDecision | null>,
|
||
});
|
||
|
||
type AutoAgentStateType = typeof AutoAgentState.State;
|
||
|
||
function compactText(text: string, max = 48) {
|
||
const normalized = text.replace(/\s+/g, " ").trim();
|
||
if (normalized.length <= max) return normalized;
|
||
return `${normalized.slice(0, max)}...`;
|
||
}
|
||
|
||
function createPlanGoalSummary(text: string, hasImage: boolean): string {
|
||
const compact = compactText(text || "", 72);
|
||
if (compact) return compact;
|
||
return hasImage ? "基于当前图片继续处理" : "继续当前对话";
|
||
}
|
||
|
||
function describeTargetSource(mode: AgentTargetResolutionMode): string {
|
||
switch (mode) {
|
||
case "explicit_selection":
|
||
return "画布选中图";
|
||
case "current_upload":
|
||
return "当前上传图";
|
||
case "text_reference":
|
||
return "用户文本指定图";
|
||
case "working_memory":
|
||
return "会话工作记忆图";
|
||
case "latest_result":
|
||
return "最近生成结果图";
|
||
case "clarification":
|
||
return "目标待澄清";
|
||
case "none":
|
||
default:
|
||
return "无明确图像目标";
|
||
}
|
||
}
|
||
|
||
function inferPreferredCapability(
|
||
analysis: AnalysisResult,
|
||
toolCalls?: AgentPlannedToolCall[],
|
||
): AgentPreferredCapability {
|
||
const firstTool =
|
||
toolCalls?.find((tool) =>
|
||
tool.name !== "resolveTargetImage" &&
|
||
tool.name !== "useSelectedCanvasImage" &&
|
||
tool.name !== "useLatestResultImage" &&
|
||
tool.name !== "useOriginalUploadImage"
|
||
)?.name || toolCalls?.[0]?.name;
|
||
if (firstTool === "removeBackground") return "remove_background";
|
||
if (firstTool === "removeWatermark") return "remove_watermark";
|
||
if (firstTool === "enhanceImage") return "enhance";
|
||
if (firstTool === "extendImage") return "extend";
|
||
if (firstTool === "resizeImage") return "resize";
|
||
if (firstTool === "angleEditImage") return "angle_variant";
|
||
if (firstTool === "localEditImage") return "local_edit";
|
||
if (firstTool === "boxCutout") return "box_cutout";
|
||
if (firstTool === "publishToCanvas" || firstTool === "publishResultToCanvas") return "publish_canvas";
|
||
if (firstTool === "generateVideo") return "generate_video";
|
||
if (firstTool === "generateImage" || firstTool === "generateNewImage") return "generate_image";
|
||
if (firstTool === "analyzeImage" || firstTool === "analyzeTargetImage") return "analyze";
|
||
|
||
if (analysis.intent === "video_gen") return "generate_video";
|
||
if (analysis.intent === "image_gen") return "generate_image";
|
||
if (analysis.intent === "vision") return "analyze";
|
||
if (analysis.intent === "chat") return "respond";
|
||
|
||
switch (analysis.subIntent) {
|
||
case "remove_background":
|
||
return "remove_background";
|
||
case "remove_watermark":
|
||
return "remove_watermark";
|
||
case "enhance":
|
||
return "enhance";
|
||
case "extend":
|
||
return "extend";
|
||
case "resize":
|
||
return "resize";
|
||
case "angle_variant":
|
||
case "generate_angle_variant":
|
||
return "angle_variant";
|
||
case "local_edit":
|
||
case "inpaint_region":
|
||
return "local_edit";
|
||
default:
|
||
return "generic_edit";
|
||
}
|
||
}
|
||
|
||
function defaultToolForAction(action: AgentPlanNextAction): AgentProjectToolName {
|
||
switch (action) {
|
||
case "analyze_image":
|
||
return "analyzeTargetImage";
|
||
case "edit_image":
|
||
return "editTargetImage";
|
||
case "generate_image":
|
||
return "generateNewImage";
|
||
case "generate_video":
|
||
return "generateVideo";
|
||
case "clarify":
|
||
case "respond":
|
||
default:
|
||
return "respondDirect";
|
||
}
|
||
}
|
||
|
||
function candidateToolsForAction(action: AgentPlanNextAction): AgentProjectToolName[] {
|
||
switch (action) {
|
||
case "analyze_image":
|
||
return ["analyzeTargetImage", "analyzeImage"];
|
||
case "edit_image":
|
||
return ["editTargetImage", "editImage"];
|
||
case "generate_image":
|
||
return ["generateNewImage", "generateImage"];
|
||
case "generate_video":
|
||
return ["generateVideo"];
|
||
case "clarify":
|
||
case "respond":
|
||
default:
|
||
return ["respondDirect"];
|
||
}
|
||
}
|
||
|
||
function pickAvailableToolForAction(
|
||
action: AgentPlanNextAction,
|
||
availableNames: Set<AgentProjectToolName>,
|
||
): AgentProjectToolName | undefined {
|
||
return candidateToolsForAction(action).find((name) => availableNames.has(name));
|
||
}
|
||
|
||
function mapIntentToAction(intent: AnalysisResult["intent"]): AgentPlanNextAction {
|
||
switch (intent) {
|
||
case "vision":
|
||
return "analyze_image";
|
||
case "image_edit":
|
||
return "edit_image";
|
||
case "image_gen":
|
||
return "generate_image";
|
||
case "video_gen":
|
||
return "generate_video";
|
||
case "chat":
|
||
default:
|
||
return "respond";
|
||
}
|
||
}
|
||
|
||
function mapActionToIntent(
|
||
action: AgentPlanNextAction,
|
||
fallbackIntent: AnalysisResult["intent"],
|
||
): AnalysisResult["intent"] {
|
||
switch (action) {
|
||
case "analyze_image":
|
||
return "vision";
|
||
case "edit_image":
|
||
return "image_edit";
|
||
case "generate_image":
|
||
return "image_gen";
|
||
case "generate_video":
|
||
return "video_gen";
|
||
case "clarify":
|
||
case "respond":
|
||
return fallbackIntent === "vision" ? "vision" : "chat";
|
||
default:
|
||
return fallbackIntent;
|
||
}
|
||
}
|
||
|
||
function defaultResponseMode(action: AgentPlanNextAction): AgentPlanResponseMode {
|
||
switch (action) {
|
||
case "clarify":
|
||
return "clarify";
|
||
case "respond":
|
||
return "direct_answer";
|
||
case "analyze_image":
|
||
return "tool_then_answer";
|
||
case "edit_image":
|
||
case "generate_image":
|
||
case "generate_video":
|
||
return "tool_then_execute";
|
||
default:
|
||
return "direct_answer";
|
||
}
|
||
}
|
||
|
||
function shouldUseStructuredReply(action: AgentPlanNextAction): boolean {
|
||
return action === "edit_image" || action === "generate_image" || action === "generate_video";
|
||
}
|
||
|
||
function sanitizePlannedToolCalls(
|
||
rawToolCalls: unknown,
|
||
availableTools: AgentProjectToolDefinition[],
|
||
fallbackAction: AgentPlanNextAction,
|
||
options?: {
|
||
targetSource?: AgentTargetResolutionMode;
|
||
},
|
||
): AgentPlannedToolCall[] {
|
||
const availableNames = new Set(availableTools.map((tool) => tool.name));
|
||
const list = Array.isArray(rawToolCalls) ? rawToolCalls : [];
|
||
const sanitized = list
|
||
.map((item) => {
|
||
if (!item || typeof item !== "object") return null;
|
||
const rawName = String((item as { name?: unknown }).name || "").trim() as AgentProjectToolName;
|
||
if (!availableNames.has(rawName)) return null;
|
||
const args = (item as { args?: unknown }).args;
|
||
return {
|
||
name: rawName,
|
||
reason: String((item as { reason?: unknown }).reason || "").trim() || undefined,
|
||
args: args && typeof args === "object" && !Array.isArray(args) ? (args as Record<string, unknown>) : undefined,
|
||
} satisfies AgentPlannedToolCall;
|
||
})
|
||
.filter(Boolean) as AgentPlannedToolCall[];
|
||
|
||
const shouldInjectTargetResolver =
|
||
(fallbackAction === "analyze_image" || fallbackAction === "edit_image") &&
|
||
options?.targetSource &&
|
||
options.targetSource !== "none" &&
|
||
options.targetSource !== "clarification" &&
|
||
availableNames.has("resolveTargetImage") &&
|
||
!sanitized.some((tool) =>
|
||
tool.name === "resolveTargetImage" ||
|
||
tool.name === "useSelectedCanvasImage" ||
|
||
tool.name === "useLatestResultImage" ||
|
||
tool.name === "useOriginalUploadImage"
|
||
);
|
||
|
||
if (shouldInjectTargetResolver) {
|
||
const source =
|
||
options?.targetSource === "explicit_selection"
|
||
? "selected_canvas"
|
||
: options?.targetSource === "current_upload"
|
||
? "original_upload"
|
||
: options?.targetSource === "latest_result" || options?.targetSource === "working_memory"
|
||
? "latest_result"
|
||
: "auto";
|
||
sanitized.unshift({
|
||
name: "resolveTargetImage",
|
||
reason: "先锁定本轮要操作的目标图，再执行分析或编辑。",
|
||
args: { source },
|
||
});
|
||
}
|
||
|
||
if (sanitized.length > 0) {
|
||
const hasPrimaryActionTool = sanitized.some((tool) =>
|
||
candidateToolsForAction(fallbackAction).includes(tool.name),
|
||
);
|
||
const fallbackTool = pickAvailableToolForAction(fallbackAction, availableNames);
|
||
if (
|
||
!hasPrimaryActionTool &&
|
||
fallbackTool &&
|
||
fallbackAction !== "respond" &&
|
||
fallbackAction !== "clarify"
|
||
) {
|
||
sanitized.push({
|
||
name: fallbackTool,
|
||
reason: "补齐本轮主执行工具，避免只解析目标图后停住。",
|
||
});
|
||
}
|
||
return sanitized;
|
||
}
|
||
|
||
const fallbackTool = pickAvailableToolForAction(fallbackAction, availableNames) || defaultToolForAction(fallbackAction);
|
||
if (
|
||
(fallbackAction === "analyze_image" || fallbackAction === "edit_image") &&
|
||
options?.targetSource &&
|
||
options.targetSource !== "none" &&
|
||
options.targetSource !== "clarification" &&
|
||
availableNames.has("resolveTargetImage")
|
||
) {
|
||
const source =
|
||
options.targetSource === "explicit_selection"
|
||
? "selected_canvas"
|
||
: options.targetSource === "current_upload"
|
||
? "original_upload"
|
||
: options.targetSource === "latest_result" || options.targetSource === "working_memory"
|
||
? "latest_result"
|
||
: "auto";
|
||
if (availableNames.has(fallbackTool)) {
|
||
return [
|
||
{
|
||
name: "resolveTargetImage",
|
||
reason: "先确定要操作的目标图。",
|
||
args: { source },
|
||
},
|
||
{ name: fallbackTool },
|
||
];
|
||
}
|
||
}
|
||
if (availableNames.has(fallbackTool)) {
|
||
return [{ name: fallbackTool }];
|
||
}
|
||
if (availableNames.has("respondDirect")) {
|
||
return [{ name: "respondDirect" }];
|
||
}
|
||
return [];
|
||
}
|
||
|
||
function buildFallbackPlan(params: {
|
||
analysis: AnalysisResult;
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageFromHistory: boolean;
|
||
targetResolutionMode: AgentTargetResolutionMode;
|
||
availableTools: AgentProjectToolDefinition[];
|
||
}): AgentExecutionPlan {
|
||
const baseAction = params.analysis.needsClarification
|
||
? "clarify"
|
||
: mapIntentToAction(params.analysis.intent);
|
||
let toolCalls = sanitizePlannedToolCalls([], params.availableTools, baseAction, {
|
||
targetSource: params.targetResolutionMode,
|
||
});
|
||
if (
|
||
baseAction === "edit_image" &&
|
||
hasDirectAgentLocalEditIntent(params.text) &&
|
||
params.availableTools.some((tool) => tool.name === "localEditImage") &&
|
||
!toolCalls.some((tool) => tool.name === "localEditImage")
|
||
) {
|
||
toolCalls = [
|
||
...toolCalls.filter((tool) => tool.name !== "editTargetImage" && tool.name !== "editImage"),
|
||
{
|
||
name: "localEditImage",
|
||
reason: "用户要求按局部区域或批注执行编辑。",
|
||
},
|
||
];
|
||
}
|
||
return {
|
||
goal: createPlanGoalSummary(params.text, params.hasImage),
|
||
nextAction: baseAction,
|
||
intent: mapActionToIntent(baseAction, params.analysis.intent),
|
||
subIntent:
|
||
baseAction === "edit_image"
|
||
? hasDirectAgentLocalEditIntent(params.text)
|
||
? "local_edit"
|
||
: params.analysis.subIntent || "enhance"
|
||
: undefined,
|
||
targetSource: params.targetResolutionMode,
|
||
preferredCapability: inferPreferredCapability(params.analysis, toolCalls),
|
||
responseMode: defaultResponseMode(baseAction),
|
||
shouldUseHistoryImage: params.imageFromHistory,
|
||
shouldUseStructuredReply: shouldUseStructuredReply(baseAction),
|
||
confidence: Math.max(params.analysis.confidence || 0.6, 0.68),
|
||
toolCalls,
|
||
};
|
||
}
|
||
|
||
function alignAnalysisWithPlan(
|
||
analysis: AnalysisResult,
|
||
plan: AgentExecutionPlan,
|
||
): AnalysisResult {
|
||
const intent = plan.intent || mapActionToIntent(plan.nextAction, analysis.intent);
|
||
return {
|
||
...analysis,
|
||
intent,
|
||
subIntent: intent === "image_edit" ? plan.subIntent || analysis.subIntent : undefined,
|
||
needsClarification: plan.nextAction === "clarify",
|
||
clarifyingQuestions:
|
||
plan.nextAction === "clarify"
|
||
? analysis.clarifyingQuestions?.length
|
||
? analysis.clarifyingQuestions
|
||
: ["我先确认一下你希望我下一步执行什么。"]
|
||
: [],
|
||
};
|
||
}
|
||
|
||
function prepareExecutionFromPlan(params: {
|
||
plan: AgentExecutionPlan;
|
||
availableTools: AgentProjectToolDefinition[];
|
||
}): AgentPreparedExecution {
|
||
const availableToolNames = params.availableTools.map((tool) => tool.name);
|
||
const mode = params.plan.nextAction;
|
||
return {
|
||
mode,
|
||
routeIntent: params.plan.intent,
|
||
routeSubIntent: params.plan.subIntent,
|
||
targetSource: params.plan.targetSource,
|
||
preferredCapability: params.plan.preferredCapability,
|
||
shouldUseStructuredReply: params.plan.shouldUseStructuredReply,
|
||
shouldOptimizePrompt:
|
||
mode === "edit_image" || mode === "generate_image" || mode === "generate_video",
|
||
shouldApplyImagePipeline: mode === "edit_image" || mode === "generate_image",
|
||
shouldApplyVideoPipeline: mode === "generate_video",
|
||
shouldStayConversational: mode === "respond" || mode === "analyze_image" || mode === "clarify",
|
||
shouldPublishToCanvas: params.plan.toolCalls.some(
|
||
(tool) => tool.name === "publishToCanvas" || tool.name === "publishResultToCanvas"
|
||
),
|
||
toolCalls: params.plan.toolCalls,
|
||
availableTools: availableToolNames,
|
||
};
|
||
}
|
||
|
||
function buildWorkflowGateBypassPayload(
|
||
state: AutoAgentStateType,
|
||
decision: HumanConfirmationGateDecision,
|
||
) {
|
||
const analysis: AnalysisResult = {
|
||
intent: "chat",
|
||
confidence: 0.98,
|
||
needsClarification: decision.action === "awaiting_confirmation",
|
||
clarifyingQuestions:
|
||
decision.action === "awaiting_confirmation"
|
||
? ["请先确认或修改当前待确认流程，我再继续执行。"]
|
||
: [],
|
||
conversationalReply: decision.friendlyMessage,
|
||
};
|
||
|
||
const plan: AgentExecutionPlan = {
|
||
goal: createPlanGoalSummary(state.text, state.hasImage),
|
||
nextAction: "respond",
|
||
intent: "chat",
|
||
targetSource: state.targetResolutionMode,
|
||
preferredCapability: "respond",
|
||
responseMode: "direct_answer",
|
||
shouldUseHistoryImage: false,
|
||
shouldUseStructuredReply: false,
|
||
confidence: 0.98,
|
||
toolCalls: [
|
||
{
|
||
name: "respondDirect",
|
||
reason: "当前工作流处于人工确认门控阶段，先处理用户确认或修改意图。",
|
||
},
|
||
],
|
||
};
|
||
|
||
const execution = prepareExecutionFromPlan({
|
||
plan,
|
||
availableTools: [],
|
||
});
|
||
|
||
return {
|
||
analysis,
|
||
plan,
|
||
execution,
|
||
friendlyMessage: decision.friendlyMessage || null,
|
||
workflowDecision: decision,
|
||
};
|
||
}
|
||
|
||
function getEnvInt(name: string, fallback: number, min: number, max: number): number {
|
||
const raw = process.env[name];
|
||
const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
|
||
if (!Number.isFinite(parsed)) return fallback;
|
||
return Math.max(min, Math.min(max, parsed));
|
||
}
|
||
|
||
async function withTimeoutFallback<T>(
|
||
promise: Promise<T>,
|
||
timeoutMs: number,
|
||
fallbackValue: T,
|
||
): Promise<T> {
|
||
let timeoutId: NodeJS.Timeout | undefined;
|
||
try {
|
||
return await Promise.race([
|
||
promise,
|
||
new Promise<T>((resolve) => {
|
||
timeoutId = setTimeout(() => resolve(fallbackValue), timeoutMs);
|
||
}),
|
||
]);
|
||
} finally {
|
||
if (timeoutId) clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
const AGENT_VISION_MAX_IMAGES = 4;
|
||
const AGENT_VISION_MAX_SIDE = 500;
|
||
const AGENT_OPERATION_TIMEOUT_MS = getEnvInt(
|
||
"AGENT_OPERATION_TIMEOUT_MS",
|
||
18000,
|
||
8000,
|
||
120000,
|
||
);
|
||
const AGENT_VISION_FETCH_TIMEOUT_MS = getEnvInt(
|
||
"AGENT_VISION_FETCH_TIMEOUT_MS",
|
||
12000,
|
||
3000,
|
||
60000,
|
||
);
|
||
|
||
function isHttpUrl(value: string) {
|
||
return /^https?:\/\//i.test(value);
|
||
}
|
||
|
||
async function fetchWithTimeout(url: string, timeoutMs: number): Promise<Response> {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||
try {
|
||
return await fetch(url, {
|
||
signal: controller.signal,
|
||
cache: "no-store",
|
||
headers: {
|
||
"user-agent": "ai-chatbot-auto-agent/1.0",
|
||
},
|
||
});
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
async function toAgentVisionDataUrl(imageUrl: string): Promise<string | null> {
|
||
try {
|
||
const response = await fetchWithTimeout(imageUrl, AGENT_VISION_FETCH_TIMEOUT_MS);
|
||
if (!response.ok) {
|
||
console.warn(
|
||
`[Auto Agent] Failed to fetch image for inline vision payload (${response.status}): ${imageUrl}`,
|
||
);
|
||
return null;
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type") || "";
|
||
if (!contentType.startsWith("image/")) {
|
||
console.warn(`[Auto Agent] Skip non-image content-type for vision payload: ${contentType}`);
|
||
return null;
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const optimized = await sharp(Buffer.from(arrayBuffer))
|
||
.rotate()
|
||
.resize({
|
||
width: AGENT_VISION_MAX_SIDE,
|
||
height: AGENT_VISION_MAX_SIDE,
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({
|
||
quality: 80,
|
||
mozjpeg: true,
|
||
})
|
||
.toBuffer();
|
||
|
||
return `data:image/jpeg;base64,${optimized.toString("base64")}`;
|
||
} catch (error) {
|
||
console.warn("[Auto Agent] Failed to convert image URL to inline base64:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function buildAgentVisionInputs(
|
||
imageUrls: string[],
|
||
options?: {
|
||
preferRemoteUrls?: boolean;
|
||
},
|
||
) {
|
||
if (imageUrls.length === 0) return [] as Array<{ type: "image"; image: string }>;
|
||
if (options?.preferRemoteUrls) {
|
||
return imageUrls.map((url) => ({
|
||
type: "image" as const,
|
||
image: url,
|
||
}));
|
||
}
|
||
|
||
const convertedResults = await Promise.all(imageUrls.map((url) => toAgentVisionDataUrl(url)));
|
||
const inlineCount = convertedResults.filter(Boolean).length;
|
||
console.log(
|
||
`[Auto Agent] Prepared vision images: total=${imageUrls.length}, inline=${inlineCount}, remote=${imageUrls.length - inlineCount}, maxSide=${AGENT_VISION_MAX_SIDE}`,
|
||
);
|
||
|
||
return imageUrls.map((originalUrl, index) => ({
|
||
type: "image" as const,
|
||
image: convertedResults[index] || originalUrl,
|
||
}));
|
||
}
|
||
|
||
function normalizeAspectRatio(value?: string): string | undefined {
|
||
if (!value) return undefined;
|
||
const normalized = value.trim().toLowerCase();
|
||
if (normalized === "auto") return "auto";
|
||
const raw = value.trim();
|
||
const allowed = new Set([
|
||
"1:1",
|
||
"2:3",
|
||
"3:2",
|
||
"3:4",
|
||
"4:3",
|
||
"4:5",
|
||
"5:4",
|
||
"9:16",
|
||
"16:9",
|
||
"21:9",
|
||
]);
|
||
return allowed.has(raw) ? raw : undefined;
|
||
}
|
||
|
||
function parseChineseCountToken(token?: string): number | undefined {
|
||
const normalized = String(token || "").trim().toLowerCase();
|
||
if (!normalized) return undefined;
|
||
const numeric = Number(normalized);
|
||
if (Number.isFinite(numeric) && numeric >= 1) return Math.round(numeric);
|
||
const map: Record<string, number> = {
|
||
一: 1,
|
||
二: 2,
|
||
两: 2,
|
||
三: 3,
|
||
四: 4,
|
||
五: 5,
|
||
六: 6,
|
||
七: 7,
|
||
八: 8,
|
||
九: 9,
|
||
十: 10,
|
||
};
|
||
return map[normalized];
|
||
}
|
||
|
||
function inferExplicitImageCountFromText(text: string): number | undefined {
|
||
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
|
||
const match =
|
||
normalized.match(/([1-9]\d?|一|二|两|三|四|五|六|七|八|九|十)张/) ||
|
||
normalized.match(/([1-9]\d?|一|二|两|三|四|五|六|七|八|九|十)(?:个|组)?(?:版本|方案|变体)/);
|
||
const parsed = parseChineseCountToken(match?.[1]);
|
||
if (parsed && parsed >= 1) return Math.min(parsed, 12);
|
||
if (/多张/.test(normalized)) return 3;
|
||
return undefined;
|
||
}
|
||
|
||
function inferExplicitAspectRatioFromText(text: string): string | undefined {
|
||
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
|
||
const explicitRatio = normalized.match(/(21:9|16:9|9:16|5:4|4:5|4:3|3:4|3:2|2:3|1:1)/)?.[1];
|
||
if (explicitRatio) return explicitRatio;
|
||
if (/竖图|竖版|竖屏|手机壁纸|9比16/.test(normalized)) return "9:16";
|
||
if (/横图|横版|横屏|宽屏|banner|16比9/.test(normalized)) return "16:9";
|
||
if (/方图|方形|头像|logo|电商主图|1比1/.test(normalized)) return "1:1";
|
||
return undefined;
|
||
}
|
||
|
||
function inferExplicitImageSizeFromText(text: string): "1K" | "2K" | "4K" | undefined {
|
||
const normalized = String(text || "").toLowerCase();
|
||
if (/(^|[^0-9a-z])4\s*k([^0-9a-z]|$)/.test(normalized)) return "4K";
|
||
if (/(^|[^0-9a-z])2\s*k([^0-9a-z]|$)/.test(normalized)) return "2K";
|
||
if (/(^|[^0-9a-z])1\s*k([^0-9a-z]|$)/.test(normalized)) return "1K";
|
||
return undefined;
|
||
}
|
||
|
||
function extractExplicitImageParamsFromText(
|
||
text: string,
|
||
): Pick<AgentRecommendedImageParams, "aspectRatio" | "imageCount" | "imageSize"> {
|
||
const output: Pick<AgentRecommendedImageParams, "aspectRatio" | "imageCount" | "imageSize"> = {};
|
||
const aspectRatio = inferExplicitAspectRatioFromText(text);
|
||
const imageCount = inferExplicitImageCountFromText(text);
|
||
const imageSize = inferExplicitImageSizeFromText(text);
|
||
if (aspectRatio) output.aspectRatio = aspectRatio;
|
||
if (imageCount) output.imageCount = imageCount;
|
||
if (imageSize) output.imageSize = imageSize;
|
||
return output;
|
||
}
|
||
|
||
function normalizeImageSize(value?: string): string | undefined {
|
||
if (!value) return undefined;
|
||
const normalized = value.trim().toUpperCase();
|
||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
|
||
return normalized;
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
function normalizeVideoMode(value?: string): AgentRecommendedVideoParams["mode"] | undefined {
|
||
if (!value) return undefined;
|
||
if (value === "text-to-video") return "text-to-video";
|
||
if (value === "first-frame") return "first-frame";
|
||
if (value === "first-last-frame") return "first-last-frame";
|
||
if (value === "reference-images") return "reference-images";
|
||
return undefined;
|
||
}
|
||
|
||
function normalizeVideoResolution(
|
||
value?: string,
|
||
): AgentRecommendedVideoParams["resolution"] | undefined {
|
||
if (!value) return undefined;
|
||
const normalized = value.toLowerCase();
|
||
if (normalized === "480p") return "480p";
|
||
if (normalized === "720p") return "720p";
|
||
if (normalized === "1080p") return "1080p";
|
||
return undefined;
|
||
}
|
||
|
||
function normalizeVideoRatio(value?: string): AgentRecommendedVideoParams["ratio"] | undefined {
|
||
if (!value) return undefined;
|
||
const normalized = value.trim();
|
||
const allowed = new Set(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"]);
|
||
return allowed.has(normalized) ? (normalized as AgentRecommendedVideoParams["ratio"]) : undefined;
|
||
}
|
||
|
||
function normalizeVideoDuration(value?: number): number | undefined {
|
||
if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
|
||
return Math.min(15, Math.max(3, Math.round(value)));
|
||
}
|
||
|
||
function sanitizeRecommendedVideoParams(
|
||
params: AgentRecommendedVideoParams | undefined,
|
||
): AgentRecommendedVideoParams | undefined {
|
||
if (!params) return undefined;
|
||
const safe: AgentRecommendedVideoParams = {};
|
||
const mode = normalizeVideoMode(params.mode);
|
||
const resolution = normalizeVideoResolution(params.resolution);
|
||
const ratio = normalizeVideoRatio(params.ratio);
|
||
const duration = normalizeVideoDuration(params.duration);
|
||
|
||
if (mode) safe.mode = mode;
|
||
if (resolution) safe.resolution = resolution;
|
||
if (ratio) safe.ratio = ratio;
|
||
if (duration) safe.duration = duration;
|
||
if (typeof params.generateAudio === "boolean") safe.generateAudio = params.generateAudio;
|
||
if (typeof params.cameraMove === "string" && params.cameraMove.trim()) {
|
||
safe.cameraMove = params.cameraMove.trim().slice(0, 32);
|
||
}
|
||
|
||
return Object.keys(safe).length > 0 ? safe : undefined;
|
||
}
|
||
|
||
function inferVideoParamsByText(text: string, hasImage: boolean): AgentRecommendedVideoParams {
|
||
const normalized = text.toLowerCase();
|
||
const durationMatch = normalized.match(/(\d{1,2})\s*秒/);
|
||
const duration = durationMatch ? normalizeVideoDuration(Number(durationMatch[1])) : undefined;
|
||
|
||
const mode = /首尾帧|首帧.*尾帧|结尾帧|尾帧/.test(normalized)
|
||
? "first-last-frame"
|
||
: hasImage
|
||
? "first-frame"
|
||
: "text-to-video";
|
||
|
||
const ratio = /竖屏|手机|9:16|竖版/.test(normalized)
|
||
? "9:16"
|
||
: /方形|方图|1:1/.test(normalized)
|
||
? "1:1"
|
||
: /超宽|21:9|电影感/.test(normalized)
|
||
? "21:9"
|
||
: "16:9";
|
||
|
||
const generateAudio = /配音|旁白|音效|有声|带声音/.test(normalized);
|
||
return {
|
||
mode,
|
||
ratio,
|
||
duration: duration || 5,
|
||
resolution: "720p",
|
||
generateAudio,
|
||
};
|
||
}
|
||
|
||
function buildVideoFinalPrompt(params: {
|
||
script: string;
|
||
storyboard: AgentVideoStoryboardShot[];
|
||
originalText: string;
|
||
}): string {
|
||
const script = params.script.trim();
|
||
const storyboard = params.storyboard
|
||
.slice(0, 8)
|
||
.map(
|
||
(shot) =>
|
||
`镜头${shot.shot}（${shot.duration}s）: ${shot.prompt || shot.description || shot.title}`.trim()
|
||
)
|
||
.join("\n");
|
||
|
||
return [
|
||
`【用户目标】${params.originalText.trim() || "生成视频"}`,
|
||
script ? `【脚本】${script}` : "",
|
||
storyboard ? `【分镜】\n${storyboard}` : "",
|
||
"【执行要求】镜头衔接自然，人物与主体保持一致，动作清晰，避免突兀跳切。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n")
|
||
.trim();
|
||
}
|
||
|
||
function heuristicVideoPlan(state: AutoAgentStateType): AgentVideoPlan {
|
||
const recommendedVideoParams = inferVideoParamsByText(state.text, state.hasImage);
|
||
const duration = recommendedVideoParams.duration || 5;
|
||
const perShot = Math.max(2, Math.round(duration / 3));
|
||
const storyboard: AgentVideoStoryboardShot[] = [
|
||
{
|
||
shot: 1,
|
||
title: "开场建立",
|
||
duration: perShot,
|
||
description: "建立主体与环境关系，明确视觉基调。",
|
||
prompt: `开场镜头，展示主体与环境，围绕「${compactText(state.text, 50)}」建立氛围。`,
|
||
camera: "稳定推进",
|
||
},
|
||
{
|
||
shot: 2,
|
||
title: "动作推进",
|
||
duration: perShot,
|
||
description: "主体动作或情绪推进，形成主要叙事段落。",
|
||
prompt: `中段镜头，突出主体动作与情绪变化，强化故事推进。`,
|
||
camera: "跟拍或环绕",
|
||
},
|
||
{
|
||
shot: 3,
|
||
title: "收束结尾",
|
||
duration: Math.max(2, duration - perShot * 2),
|
||
description: "情绪收束并输出最终视觉记忆点。",
|
||
prompt: "结尾镜头，画面收束，保留主体识别与情绪余韵。",
|
||
camera: "缓慢拉远",
|
||
},
|
||
];
|
||
const script = `围绕“${compactText(state.text, 80)}”制作一条节奏清晰、主体连续的短视频，包含开场建立、过程推进与结尾收束三段。`;
|
||
return {
|
||
script,
|
||
storyboard,
|
||
finalPrompt: buildVideoFinalPrompt({
|
||
script,
|
||
storyboard,
|
||
originalText: state.text,
|
||
}),
|
||
recommendedVideoParams,
|
||
};
|
||
}
|
||
|
||
function sanitizeVideoPlan(raw: unknown, state: AutoAgentStateType): AgentVideoPlan {
|
||
if (!raw || typeof raw !== "object") return heuristicVideoPlan(state);
|
||
const payload = raw as Record<string, any>;
|
||
const recommendedVideoParams = sanitizeRecommendedVideoParams(payload.recommendedVideoParams);
|
||
const script =
|
||
typeof payload.script === "string" && payload.script.trim()
|
||
? payload.script.trim().slice(0, 400)
|
||
: `围绕“${compactText(state.text, 80)}”生成视频。`;
|
||
|
||
const storyboardInput = Array.isArray(payload.storyboard) ? payload.storyboard : [];
|
||
const storyboard: AgentVideoStoryboardShot[] = storyboardInput
|
||
.map((item: any, idx) => {
|
||
const shot = Number.isFinite(item?.shot) ? Math.max(1, Math.round(item.shot)) : idx + 1;
|
||
const duration = normalizeVideoDuration(Number(item?.duration)) || 3;
|
||
const title =
|
||
typeof item?.title === "string" && item.title.trim()
|
||
? item.title.trim().slice(0, 24)
|
||
: `镜头${shot}`;
|
||
const description =
|
||
typeof item?.description === "string" && item.description.trim()
|
||
? item.description.trim().slice(0, 180)
|
||
: title;
|
||
const prompt =
|
||
typeof item?.prompt === "string" && item.prompt.trim()
|
||
? item.prompt.trim().slice(0, 260)
|
||
: description;
|
||
const camera =
|
||
typeof item?.camera === "string" && item.camera.trim()
|
||
? item.camera.trim().slice(0, 24)
|
||
: undefined;
|
||
return { shot, title, duration, description, prompt, camera };
|
||
})
|
||
.filter((item) => item.prompt);
|
||
|
||
const fallback = heuristicVideoPlan(state);
|
||
const safeStoryboard = storyboard.length > 0 ? storyboard.slice(0, 8) : fallback.storyboard;
|
||
const finalPrompt =
|
||
typeof payload.finalPrompt === "string" && payload.finalPrompt.trim()
|
||
? payload.finalPrompt.trim().slice(0, 1400)
|
||
: buildVideoFinalPrompt({
|
||
script,
|
||
storyboard: safeStoryboard,
|
||
originalText: state.text,
|
||
});
|
||
|
||
const negativePrompt =
|
||
typeof payload.negativePrompt === "string" && payload.negativePrompt.trim()
|
||
? payload.negativePrompt.trim().slice(0, 360)
|
||
: undefined;
|
||
|
||
return {
|
||
script,
|
||
storyboard: safeStoryboard,
|
||
finalPrompt,
|
||
negativePrompt,
|
||
recommendedVideoParams: recommendedVideoParams || fallback.recommendedVideoParams,
|
||
};
|
||
}
|
||
|
||
function sanitizeRecommendedImageParams(
|
||
params: AgentRecommendedImageParams | undefined,
|
||
intent?: AnalysisResult["intent"],
|
||
): AgentRecommendedImageParams | undefined {
|
||
if (!params) return undefined;
|
||
const safe: AgentRecommendedImageParams = {};
|
||
const normalizedAspectRatio = normalizeAspectRatio(params.aspectRatio);
|
||
const normalizedImageSize = normalizeImageSize(params.imageSize);
|
||
if (normalizedAspectRatio) safe.aspectRatio = normalizedAspectRatio;
|
||
if (normalizedImageSize) safe.imageSize = normalizedImageSize;
|
||
|
||
if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) {
|
||
const min = 1;
|
||
const max = intent === "image_edit" ? 1 : 12;
|
||
safe.imageCount = Math.min(Math.max(Math.round(params.imageCount), min), max);
|
||
}
|
||
if (intent === "image_edit") {
|
||
safe.imageCount = 1;
|
||
}
|
||
|
||
if (typeof params.negativePrompt === "string" && params.negativePrompt.trim()) {
|
||
safe.negativePrompt = params.negativePrompt.trim().slice(0, 500);
|
||
}
|
||
if (typeof params.promptExtend === "boolean") {
|
||
safe.promptExtend = params.promptExtend;
|
||
}
|
||
|
||
if (intent === "image_edit") {
|
||
safe.promptExtend = false;
|
||
}
|
||
|
||
return Object.keys(safe).length > 0 ? safe : undefined;
|
||
}
|
||
|
||
function heuristicImageParams(
|
||
text: string,
|
||
analysis: AnalysisResult,
|
||
): AgentRecommendedImageParams | undefined {
|
||
const normalized = text.toLowerCase();
|
||
const explicitImageSize = inferExplicitImageSizeFromText(text);
|
||
const params: AgentRecommendedImageParams = {
|
||
imageSize: explicitImageSize || (analysis.intent === "image_edit" ? "4K" : "2K"),
|
||
promptExtend: analysis.intent === "image_edit" ? false : true,
|
||
};
|
||
|
||
if (analysis.intent === "image_edit") {
|
||
params.imageCount = 1;
|
||
} else {
|
||
const explicitCount = inferExplicitImageCountFromText(text);
|
||
if (explicitCount && explicitCount >= 1) {
|
||
params.imageCount = Math.min(explicitCount, 12);
|
||
} else if (/多张|四张|4张|三张|3张/.test(normalized)) {
|
||
params.imageCount = /四张|4张/.test(normalized) ? 4 : 3;
|
||
} else if (/两张|2张|双图/.test(normalized)) {
|
||
params.imageCount = 2;
|
||
} else {
|
||
params.imageCount = 1;
|
||
}
|
||
}
|
||
|
||
const explicitAspectRatio = inferExplicitAspectRatioFromText(text);
|
||
if (explicitAspectRatio) {
|
||
params.aspectRatio = explicitAspectRatio;
|
||
} else if (/横图|横版|宽屏|封面|banner|海报/.test(normalized)) {
|
||
params.aspectRatio = "16:9";
|
||
} else if (/竖图|竖版|手机壁纸|全身|海报竖版/.test(normalized)) {
|
||
params.aspectRatio = "9:16";
|
||
} else if (/方图|方形|头像|logo|电商主图/.test(normalized)) {
|
||
params.aspectRatio = "1:1";
|
||
} else {
|
||
params.aspectRatio = "auto";
|
||
}
|
||
|
||
return sanitizeRecommendedImageParams(params, analysis.intent);
|
||
}
|
||
|
||
function buildStrategicFriendlyMessage(
|
||
analysis: AnalysisResult,
|
||
state: AutoAgentStateType,
|
||
): Promise<string | null> {
|
||
const fallbackQuestion = analysis.clarifyingQuestions?.[0];
|
||
const existingReply = analysis.conversationalReply?.trim();
|
||
|
||
if (analysis.needsClarification) {
|
||
return Promise.resolve(
|
||
analysis.conversationalReply || fallbackQuestion || "我先确认一下，你希望我优先调整哪一部分？"
|
||
);
|
||
}
|
||
|
||
if (existingReply) {
|
||
const isStructuredReply =
|
||
isAgentPlanningBriefMarkdown(existingReply) ||
|
||
/###\s*意图分析/.test(existingReply) ||
|
||
/###\s*任务规划/.test(existingReply) ||
|
||
/###\s*需求理解/.test(existingReply) ||
|
||
/###\s*图片分析/.test(existingReply) ||
|
||
/###\s*执行计划/.test(existingReply) ||
|
||
/【\s*意图分析\s*】/.test(existingReply) ||
|
||
/【\s*任务规划\s*】/.test(existingReply) ||
|
||
/【\s*需求理解\s*】/.test(existingReply) ||
|
||
/【\s*图片分析\s*】/.test(existingReply) ||
|
||
/【\s*执行计划\s*】/.test(existingReply);
|
||
if (isStructuredReply) {
|
||
return Promise.resolve(existingReply);
|
||
}
|
||
}
|
||
|
||
if (
|
||
analysis.intent === "image_edit" ||
|
||
analysis.intent === "image_gen" ||
|
||
analysis.intent === "video_gen"
|
||
) {
|
||
if (existingReply) {
|
||
return Promise.resolve(existingReply);
|
||
}
|
||
|
||
const imageParams = heuristicImageParams(state.text, analysis);
|
||
const planningBrief = buildAgentPlanningBriefMarkdown({
|
||
userText: state.text,
|
||
intent: analysis.intent,
|
||
subIntent: analysis.subIntent,
|
||
nextAction: state.plan?.nextAction,
|
||
preferredCapability: state.plan?.preferredCapability,
|
||
targetSource: state.plan?.targetSource || state.targetResolutionMode,
|
||
hasImage: state.hasImage,
|
||
imageFromHistory: state.imageFromHistory,
|
||
selectedImageCount: state.selectedImageCount,
|
||
currentUploadImageCount: state.currentUploadImageCount,
|
||
historyImageCount: state.historyImageCount,
|
||
imageCount: imageParams?.imageCount,
|
||
aspectRatio: imageParams?.aspectRatio,
|
||
imageSize: imageParams?.imageSize,
|
||
planGoal: state.plan?.goal,
|
||
});
|
||
if (process.env.AGENT_LLM_PLANNING_BRIEF_ENABLED !== "true") {
|
||
return Promise.resolve(planningBrief);
|
||
}
|
||
return withTimeoutFallback(
|
||
generateAgentConversation(analysis, state),
|
||
AGENT_OPERATION_TIMEOUT_MS,
|
||
planningBrief,
|
||
);
|
||
}
|
||
|
||
return withTimeoutFallback(
|
||
generateAgentConversation(analysis, state),
|
||
AGENT_OPERATION_TIMEOUT_MS,
|
||
existingReply || null,
|
||
);
|
||
}
|
||
|
||
type EnabledModelWithProvider = Awaited<
|
||
ReturnType<typeof import("@/lib/db/queries")["getEnabledAiModelsWithProvider"]>
|
||
>[number];
|
||
|
||
function isAgentGenerateTextProviderType(providerType: string): boolean {
|
||
return ["zhipu", "alibaba", "openai", "custom", "local", "vip98k"].includes(providerType);
|
||
}
|
||
|
||
function isLikelyVisionCapableModel(item: EnabledModelWithProvider) {
|
||
if (!item.model.isEnabled || item.model.isSubModel) return false;
|
||
return Boolean(item.model.supportsVision) || isAiModelMultimodal(item.model);
|
||
}
|
||
|
||
function getAgentModelIncompatibleReason(params: {
|
||
item: EnabledModelWithProvider;
|
||
requireVision: boolean;
|
||
}): string | null {
|
||
const providerType = String(params.item.provider.type || "");
|
||
if (!params.item.model.isEnabled || params.item.model.isSubModel) {
|
||
return "disabled_or_submodel";
|
||
}
|
||
if (!isAgentGenerateTextProviderType(providerType)) {
|
||
return `unsupported_provider:${providerType || "unknown"}`;
|
||
}
|
||
if (params.requireVision && !isLikelyVisionCapableModel(params.item)) {
|
||
return "vision_capability_missing";
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function isAgentGenerateTextCompatibleModel(params: {
|
||
item: EnabledModelWithProvider;
|
||
requireVision: boolean;
|
||
}): boolean {
|
||
return !getAgentModelIncompatibleReason(params);
|
||
}
|
||
|
||
function shouldPreferRemoteVisionUrls(item?: EnabledModelWithProvider) {
|
||
if (!item) return false;
|
||
const providerType = String(item.provider.type || "");
|
||
const baseUrl = String((item.provider as { baseUrl?: string | null })?.baseUrl || "").toLowerCase();
|
||
const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase();
|
||
|
||
if (
|
||
providerType === "volcano-multimodal" ||
|
||
providerType === "doubao-video" ||
|
||
providerType === "volcano-video" ||
|
||
providerType === "ruxa-video"
|
||
) {
|
||
return true;
|
||
}
|
||
if (baseUrl.includes("ark.cn-beijing.volces.com")) {
|
||
return true;
|
||
}
|
||
if (/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText)) {
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
function pickQwenVisionModel(models: EnabledModelWithProvider[]) {
|
||
const visionModels = models.filter((item) =>
|
||
isAgentGenerateTextCompatibleModel({ item, requireVision: true }),
|
||
);
|
||
const zhipuVision = visionModels.find(
|
||
(item) =>
|
||
item.provider.type === "zhipu" ||
|
||
/(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test(
|
||
`${item.model.displayName} ${item.model.modelId}`,
|
||
),
|
||
);
|
||
if (zhipuVision) return zhipuVision;
|
||
|
||
const qwenByName = visionModels.find(
|
||
(item) =>
|
||
item.provider.type === "alibaba" &&
|
||
/(qwen|通义|vl)/i.test(`${item.model.displayName} ${item.model.modelId}`)
|
||
);
|
||
if (qwenByName) return qwenByName;
|
||
|
||
const alibabaVision = visionModels.find((item) => item.provider.type === "alibaba");
|
||
if (alibabaVision) return alibabaVision;
|
||
return visionModels[0];
|
||
}
|
||
|
||
function pickAgentVisionModel(
|
||
agentModel: EnabledModelWithProvider | undefined,
|
||
models: EnabledModelWithProvider[],
|
||
) {
|
||
if (
|
||
agentModel &&
|
||
isAgentGenerateTextCompatibleModel({ item: agentModel, requireVision: true })
|
||
) {
|
||
return agentModel;
|
||
}
|
||
|
||
return pickQwenVisionModel(models);
|
||
}
|
||
|
||
function resolvePreferredModel(
|
||
preferredModelId: string | null | undefined,
|
||
models: EnabledModelWithProvider[],
|
||
) {
|
||
const normalizedId = String(preferredModelId || "").trim();
|
||
if (!normalizedId) return undefined;
|
||
return models.find(
|
||
(item) => item.model.id === normalizedId || item.model.modelId === normalizedId,
|
||
);
|
||
}
|
||
|
||
function resolvePrimaryAgentModel(params: {
|
||
preferredModelId?: string | null;
|
||
configuredAgentModel?: EnabledModelWithProvider | null;
|
||
allModels: EnabledModelWithProvider[];
|
||
}) {
|
||
const preferred = resolvePreferredModel(params.preferredModelId, params.allModels);
|
||
if (preferred) return preferred;
|
||
return params.configuredAgentModel || undefined;
|
||
}
|
||
|
||
function pickAgentReasoningModel(
|
||
agentModel: EnabledModelWithProvider | undefined,
|
||
models: EnabledModelWithProvider[],
|
||
) {
|
||
if (
|
||
agentModel &&
|
||
isAgentGenerateTextCompatibleModel({ item: agentModel, requireVision: false })
|
||
) {
|
||
return agentModel;
|
||
}
|
||
|
||
const candidates = models.filter((item) =>
|
||
isAgentGenerateTextCompatibleModel({ item, requireVision: false }),
|
||
);
|
||
const zhipuModel = candidates.find(
|
||
(item) =>
|
||
item.provider.type === "zhipu" &&
|
||
item.model.isEnabled &&
|
||
!item.model.isSubModel
|
||
);
|
||
if (zhipuModel) return zhipuModel;
|
||
return candidates[0];
|
||
}
|
||
|
||
function parseJsonBlock<T = Record<string, unknown>>(text: string): T | null {
|
||
const match = text.match(/\{[\s\S]*\}/);
|
||
if (!match) return null;
|
||
try {
|
||
return JSON.parse(match[0]) as T;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function resolveAgentCodingModelOverride(
|
||
selectedModelId: string,
|
||
configuredAgentModel:
|
||
| {
|
||
model?: {
|
||
id?: string | null;
|
||
defaultParams?: unknown;
|
||
capabilities?: unknown;
|
||
} | null;
|
||
}
|
||
| null
|
||
| undefined,
|
||
): string | null {
|
||
const configuredId = configuredAgentModel?.model?.id || null;
|
||
if (!configuredId || configuredId !== selectedModelId) {
|
||
return null;
|
||
}
|
||
return getAiModelAgentCodingModelId(configuredAgentModel?.model || {}) || null;
|
||
}
|
||
|
||
function sanitizeAgentPlan(
|
||
raw: unknown,
|
||
fallbackPlan: AgentExecutionPlan,
|
||
availableTools: AgentProjectToolDefinition[],
|
||
): AgentExecutionPlan {
|
||
if (!raw || typeof raw !== "object") {
|
||
return fallbackPlan;
|
||
}
|
||
|
||
const payload = raw as Record<string, unknown>;
|
||
const rawAction = String(payload.nextAction || "").trim() as AgentPlanNextAction;
|
||
const nextAction: AgentPlanNextAction = [
|
||
"respond",
|
||
"clarify",
|
||
"analyze_image",
|
||
"generate_image",
|
||
"edit_image",
|
||
"generate_video",
|
||
].includes(rawAction)
|
||
? rawAction
|
||
: fallbackPlan.nextAction;
|
||
|
||
const fallbackIntent = fallbackPlan.intent;
|
||
const rawIntent = String(payload.intent || "").trim() as AnalysisResult["intent"];
|
||
const intent: AnalysisResult["intent"] = [
|
||
"chat",
|
||
"vision",
|
||
"image_edit",
|
||
"image_gen",
|
||
"video_gen",
|
||
].includes(rawIntent)
|
||
? rawIntent
|
||
: mapActionToIntent(nextAction, fallbackIntent);
|
||
|
||
const rawResponseMode = String(payload.responseMode || "").trim() as AgentPlanResponseMode;
|
||
const responseMode: AgentPlanResponseMode = [
|
||
"direct_answer",
|
||
"tool_then_answer",
|
||
"tool_then_execute",
|
||
"clarify",
|
||
].includes(rawResponseMode)
|
||
? rawResponseMode
|
||
: defaultResponseMode(nextAction);
|
||
|
||
const confidenceNumber = Number(payload.confidence);
|
||
const confidence = Number.isFinite(confidenceNumber)
|
||
? Math.max(0.3, Math.min(0.99, confidenceNumber))
|
||
: fallbackPlan.confidence;
|
||
|
||
const goal =
|
||
typeof payload.goal === "string" && payload.goal.trim()
|
||
? payload.goal.trim().slice(0, 160)
|
||
: fallbackPlan.goal;
|
||
|
||
const rawSubIntent =
|
||
typeof payload.subIntent === "string" && payload.subIntent.trim()
|
||
? payload.subIntent.trim().slice(0, 48)
|
||
: undefined;
|
||
const rawTargetSource = String(payload.targetSource || "").trim() as AgentTargetResolutionMode;
|
||
const targetSource: AgentTargetResolutionMode = [
|
||
"explicit_selection",
|
||
"current_upload",
|
||
"text_reference",
|
||
"working_memory",
|
||
"latest_result",
|
||
"clarification",
|
||
"none",
|
||
].includes(rawTargetSource)
|
||
? rawTargetSource
|
||
: fallbackPlan.targetSource;
|
||
const sanitizedToolCalls = sanitizePlannedToolCalls(payload.toolCalls, availableTools, nextAction, {
|
||
targetSource,
|
||
});
|
||
const rawCapability = String(payload.preferredCapability || "").trim() as AgentPreferredCapability;
|
||
const derivedAnalysis: AnalysisResult = {
|
||
intent,
|
||
subIntent: intent === "image_edit" ? rawSubIntent || fallbackPlan.subIntent : undefined,
|
||
confidence,
|
||
};
|
||
const preferredCapability: AgentPreferredCapability = [
|
||
"respond",
|
||
"analyze",
|
||
"generic_edit",
|
||
"remove_background",
|
||
"remove_watermark",
|
||
"enhance",
|
||
"extend",
|
||
"resize",
|
||
"angle_variant",
|
||
"local_edit",
|
||
"box_cutout",
|
||
"generate_image",
|
||
"generate_video",
|
||
"publish_canvas",
|
||
].includes(rawCapability)
|
||
? rawCapability
|
||
: inferPreferredCapability(derivedAnalysis, sanitizedToolCalls);
|
||
|
||
return {
|
||
goal,
|
||
nextAction,
|
||
intent,
|
||
subIntent: intent === "image_edit" ? rawSubIntent || fallbackPlan.subIntent : undefined,
|
||
targetSource,
|
||
preferredCapability,
|
||
responseMode,
|
||
shouldUseHistoryImage:
|
||
typeof payload.shouldUseHistoryImage === "boolean"
|
||
? payload.shouldUseHistoryImage
|
||
: fallbackPlan.shouldUseHistoryImage,
|
||
shouldUseStructuredReply:
|
||
typeof payload.shouldUseStructuredReply === "boolean"
|
||
? payload.shouldUseStructuredReply
|
||
: shouldUseStructuredReply(nextAction),
|
||
confidence,
|
||
toolCalls: sanitizedToolCalls,
|
||
};
|
||
}
|
||
|
||
async function refinePlanWithSkillRoute(params: {
|
||
text: string;
|
||
analysis: AnalysisResult;
|
||
plan: AgentExecutionPlan;
|
||
availableTools: AgentProjectToolDefinition[];
|
||
skillRoute?: AgentSkillRouteResult;
|
||
hasImage: boolean;
|
||
hasCanvasSelection: boolean;
|
||
targetResolutionMode: AgentTargetResolutionMode;
|
||
mediaSource: "current_message" | "canvas_selection" | "history" | "none";
|
||
selectedImageCount: number;
|
||
selectedSlotCount: number;
|
||
hasSelectedAgentSlot: boolean;
|
||
threadState: AgentThreadState | null;
|
||
}): Promise<AgentExecutionPlan> {
|
||
if (!params.skillRoute) return params.plan;
|
||
|
||
const { refinePlanWithSpecialist } = await import("./specialists");
|
||
let refined = params.plan;
|
||
for (const specialist of [params.skillRoute.primary, ...params.skillRoute.overlays]) {
|
||
refined = refinePlanWithSpecialist(specialist, {
|
||
text: params.text,
|
||
analysis: params.analysis,
|
||
plan: refined,
|
||
availableTools: params.availableTools,
|
||
hasImage: params.hasImage,
|
||
hasCanvasSelection: params.hasCanvasSelection,
|
||
targetResolutionMode: params.targetResolutionMode,
|
||
mediaSource: params.mediaSource,
|
||
selectedImageCount: params.selectedImageCount,
|
||
selectedSlotCount: params.selectedSlotCount,
|
||
hasSelectedAgentSlot: params.hasSelectedAgentSlot,
|
||
threadState: params.threadState,
|
||
});
|
||
}
|
||
return refined;
|
||
}
|
||
|
||
function shouldUseLightweightPlanner(
|
||
state: AutoAgentStateType,
|
||
analysis: AnalysisResult,
|
||
): boolean {
|
||
if (process.env.AGENT_FORCE_LLM_PLANNER === "true") return false;
|
||
if (analysis.needsClarification) return false;
|
||
if (analysis.confidence < 0.9) return false;
|
||
if (analysis.intent === "image_gen") return true;
|
||
if (analysis.intent === "video_gen") return true;
|
||
if (
|
||
analysis.intent === "image_edit" &&
|
||
state.targetResolutionMode !== "clarification" &&
|
||
(state.hasImage || state.imageFromHistory || state.historyImageCount > 0 || state.selectedImageCount > 0)
|
||
) {
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
async function buildExecutionPlanWithAgent(
|
||
state: AutoAgentStateType,
|
||
analysis: AnalysisResult,
|
||
availableTools: AgentProjectToolDefinition[],
|
||
skillRoute?: AgentSkillRouteResult,
|
||
): Promise<AgentExecutionPlan> {
|
||
const fallbackPlan = buildFallbackPlan({
|
||
analysis,
|
||
text: state.text,
|
||
hasImage: state.hasImage,
|
||
imageFromHistory: state.imageFromHistory,
|
||
targetResolutionMode: state.targetResolutionMode,
|
||
availableTools,
|
||
});
|
||
|
||
if (shouldUseLightweightPlanner(state, analysis)) {
|
||
const refinedFallbackPlan = await refinePlanWithSkillRoute({
|
||
text: state.text,
|
||
analysis,
|
||
plan: fallbackPlan,
|
||
availableTools,
|
||
skillRoute,
|
||
hasImage: state.hasImage,
|
||
hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0,
|
||
targetResolutionMode: state.targetResolutionMode,
|
||
mediaSource: state.mediaSource,
|
||
selectedImageCount: state.selectedImageCount,
|
||
selectedSlotCount: state.selectedSlotCount,
|
||
hasSelectedAgentSlot: state.hasSelectedAgentSlot,
|
||
threadState: state.threadState,
|
||
});
|
||
console.log("[Auto Agent] Lightweight planner used:", {
|
||
intent: analysis.intent,
|
||
confidence: analysis.confidence,
|
||
nextAction: refinedFallbackPlan.nextAction,
|
||
});
|
||
return refinedFallbackPlan;
|
||
}
|
||
|
||
try {
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const primaryAgentModel = resolvePrimaryAgentModel({
|
||
preferredModelId: state.preferredModelId,
|
||
configuredAgentModel: agentModel,
|
||
allModels,
|
||
});
|
||
const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels);
|
||
if (!plannerModelConfig) {
|
||
return fallbackPlan;
|
||
}
|
||
const plannerModel = await getLanguageModelById(
|
||
plannerModelConfig.model.id,
|
||
resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel),
|
||
);
|
||
if (!plannerModel) {
|
||
return fallbackPlan;
|
||
}
|
||
|
||
const toolCatalog = availableTools
|
||
.filter((tool) => tool.plannerHidden !== true)
|
||
.map((tool) => `- ${tool.name}: ${tool.title}｜${tool.description}`)
|
||
.join("\n");
|
||
const historyContext = (state.historyMessages || [])
|
||
.slice(-6)
|
||
.map((item) => `${item.role}: ${compactText(item.content || "", 120)}`)
|
||
.join("\n");
|
||
|
||
const { text } = await generateText({
|
||
model: plannerModel,
|
||
system: `你是一个多模态 Agent Planner。
|
||
你的职责不是直接回答，而是决定当前回合下一步该走哪条能力路径。
|
||
只输出 JSON，不要输出解释，不要输出 markdown。
|
||
|
||
可选 nextAction：
|
||
- respond：纯对话、策略讨论、解释说明、普通问答
|
||
- clarify：信息不足，需要先澄清
|
||
- analyze_image：用户要你看图、识图、评估、分析、给建议
|
||
- edit_image：基于当前图/历史图做修改、抠图、去水印、增强、换背景、调色、尺寸调整、局部重绘
|
||
- generate_image：生成新图
|
||
- generate_video：生成视频
|
||
|
||
规则：
|
||
1. 用户只是聊天、问图、讨论方案时，绝不能走生图/生视频。
|
||
2. 只有用户明确要求“生成/修改/做视频”等执行任务时，才走执行型 action。
|
||
3. 如果用户上传了图片但只是问“这是什么/适不适合/帮我分析一下”，应选择 analyze_image 或 respond。
|
||
4. 对 analyze_image / edit_image 任务，若目标图来源不是 none/clarification，应优先先调用 resolveTargetImage，再调用主工具。
|
||
5. 优先保持一次只选择一个主工具；toolCalls 只从提供的工具列表里选。
|
||
6. shouldUseStructuredReply 只在 edit_image / generate_image / generate_video 时设为 true。
|
||
7. targetSource 要忠实反映当前目标图来源：explicit_selection/current_upload/text_reference/working_memory/latest_result/clarification/none。
|
||
8. preferredCapability 要尽量具体，例如 remove_background / remove_watermark / enhance / extend / resize / angle_variant / local_edit / box_cutout / publish_canvas。
|
||
9. 必须优先使用新的强语义工具名：resolveTargetImage / analyzeTargetImage / editTargetImage / generateNewImage / publishResultToCanvas。
|
||
10. 必须遵守当前专家路由约束；如果约束要求纯对话，则绝不能输出 generate_image / edit_image / generate_video。
|
||
|
||
输出格式：
|
||
{
|
||
"goal": "一句话概括当前回合目标",
|
||
"nextAction": "respond|clarify|analyze_image|edit_image|generate_image|generate_video",
|
||
"intent": "chat|vision|image_edit|image_gen|video_gen",
|
||
"subIntent": "可选，仅 image_edit 时填写",
|
||
"targetSource": "explicit_selection|current_upload|text_reference|working_memory|latest_result|clarification|none",
|
||
"preferredCapability": "respond|analyze|generic_edit|remove_background|remove_watermark|enhance|extend|resize|angle_variant|local_edit|box_cutout|generate_image|generate_video|publish_canvas",
|
||
"responseMode": "direct_answer|tool_then_answer|tool_then_execute|clarify",
|
||
"shouldUseHistoryImage": true,
|
||
"shouldUseStructuredReply": false,
|
||
"confidence": 0.92,
|
||
"toolCalls": [
|
||
{
|
||
"name": "resolveTargetImage",
|
||
"reason": "先锁定本轮要处理的目标图",
|
||
"args": { "source": "selected_canvas" }
|
||
},
|
||
{
|
||
"name": "analyzeTargetImage",
|
||
"reason": "为什么选这个工具",
|
||
"args": {}
|
||
}
|
||
]
|
||
}`,
|
||
prompt: [
|
||
`用户输入：${state.text || "（空）"}`,
|
||
`有当前或历史图片：${state.hasImage ? "是" : "否"}`,
|
||
`图片来自历史：${state.imageFromHistory ? "是" : "否"}`,
|
||
`目标图来源：${state.targetResolutionMode}（${describeTargetSource(state.targetResolutionMode)}）`,
|
||
`媒体输入来源：${state.mediaSource}`,
|
||
`当前上传图数量：${state.currentUploadImageCount}`,
|
||
`历史图数量：${state.historyImageCount}`,
|
||
`画布选中图数量：${state.selectedImageCount}`,
|
||
`画布选中 Agent 槽位数量：${state.selectedSlotCount}`,
|
||
state.hasSelectedAgentSlot
|
||
? "画布中当前选中的空 Agent Slot 是有效生成目标；如果用户要求在这里生成/填充，不要因为没有图片而澄清。"
|
||
: "",
|
||
state.canvasAgentContext?.slots?.length
|
||
? `画布槽位摘要：${state.canvasAgentContext.slots
|
||
.slice(0, 6)
|
||
.map((slot) =>
|
||
[
|
||
slot.nodeId,
|
||
slot.slotType,
|
||
slot.label || "未命名槽位",
|
||
slot.isSelected ? "selected" : "",
|
||
slot.fillPolicy,
|
||
]
|
||
.filter(Boolean)
|
||
.join("/")
|
||
)
|
||
.join("；")}`
|
||
: "",
|
||
`基础分析：intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`,
|
||
skillRoute
|
||
? `专家路由：${skillRoute.primary.specialistId}｜原因：${skillRoute.reason}`
|
||
: "",
|
||
skillRoute?.planningHints?.length
|
||
? `专家约束：\n${skillRoute.planningHints.map((item, index) => `${index + 1}. ${item}`).join("\n")}`
|
||
: "",
|
||
state.threadState?.workflowState
|
||
? `当前工作流：type=${state.threadState.workflowState.type}; status=${state.threadState.workflowState.status}; summary=${compactText(state.threadState.workflowState.summary || "", 240)}`
|
||
: "",
|
||
state.context ? `画布上下文：${compactText(state.context, 600)}` : "",
|
||
historyContext ? `最近对话：\n${historyContext}` : "",
|
||
`可用工具：\n${toolCatalog}`,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n"),
|
||
temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.1),
|
||
maxTokens: 420,
|
||
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
|
||
} as any);
|
||
|
||
const parsed = parseJsonBlock<Record<string, unknown>>(text);
|
||
let planned = sanitizeAgentPlan(parsed, fallbackPlan, availableTools);
|
||
planned = await refinePlanWithSkillRoute({
|
||
text: state.text,
|
||
analysis,
|
||
plan: planned,
|
||
availableTools,
|
||
skillRoute,
|
||
hasImage: state.hasImage,
|
||
hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0,
|
||
targetResolutionMode: state.targetResolutionMode,
|
||
mediaSource: state.mediaSource,
|
||
selectedImageCount: state.selectedImageCount,
|
||
selectedSlotCount: state.selectedSlotCount,
|
||
hasSelectedAgentSlot: state.hasSelectedAgentSlot,
|
||
threadState: state.threadState,
|
||
});
|
||
console.log(
|
||
`[Auto Agent] Planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})`,
|
||
);
|
||
console.log("[Auto Agent] Execution plan:", planned);
|
||
return planned;
|
||
} catch (error) {
|
||
console.error("[Auto Agent] Planner failed, fallback to heuristic plan:", error);
|
||
return fallbackPlan;
|
||
}
|
||
}
|
||
|
||
async function generateAgentConversation(
|
||
analysis: AnalysisResult,
|
||
state: AutoAgentStateType
|
||
): Promise<string | null> {
|
||
try {
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const primaryAgentModel = resolvePrimaryAgentModel({
|
||
preferredModelId: state.preferredModelId,
|
||
configuredAgentModel: agentModel,
|
||
allModels,
|
||
});
|
||
if (primaryAgentModel && state.preferredModelId) {
|
||
console.log(
|
||
`[Auto Agent] Preferred analysis model: ${primaryAgentModel.model.displayName} (${primaryAgentModel.provider.type})`,
|
||
);
|
||
}
|
||
if (primaryAgentModel) {
|
||
const reasoningIncompatibleReason = getAgentModelIncompatibleReason({
|
||
item: primaryAgentModel,
|
||
requireVision: false,
|
||
});
|
||
if (reasoningIncompatibleReason) {
|
||
console.warn(
|
||
`[Auto Agent] Preferred model incompatible for reasoning, skip direct use: ${primaryAgentModel.model.displayName} (${primaryAgentModel.provider.type}), reason=${reasoningIncompatibleReason}`,
|
||
);
|
||
}
|
||
}
|
||
const reasoningModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels);
|
||
if (!reasoningModelConfig) {
|
||
return analysis.conversationalReply || null;
|
||
}
|
||
const reasoningModel = await getLanguageModelById(
|
||
reasoningModelConfig.model.id,
|
||
resolveAgentCodingModelOverride(reasoningModelConfig.model.id, agentModel),
|
||
);
|
||
if (!reasoningModel) {
|
||
return analysis.conversationalReply || null;
|
||
}
|
||
|
||
const history = state.historyMessages
|
||
.slice(-6)
|
||
.map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`)
|
||
.join("\n");
|
||
|
||
const validImageUrls = state.imageUrls
|
||
.filter((url) => isHttpUrl(url))
|
||
.slice(0, AGENT_VISION_MAX_IMAGES);
|
||
|
||
const imageInfo = state.hasImage
|
||
? `用户本轮可用图片数量：${state.imageUrls.length || 1}。图片来源：${state.imageFromHistory ? "历史消息复用" : "本轮上传"}。`
|
||
: "用户本轮没有上传图片。";
|
||
|
||
let visualAnalysis = "";
|
||
if (state.hasImage && validImageUrls.length > 0) {
|
||
try {
|
||
const visionModelConfig = pickAgentVisionModel(primaryAgentModel, allModels);
|
||
if (!visionModelConfig) {
|
||
throw new Error("missing_vision_model");
|
||
}
|
||
const visionModel = await getLanguageModelById(visionModelConfig.model.id);
|
||
if (!visionModel) {
|
||
throw new Error("missing_vision_runtime");
|
||
}
|
||
|
||
const visionPrompt = `请分析以下参考图片，给出可用于生图/修图执行前决策的结果。
|
||
要求：
|
||
1) 每张图输出 1 行，格式：图X：主体...；风格...；构图...；光影与色调...；材质/氛围...
|
||
2) 只写客观可见信息，不要编造品牌、尺寸、镜头参数。
|
||
3) 全部输出控制在 6 行内。`;
|
||
|
||
const preferRemoteVisionUrls = shouldPreferRemoteVisionUrls(visionModelConfig);
|
||
const visionInputs = await buildAgentVisionInputs(validImageUrls, {
|
||
preferRemoteUrls: preferRemoteVisionUrls,
|
||
});
|
||
const { text: visionText } = await generateText({
|
||
model: visionModel,
|
||
messages: [
|
||
{
|
||
role: "user" as const,
|
||
content: [{ type: "text" as const, text: visionPrompt }, ...visionInputs],
|
||
},
|
||
],
|
||
temperature: getCompatibleTextGenerationTemperature(visionModelConfig, 0.2),
|
||
maxTokens: 360,
|
||
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
|
||
} as any);
|
||
|
||
visualAnalysis = visionText.trim();
|
||
console.log(
|
||
`[Auto Agent] Vision analysis model: ${visionModelConfig.model.displayName} (${visionModelConfig.provider.type}), preferRemote=${preferRemoteVisionUrls}`
|
||
);
|
||
} catch (error) {
|
||
console.warn("[Auto Agent] Vision analysis failed, continue without visual summary");
|
||
visualAnalysis = "";
|
||
}
|
||
}
|
||
|
||
const { buildAgentRoleIdentityInstruction } = await import(
|
||
"@/lib/ai/runtime/agent-role-profile"
|
||
);
|
||
const identityInstruction = await buildAgentRoleIdentityInstruction({
|
||
allowVideoGeneration: true,
|
||
hasCanvasContext: Boolean(state.context),
|
||
});
|
||
const systemPrompt = `${identityInstruction}
|
||
|
||
你是一个真实执行中的 AI 生图/修图 Agent。你的核心目标是提升用户一次生成成功率。
|
||
|
||
你必须按以下 AGENT SKILLS 执行：
|
||
1) 需求拆解：把用户目标拆成可执行指令，优先满足用户明确要求。
|
||
2) 风险预判：主动识别可能导致失败的歧义（主体数量、视角、风格冲突、文字内容）。
|
||
3) 提示词策略：给出可落地的正向约束和必要的负向约束，避免空泛描述。
|
||
4) 参数策略：在输出设置中明确建议宽高比、清晰度档位、出图数量（若用户未指定）。
|
||
|
||
输出要求：
|
||
- 不要说“看不到图片/无法访问图片”。
|
||
- 不暴露内部实现、模型路由、算法细节。
|
||
- 输出控制在 8-14 行，标题行不要使用 # 号。
|
||
- 有图片时必须按顺序输出以下标题（逐字一致）：
|
||
【需求理解】
|
||
【图片分析】
|
||
【执行计划】
|
||
【输出设置】
|
||
- “图片分析”必须基于视觉分析结果写具体观察，不允许写泛化空话。
|
||
- 只有明确拿到原图分辨率时，才能写具体像素尺寸；否则只写宽高比/清晰度目标。`;
|
||
|
||
const prompt = [
|
||
`【用户原始需求】${state.text || "（无文本）"}`,
|
||
`【意图分析】intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`,
|
||
`【图片信息】${imageInfo}`,
|
||
visualAnalysis ? `【视觉模型预分析】\n${visualAnalysis}` : "",
|
||
state.context ? `【画布上下文】${compactText(state.context, 300)}` : "",
|
||
history ? `【最近对话】\n${history}` : "",
|
||
state.hasImage
|
||
? "请先做本轮图片分析，再输出执行前汇报。"
|
||
: "请输出本轮执行前汇报。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n");
|
||
|
||
const { text } = await generateText({
|
||
model: reasoningModel,
|
||
system: systemPrompt,
|
||
prompt,
|
||
temperature: getCompatibleTextGenerationTemperature(reasoningModelConfig, 0.35),
|
||
maxTokens: 520,
|
||
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
|
||
} as any);
|
||
|
||
const cleaned = text.trim();
|
||
if (cleaned.length < 10) {
|
||
return analysis.conversationalReply || null;
|
||
}
|
||
console.log(
|
||
`[Auto Agent] Conversation model: ${reasoningModelConfig.model.displayName} (${reasoningModelConfig.provider.type})`
|
||
);
|
||
return cleaned;
|
||
} catch (error) {
|
||
return analysis.conversationalReply || null;
|
||
}
|
||
}
|
||
|
||
async function recommendImageParamsWithAgent(
|
||
analysis: AnalysisResult,
|
||
state: AutoAgentStateType,
|
||
): Promise<AgentRecommendedImageParams | undefined> {
|
||
if (analysis.intent !== "image_edit" && analysis.intent !== "image_gen") {
|
||
return undefined;
|
||
}
|
||
|
||
try {
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const primaryAgentModel = resolvePrimaryAgentModel({
|
||
preferredModelId: state.preferredModelId,
|
||
configuredAgentModel: agentModel,
|
||
allModels,
|
||
});
|
||
const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels);
|
||
if (!plannerModelConfig) {
|
||
return heuristicImageParams(state.text, analysis);
|
||
}
|
||
const plannerModel = await getLanguageModelById(
|
||
plannerModelConfig.model.id,
|
||
resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel),
|
||
);
|
||
if (!plannerModel) {
|
||
return heuristicImageParams(state.text, analysis);
|
||
}
|
||
|
||
const history = state.historyMessages
|
||
.slice(-4)
|
||
.map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`)
|
||
.join("\n");
|
||
|
||
const requestText = [
|
||
`用户请求：${state.text || "（无文本）"}`,
|
||
`识别意图：${analysis.intent}/${analysis.subIntent || "none"}`,
|
||
`是否有参考图：${state.hasImage ? "是" : "否"}`,
|
||
history ? `最近对话：\n${history}` : "",
|
||
"请给出本次最稳妥的生图参数建议。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n");
|
||
|
||
const { text } = await generateText({
|
||
model: plannerModel,
|
||
system: `你是生图参数规划 Agent。目标是提升首图命中率和稳定性。
|
||
只输出 JSON，不要输出解释。格式如下：
|
||
{
|
||
"aspectRatio": "1:1|2:3|3:2|3:4|4:3|4:5|5:4|9:16|16:9|21:9|auto",
|
||
"imageSize": "1K|2K|4K",
|
||
"imageCount": 1,
|
||
"negativePrompt": "可选，简洁",
|
||
"promptExtend": true
|
||
}
|
||
规则：
|
||
- image_edit 场景强制 imageCount=1。
|
||
- 若用户未指定比例，优先 auto。
|
||
- negativePrompt 仅在明显需要避坑时填写。`,
|
||
prompt: requestText,
|
||
temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.15),
|
||
maxTokens: 220,
|
||
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
|
||
} as any);
|
||
|
||
const parsed = parseJsonBlock<AgentRecommendedImageParams>(text);
|
||
const sanitized = sanitizeRecommendedImageParams(parsed || undefined, analysis.intent);
|
||
if (sanitized) {
|
||
console.log(
|
||
`[Auto Agent] Parameter planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})`
|
||
);
|
||
return sanitized;
|
||
}
|
||
return heuristicImageParams(state.text, analysis);
|
||
} catch {
|
||
return heuristicImageParams(state.text, analysis);
|
||
}
|
||
}
|
||
|
||
async function recommendVideoPlanWithAgent(
|
||
state: AutoAgentStateType,
|
||
): Promise<AgentVideoPlan> {
|
||
try {
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const primaryAgentModel = resolvePrimaryAgentModel({
|
||
preferredModelId: state.preferredModelId,
|
||
configuredAgentModel: agentModel,
|
||
allModels,
|
||
});
|
||
const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels);
|
||
if (!plannerModelConfig) {
|
||
return heuristicVideoPlan(state);
|
||
}
|
||
const plannerModel = await getLanguageModelById(
|
||
plannerModelConfig.model.id,
|
||
resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel),
|
||
);
|
||
if (!plannerModel) {
|
||
return heuristicVideoPlan(state);
|
||
}
|
||
|
||
const history = state.historyMessages
|
||
.slice(-6)
|
||
.map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`)
|
||
.join("\n");
|
||
|
||
const prompt = [
|
||
`用户请求：${state.text || "生成视频"}`,
|
||
`是否有参考图：${state.hasImage ? "是" : "否"}（${state.imageUrls.length} 张）`,
|
||
history ? `最近对话：\n${history}` : "",
|
||
"请产出可直接用于视频生成的脚本与分镜。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n");
|
||
|
||
const { text } = await generateText({
|
||
model: plannerModel,
|
||
system: `你是视频生成 Agent 规划器，目标是最大化一次出片成功率。
|
||
只输出 JSON，不要输出解释。格式：
|
||
{
|
||
"script": "120字内剧情脚本",
|
||
"recommendedVideoParams": {
|
||
"mode": "text-to-video|first-frame|first-last-frame|reference-images",
|
||
"resolution": "480p|720p|1080p",
|
||
"ratio": "16:9|4:3|1:1|3:4|9:16|21:9|adaptive",
|
||
"duration": 5,
|
||
"generateAudio": false,
|
||
"cameraMove": "none|static|handheld|dynamic-orbit"
|
||
},
|
||
"storyboard": [
|
||
{
|
||
"shot": 1,
|
||
"title": "镜头标题",
|
||
"duration": 3,
|
||
"description": "画面描述",
|
||
"camera": "运镜方式",
|
||
"prompt": "该镜头可直接喂给视频模型的提示词"
|
||
}
|
||
],
|
||
"negativePrompt": "可选",
|
||
"finalPrompt": "整合后的最终视频提示词"
|
||
}
|
||
要求：
|
||
- 分镜 2~6 个，时长总计与 duration 接近。
|
||
- 有参考图时优先 first-frame 或 first-last-frame。
|
||
- 避免抽象空话，必须可执行。`,
|
||
prompt,
|
||
temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.2),
|
||
maxTokens: 720,
|
||
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
|
||
} as any);
|
||
|
||
const parsed = parseJsonBlock<Record<string, unknown>>(text);
|
||
const plan = sanitizeVideoPlan(parsed, state);
|
||
console.log(
|
||
`[Auto Agent] Video planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})`
|
||
);
|
||
return plan;
|
||
} catch {
|
||
return heuristicVideoPlan(state);
|
||
}
|
||
}
|
||
|
||
async function refineRouteWithAgent(
|
||
state: AutoAgentStateType,
|
||
baseRoute: RouteResult,
|
||
): Promise<RouteResult> {
|
||
if (!state.analysis || (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen")) {
|
||
return baseRoute;
|
||
}
|
||
|
||
try {
|
||
const { db } = await import("@/lib/db");
|
||
const { siteSettings } = await import("@/lib/db/schema");
|
||
const [settings] = await db.select().from(siteSettings).limit(1);
|
||
const configuredDefaultModelId = settings?.canvasAutoModeDefaultModelId || null;
|
||
if (configuredDefaultModelId && baseRoute.id === configuredDefaultModelId) {
|
||
console.log(
|
||
`[Auto Agent] Keep admin default model for auto mode: ${baseRoute.displayName}`
|
||
);
|
||
return baseRoute;
|
||
}
|
||
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
const { isAiModelAgentAuto } = await import("@/lib/db/ai-model-flags");
|
||
const allModelsRaw = await getEnabledAiModelsWithProvider(false);
|
||
const allModels = allModelsRaw.filter((item) => !isAiModelAgentAuto(item.model));
|
||
const imageCandidates = allModels
|
||
.filter((item) => item.model.supportsImageGen && item.model.isEnabled && !item.model.isSubModel)
|
||
.slice(0, 12);
|
||
if (imageCandidates.length <= 1) return baseRoute;
|
||
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const primaryAgentModel = resolvePrimaryAgentModel({
|
||
preferredModelId: state.preferredModelId,
|
||
configuredAgentModel: agentModel,
|
||
allModels: allModelsRaw,
|
||
});
|
||
const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModelsRaw);
|
||
if (!plannerModelConfig) return baseRoute;
|
||
const plannerModel = await getLanguageModelById(
|
||
plannerModelConfig.model.id,
|
||
resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel),
|
||
);
|
||
if (!plannerModel) return baseRoute;
|
||
|
||
const candidatesText = imageCandidates
|
||
.map((item, index) => `${index + 1}. id=${item.model.id}; 名称=${item.model.displayName}; provider=${item.provider.type}`)
|
||
.join("\n");
|
||
|
||
const { text } = await generateText({
|
||
model: plannerModel,
|
||
system: `你是生图模型选择 Agent。目标是优先提高一次成功率。
|
||
只输出 JSON：
|
||
{
|
||
"modelId": "候选里的id",
|
||
"reason": "一句话中文理由，不超过24字"
|
||
}
|
||
如果无法判断，也必须给出候选中的 modelId。`,
|
||
prompt: `用户请求：${state.text || "（无文本）"}
|
||
意图：${state.analysis.intent}/${state.analysis.subIntent || "none"}
|
||
候选模型：
|
||
${candidatesText}`,
|
||
temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.1),
|
||
maxTokens: 120,
|
||
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
|
||
} as any);
|
||
|
||
const parsed = parseJsonBlock<{ modelId?: string; reason?: string }>(text);
|
||
if (!parsed?.modelId) return baseRoute;
|
||
const chosen = imageCandidates.find((item) => item.model.id === parsed.modelId);
|
||
if (!chosen) return baseRoute;
|
||
|
||
return {
|
||
id: chosen.model.id,
|
||
displayName: chosen.model.displayName,
|
||
providerType: chosen.provider.type,
|
||
modelId: chosen.model.modelId,
|
||
reason: `Agent规划: ${(parsed.reason || chosen.model.displayName).slice(0, 40)}`,
|
||
};
|
||
} catch {
|
||
return baseRoute;
|
||
}
|
||
}
|
||
|
||
const analyzeNode = async (state: AutoAgentStateType) => {
|
||
const { analyzeIntent } = await import("./smart-analyzer");
|
||
const analysis = await analyzeIntent({
|
||
text: state.text,
|
||
hasImage: state.hasImage,
|
||
imageUrls: state.imageUrls,
|
||
assetIds: state.assetIds,
|
||
imageFromHistory: state.imageFromHistory,
|
||
historyMessages: state.historyMessages,
|
||
context: state.context ?? undefined,
|
||
preferredModelId: state.preferredModelId ?? undefined,
|
||
});
|
||
|
||
return { analysis };
|
||
};
|
||
|
||
const confirmationGateNode = async (state: AutoAgentStateType) => {
|
||
const workflowDecision = await runHumanConfirmationGate({
|
||
text: state.text,
|
||
threadState: state.threadState,
|
||
});
|
||
|
||
if (!workflowDecision.shouldBypassPlanner) {
|
||
return {
|
||
workflowDecision,
|
||
};
|
||
}
|
||
|
||
return buildWorkflowGateBypassPayload(state, workflowDecision);
|
||
};
|
||
|
||
const shouldBypassPlanner = (state: AutoAgentStateType) => {
|
||
return state.workflowDecision?.shouldBypassPlanner ? "route" : "analyze";
|
||
};
|
||
|
||
const planNode = async (state: AutoAgentStateType) => {
|
||
if (!state.analysis) return {};
|
||
const hasCanvasOperationTarget =
|
||
state.hasCanvasSelection ||
|
||
state.selectedImageCount > 0 ||
|
||
state.hasSelectedAgentSlot;
|
||
const baseAvailableTools = await getEnabledAgentToolDefinitionsForContext({
|
||
hasImage: state.hasImage,
|
||
hasCanvasSelection: hasCanvasOperationTarget,
|
||
hasHistoryImage: state.historyImageCount > 0 || state.imageFromHistory,
|
||
hasCurrentUpload: state.currentUploadImageCount > 0,
|
||
});
|
||
const { filterToolsForSkillRoute, routeAgentSkill } = await import("./skill-router");
|
||
const skillRoute = routeAgentSkill({
|
||
text: state.text,
|
||
analysis: state.analysis,
|
||
hasImage: state.hasImage,
|
||
hasCanvasSelection: hasCanvasOperationTarget,
|
||
targetResolutionMode: state.targetResolutionMode,
|
||
mediaSource: state.mediaSource,
|
||
selectedImageCount: state.selectedImageCount,
|
||
selectedSlotCount: state.selectedSlotCount,
|
||
hasSelectedAgentSlot: state.hasSelectedAgentSlot,
|
||
threadState: state.threadState,
|
||
});
|
||
const availableTools = filterToolsForSkillRoute(baseAvailableTools, skillRoute);
|
||
console.log("[Auto Agent] Skill route:", {
|
||
specialist: skillRoute.primary.specialistId,
|
||
domain: skillRoute.primary.domain,
|
||
overlays: skillRoute.overlays.map((item) => item.specialistId),
|
||
allowlist: skillRoute.toolAllowlist,
|
||
reason: skillRoute.reason,
|
||
});
|
||
const plan = await buildExecutionPlanWithAgent(
|
||
state,
|
||
state.analysis,
|
||
availableTools,
|
||
skillRoute,
|
||
);
|
||
const alignedAnalysis = alignAnalysisWithPlan(state.analysis, plan);
|
||
|
||
return {
|
||
availableTools,
|
||
plan,
|
||
analysis: alignedAnalysis,
|
||
};
|
||
};
|
||
|
||
const prepareExecutionNode = async (state: AutoAgentStateType) => {
|
||
if (!state.analysis || !state.plan) {
|
||
return {};
|
||
}
|
||
|
||
const execution = prepareExecutionFromPlan({
|
||
plan: state.plan,
|
||
availableTools: state.availableTools || [],
|
||
});
|
||
|
||
const fallbackQuestion = state.analysis.needsClarification
|
||
? state.analysis.clarifyingQuestions?.[0]
|
||
: undefined;
|
||
|
||
let friendlyMessage =
|
||
state.analysis.conversationalReply || fallbackQuestion || null;
|
||
|
||
if (state.plan.nextAction === "clarify") {
|
||
friendlyMessage =
|
||
friendlyMessage || "我先确认一下你希望我下一步执行什么。";
|
||
} else if (execution.shouldUseStructuredReply) {
|
||
const strategicMessage = await buildStrategicFriendlyMessage(state.analysis, state);
|
||
friendlyMessage =
|
||
strategicMessage ||
|
||
state.analysis.conversationalReply ||
|
||
friendlyMessage;
|
||
}
|
||
|
||
return {
|
||
execution,
|
||
friendlyMessage,
|
||
};
|
||
};
|
||
|
||
const shouldOptimize = (state: AutoAgentStateType) => {
|
||
if (!state.analysis || !state.execution) return "route";
|
||
if (state.analysis.needsClarification) return "route";
|
||
return state.execution.shouldOptimizePrompt ? "optimize" : "route";
|
||
};
|
||
|
||
function shouldUseLlmPromptOptimizer(state: AutoAgentStateType): boolean {
|
||
if (!state.analysis || !state.execution?.shouldApplyImagePipeline) return false;
|
||
if (process.env.AGENT_LLM_PROMPT_OPTIMIZER_ENABLED === "false") return false;
|
||
if (
|
||
state.analysis.intent === "image_gen" &&
|
||
process.env.AGENT_IMAGE_GEN_LLM_OPTIMIZER_ENABLED !== "true"
|
||
) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
const optimizeNode = async (state: AutoAgentStateType) => {
|
||
if (!state.analysis || !state.execution) {
|
||
return {};
|
||
}
|
||
if (state.execution.mode === "generate_video") {
|
||
const videoPlan = await recommendVideoPlanWithAgent(state);
|
||
const storyboardPreview = videoPlan.storyboard
|
||
.slice(0, 4)
|
||
.map(
|
||
(shot) =>
|
||
`- 镜头${shot.shot}（${shot.duration}s）：${shot.title}｜${compactText(
|
||
shot.description || shot.prompt,
|
||
64,
|
||
)}`
|
||
)
|
||
.join("\n");
|
||
const friendlyMessage = [
|
||
"### 需求理解",
|
||
`- ${compactText(state.text || "生成视频", 110)}`,
|
||
"### 脚本规划",
|
||
`- ${videoPlan.script}`,
|
||
"### 分镜规划",
|
||
storyboardPreview || "- 已生成分镜，执行时自动拼接。",
|
||
"### 执行说明",
|
||
"- 已基于上述脚本与分镜优化提示词并提交视频模型执行。",
|
||
].join("\n");
|
||
return {
|
||
friendlyMessage,
|
||
optimizedPrompt: videoPlan.finalPrompt,
|
||
videoPlan,
|
||
};
|
||
}
|
||
if (!state.execution.shouldApplyImagePipeline) {
|
||
return {};
|
||
}
|
||
const { optimizePrompt, optimizePromptWithLLM } = await import("./prompt-optimizer");
|
||
const optimizedFromLlm = shouldUseLlmPromptOptimizer(state)
|
||
? await withTimeoutFallback<Awaited<ReturnType<typeof optimizePromptWithLLM>> | null>(
|
||
optimizePromptWithLLM({
|
||
originalPrompt: state.text,
|
||
intent: state.analysis.intent,
|
||
subIntent: state.analysis.subIntent,
|
||
historyMessages: state.historyMessages,
|
||
context: state.context || undefined,
|
||
preferredModelId: state.preferredModelId || undefined,
|
||
}),
|
||
AGENT_OPERATION_TIMEOUT_MS,
|
||
null,
|
||
)
|
||
: null;
|
||
const optimized =
|
||
optimizedFromLlm ||
|
||
optimizePrompt({
|
||
originalPrompt: state.text,
|
||
intent: state.analysis.intent,
|
||
subIntent: state.analysis.subIntent,
|
||
});
|
||
const heuristicParams = heuristicImageParams(state.text, state.analysis);
|
||
const explicitTextParams = extractExplicitImageParamsFromText(state.text);
|
||
const mergedParams = sanitizeRecommendedImageParams(
|
||
{
|
||
...heuristicParams,
|
||
...optimized.recommendedImageParams,
|
||
...explicitTextParams,
|
||
negativePrompt:
|
||
optimized.negativePrompt && optimized.negativePrompt.trim().length > 0
|
||
? optimized.negativePrompt
|
||
: optimized.recommendedImageParams?.negativePrompt,
|
||
},
|
||
state.analysis.intent,
|
||
) ?? heuristicParams;
|
||
|
||
return {
|
||
optimizedPrompt: optimized.optimizedPrompt,
|
||
recommendedImageParams: mergedParams,
|
||
};
|
||
};
|
||
|
||
const routeNode = async (state: AutoAgentStateType) => {
|
||
const { routeToModel } = await import("./model-router");
|
||
let route = await routeToModel({
|
||
intent: state.execution?.routeIntent || state.analysis?.intent || "chat",
|
||
subIntent: state.execution?.routeSubIntent || state.analysis?.subIntent,
|
||
hasImage: state.hasImage,
|
||
});
|
||
if (state.execution?.shouldApplyImagePipeline) {
|
||
route = await refineRouteWithAgent(state, route);
|
||
}
|
||
return { route };
|
||
};
|
||
|
||
let compiledGraph: ReturnType<typeof buildGraph> | null = null;
|
||
|
||
function buildGraph() {
|
||
return new StateGraph(AutoAgentState)
|
||
.addNode("confirmationGate", confirmationGateNode)
|
||
.addNode("analyze", analyzeNode)
|
||
.addNode("planner", planNode)
|
||
.addNode("prepareExecution", prepareExecutionNode)
|
||
.addNode("optimize", optimizeNode)
|
||
.addNode("routeModel", routeNode)
|
||
.addEdge(START, "confirmationGate")
|
||
.addConditionalEdges("confirmationGate", shouldBypassPlanner, {
|
||
analyze: "analyze",
|
||
route: "routeModel",
|
||
})
|
||
.addEdge("analyze", "planner")
|
||
.addEdge("planner", "prepareExecution")
|
||
.addConditionalEdges("prepareExecution", shouldOptimize, {
|
||
optimize: "optimize",
|
||
route: "routeModel",
|
||
})
|
||
.addEdge("optimize", "routeModel")
|
||
.addEdge("routeModel", END)
|
||
.compile();
|
||
}
|
||
|
||
function getGraph() {
|
||
if (!compiledGraph) {
|
||
compiledGraph = buildGraph();
|
||
}
|
||
return compiledGraph;
|
||
}
|
||
|
||
export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAgentResult> {
|
||
const graph = getGraph();
|
||
const result = await graph.invoke({
|
||
text: params.text,
|
||
hasImage: params.hasImage,
|
||
imageUrls: params.imageUrls ?? [],
|
||
assetIds: params.assetIds ?? [],
|
||
imageFromHistory: params.imageFromHistory ?? false,
|
||
targetResolutionMode: params.targetResolutionMode ?? "none",
|
||
mediaSource: params.mediaSource ?? "none",
|
||
hasCanvasSelection: params.hasCanvasSelection ?? false,
|
||
selectedImageCount: params.selectedImageCount ?? 0,
|
||
canvasAgentContext: params.canvasAgentContext ?? null,
|
||
selectedSlotCount: params.selectedSlotCount ?? 0,
|
||
hasSelectedAgentSlot: params.hasSelectedAgentSlot ?? false,
|
||
historyImageCount: params.historyImageCount ?? 0,
|
||
currentUploadImageCount: params.currentUploadImageCount ?? 0,
|
||
historyMessages: params.historyMessages ?? [],
|
||
context: params.context ?? null,
|
||
preferredModelId: params.preferredModelId ?? null,
|
||
threadState: normalizeAgentThreadState(params.threadState),
|
||
analysis: null,
|
||
availableTools: [],
|
||
plan: null,
|
||
execution: null,
|
||
friendlyMessage: null,
|
||
optimizedPrompt: null,
|
||
recommendedImageParams: null,
|
||
videoPlan: null,
|
||
route: null,
|
||
workflowDecision: null,
|
||
});
|
||
|
||
return {
|
||
analysis: result.analysis ?? undefined,
|
||
plan: result.plan ?? undefined,
|
||
execution: result.execution ?? undefined,
|
||
friendlyMessage: result.friendlyMessage ?? undefined,
|
||
optimizedPrompt: result.optimizedPrompt ?? undefined,
|
||
recommendedImageParams: result.recommendedImageParams ?? undefined,
|
||
videoPlan: result.videoPlan ?? undefined,
|
||
route: result.route ?? undefined,
|
||
threadState: result.threadState ?? undefined,
|
||
workflowDecision: result.workflowDecision ?? undefined,
|
||
};
|
||
}
|