import { Annotation, StateGraph, START, END } from "@langchain/langgraph"; import { generateText } from "ai"; import sharp from "sharp"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { RouteResult } from "@/lib/ai/model-router"; import type { AgentSkillRouteResult } from "@/lib/ai/skill-router"; import { getEnabledAgentToolDefinitionsForContext, type AgentProjectToolDefinition, type AgentProjectToolName, } from "@/lib/ai/tools-registry"; import { runHumanConfirmationGate, type HumanConfirmationGateDecision, } from "@/lib/ai/runtime/middlewares/human-confirmation-gate"; import { normalizeAgentThreadState, type AgentThreadState, } from "@/lib/ai/runtime/agent-thread-state"; import { getAiModelAgentCodingModelId, isAiModelMultimodal, } from "@/lib/db/ai-model-flags"; interface HistoryMessage { role: "user" | "assistant"; content: string; } export interface AgentRecommendedImageParams { aspectRatio?: string; imageSize?: string; imageCount?: number; negativePrompt?: string; promptExtend?: boolean; } export interface AgentRecommendedVideoParams { mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; resolution?: "480p" | "720p" | "1080p"; ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive"; duration?: number; generateAudio?: boolean; cameraMove?: string; } export interface AgentVideoStoryboardShot { shot: number; title: string; duration: number; description: string; prompt: string; camera?: string; } export interface AgentVideoPlan { script: string; storyboard: AgentVideoStoryboardShot[]; finalPrompt: string; negativePrompt?: string; recommendedVideoParams?: AgentRecommendedVideoParams; } export type AgentTargetResolutionMode = | "explicit_selection" | "current_upload" | "text_reference" | "working_memory" | "latest_result" | "clarification" | "none"; export type AgentPreferredCapability = | "respond" | "analyze" | "generic_edit" | "remove_background" | "enhance" | "extend" | "resize" | "box_cutout" | "generate_image" | "generate_video" | "publish_canvas"; export type AgentPlanNextAction = | "respond" | "clarify" | "analyze_image" | "generate_image" | "edit_image" | "generate_video"; export type AgentPlanResponseMode = | "direct_answer" | "tool_then_answer" | "tool_then_execute" | "clarify"; export interface AgentPlannedToolCall { name: AgentProjectToolName; reason?: string; args?: Record; } export interface AgentExecutionPlan { goal: string; nextAction: AgentPlanNextAction; intent: AnalysisResult["intent"]; subIntent?: string; targetSource: AgentTargetResolutionMode; preferredCapability: AgentPreferredCapability; responseMode: AgentPlanResponseMode; shouldUseHistoryImage: boolean; shouldUseStructuredReply: boolean; confidence: number; toolCalls: AgentPlannedToolCall[]; } export interface AgentPreparedExecution { mode: AgentPlanNextAction; routeIntent: AnalysisResult["intent"]; routeSubIntent?: string; targetSource: AgentTargetResolutionMode; preferredCapability: AgentPreferredCapability; shouldUseStructuredReply: boolean; shouldOptimizePrompt: boolean; shouldApplyImagePipeline: boolean; shouldApplyVideoPipeline: boolean; shouldStayConversational: boolean; shouldPublishToCanvas: boolean; toolCalls: AgentPlannedToolCall[]; availableTools: AgentProjectToolName[]; } export interface AutoAgentInput { text: string; hasImage: boolean; imageUrls?: string[]; assetIds?: string[]; imageFromHistory?: boolean; targetResolutionMode?: AgentTargetResolutionMode; mediaSource?: "current_message" | "canvas_selection" | "history" | "none"; hasCanvasSelection?: boolean; selectedImageCount?: number; historyImageCount?: number; currentUploadImageCount?: number; historyMessages?: HistoryMessage[]; context?: string; preferredModelId?: string; threadState?: AgentThreadState | null; } export interface AutoAgentResult { analysis?: AnalysisResult; plan?: AgentExecutionPlan; execution?: AgentPreparedExecution; friendlyMessage?: string; optimizedPrompt?: string; recommendedImageParams?: AgentRecommendedImageParams; videoPlan?: AgentVideoPlan; route?: RouteResult; threadState?: AgentThreadState | null; workflowDecision?: HumanConfirmationGateDecision; } const AutoAgentState = Annotation.Root({ text: Annotation, hasImage: Annotation, imageUrls: Annotation, assetIds: Annotation, imageFromHistory: Annotation, targetResolutionMode: Annotation, mediaSource: Annotation<"current_message" | "canvas_selection" | "history" | "none">, hasCanvasSelection: Annotation, selectedImageCount: Annotation, historyImageCount: Annotation, currentUploadImageCount: Annotation, historyMessages: Annotation, context: Annotation, preferredModelId: Annotation, threadState: Annotation, analysis: Annotation, availableTools: Annotation, plan: Annotation, execution: Annotation, friendlyMessage: Annotation, optimizedPrompt: Annotation, recommendedImageParams: Annotation, videoPlan: Annotation, route: Annotation, workflowDecision: Annotation, }); type AutoAgentStateType = typeof AutoAgentState.State; function compactText(text: string, max = 48) { const normalized = text.replace(/\s+/g, " ").trim(); if (normalized.length <= max) return normalized; return `${normalized.slice(0, max)}...`; } function createPlanGoalSummary(text: string, hasImage: boolean): string { const compact = compactText(text || "", 72); if (compact) return compact; return hasImage ? "基于当前图片继续处理" : "继续当前对话"; } function describeTargetSource(mode: AgentTargetResolutionMode): string { switch (mode) { case "explicit_selection": return "画布选中图"; case "current_upload": return "当前上传图"; case "text_reference": return "用户文本指定图"; case "working_memory": return "会话工作记忆图"; case "latest_result": return "最近生成结果图"; case "clarification": return "目标待澄清"; case "none": default: return "无明确图像目标"; } } function inferPreferredCapability( analysis: AnalysisResult, toolCalls?: AgentPlannedToolCall[], ): AgentPreferredCapability { const firstTool = toolCalls?.[0]?.name; if (firstTool === "removeBackground") return "remove_background"; if (firstTool === "enhanceImage") return "enhance"; if (firstTool === "extendImage") return "extend"; if (firstTool === "resizeImage") return "resize"; if (firstTool === "boxCutout") return "box_cutout"; if (firstTool === "publishToCanvas") return "publish_canvas"; if (firstTool === "generateVideo") return "generate_video"; if (firstTool === "generateImage") return "generate_image"; if (firstTool === "analyzeImage") return "analyze"; if (analysis.intent === "video_gen") return "generate_video"; if (analysis.intent === "image_gen") return "generate_image"; if (analysis.intent === "vision") return "analyze"; if (analysis.intent === "chat") return "respond"; switch (analysis.subIntent) { case "remove_background": return "remove_background"; case "enhance": return "enhance"; case "extend": return "extend"; case "resize": return "resize"; default: return "generic_edit"; } } function defaultToolForAction(action: AgentPlanNextAction): AgentProjectToolName { switch (action) { case "analyze_image": return "analyzeImage"; case "edit_image": return "editImage"; case "generate_image": return "generateImage"; case "generate_video": return "generateVideo"; case "clarify": case "respond": default: return "respondDirect"; } } function mapIntentToAction(intent: AnalysisResult["intent"]): AgentPlanNextAction { switch (intent) { case "vision": return "analyze_image"; case "image_edit": return "edit_image"; case "image_gen": return "generate_image"; case "video_gen": return "generate_video"; case "chat": default: return "respond"; } } function mapActionToIntent( action: AgentPlanNextAction, fallbackIntent: AnalysisResult["intent"], ): AnalysisResult["intent"] { switch (action) { case "analyze_image": return "vision"; case "edit_image": return "image_edit"; case "generate_image": return "image_gen"; case "generate_video": return "video_gen"; case "clarify": case "respond": return fallbackIntent === "vision" ? "vision" : "chat"; default: return fallbackIntent; } } function defaultResponseMode(action: AgentPlanNextAction): AgentPlanResponseMode { switch (action) { case "clarify": return "clarify"; case "respond": return "direct_answer"; case "analyze_image": return "tool_then_answer"; case "edit_image": case "generate_image": case "generate_video": return "tool_then_execute"; default: return "direct_answer"; } } function shouldUseStructuredReply(action: AgentPlanNextAction): boolean { return action === "edit_image" || action === "generate_image" || action === "generate_video"; } function sanitizePlannedToolCalls( rawToolCalls: unknown, availableTools: AgentProjectToolDefinition[], fallbackAction: AgentPlanNextAction, ): AgentPlannedToolCall[] { const availableNames = new Set(availableTools.map((tool) => tool.name)); const list = Array.isArray(rawToolCalls) ? rawToolCalls : []; const sanitized = list .map((item) => { if (!item || typeof item !== "object") return null; const rawName = String((item as { name?: unknown }).name || "").trim() as AgentProjectToolName; if (!availableNames.has(rawName)) return null; const args = (item as { args?: unknown }).args; return { name: rawName, reason: String((item as { reason?: unknown }).reason || "").trim() || undefined, args: args && typeof args === "object" && !Array.isArray(args) ? (args as Record) : undefined, } satisfies AgentPlannedToolCall; }) .filter(Boolean) as AgentPlannedToolCall[]; if (sanitized.length > 0) return sanitized; const fallbackTool = defaultToolForAction(fallbackAction); if (availableNames.has(fallbackTool)) { return [{ name: fallbackTool }]; } if (availableNames.has("respondDirect")) { return [{ name: "respondDirect" }]; } return []; } function buildFallbackPlan(params: { analysis: AnalysisResult; text: string; hasImage: boolean; imageFromHistory: boolean; targetResolutionMode: AgentTargetResolutionMode; availableTools: AgentProjectToolDefinition[]; }): AgentExecutionPlan { const baseAction = params.analysis.needsClarification ? "clarify" : mapIntentToAction(params.analysis.intent); const toolCalls = sanitizePlannedToolCalls([], params.availableTools, baseAction); return { goal: createPlanGoalSummary(params.text, params.hasImage), nextAction: baseAction, intent: mapActionToIntent(baseAction, params.analysis.intent), subIntent: baseAction === "edit_image" ? params.analysis.subIntent || "enhance" : undefined, targetSource: params.targetResolutionMode, preferredCapability: inferPreferredCapability(params.analysis, toolCalls), responseMode: defaultResponseMode(baseAction), shouldUseHistoryImage: params.imageFromHistory, shouldUseStructuredReply: shouldUseStructuredReply(baseAction), confidence: Math.max(params.analysis.confidence || 0.6, 0.68), toolCalls, }; } function alignAnalysisWithPlan( analysis: AnalysisResult, plan: AgentExecutionPlan, ): AnalysisResult { const intent = plan.intent || mapActionToIntent(plan.nextAction, analysis.intent); return { ...analysis, intent, subIntent: intent === "image_edit" ? plan.subIntent || analysis.subIntent : undefined, needsClarification: plan.nextAction === "clarify", clarifyingQuestions: plan.nextAction === "clarify" ? analysis.clarifyingQuestions?.length ? analysis.clarifyingQuestions : ["我先确认一下你希望我下一步执行什么。"] : [], }; } function prepareExecutionFromPlan(params: { plan: AgentExecutionPlan; availableTools: AgentProjectToolDefinition[]; }): AgentPreparedExecution { const availableToolNames = params.availableTools.map((tool) => tool.name); const mode = params.plan.nextAction; return { mode, routeIntent: params.plan.intent, routeSubIntent: params.plan.subIntent, targetSource: params.plan.targetSource, preferredCapability: params.plan.preferredCapability, shouldUseStructuredReply: params.plan.shouldUseStructuredReply, shouldOptimizePrompt: mode === "edit_image" || mode === "generate_image" || mode === "generate_video", shouldApplyImagePipeline: mode === "edit_image" || mode === "generate_image", shouldApplyVideoPipeline: mode === "generate_video", shouldStayConversational: mode === "respond" || mode === "analyze_image" || mode === "clarify", shouldPublishToCanvas: params.plan.toolCalls.some((tool) => tool.name === "publishToCanvas"), toolCalls: params.plan.toolCalls, availableTools: availableToolNames, }; } function buildWorkflowGateBypassPayload( state: AutoAgentStateType, decision: HumanConfirmationGateDecision, ) { const analysis: AnalysisResult = { intent: "chat", confidence: 0.98, needsClarification: decision.action === "awaiting_confirmation", clarifyingQuestions: decision.action === "awaiting_confirmation" ? ["请先确认或修改当前待确认流程,我再继续执行。"] : [], conversationalReply: decision.friendlyMessage, }; const plan: AgentExecutionPlan = { goal: createPlanGoalSummary(state.text, state.hasImage), nextAction: "respond", intent: "chat", targetSource: state.targetResolutionMode, preferredCapability: "respond", responseMode: "direct_answer", shouldUseHistoryImage: false, shouldUseStructuredReply: false, confidence: 0.98, toolCalls: [ { name: "respondDirect", reason: "当前工作流处于人工确认门控阶段,先处理用户确认或修改意图。", }, ], }; const execution = prepareExecutionFromPlan({ plan, availableTools: [], }); return { analysis, plan, execution, friendlyMessage: decision.friendlyMessage || null, workflowDecision: decision, }; } function getEnvInt(name: string, fallback: number, min: number, max: number): number { const raw = process.env[name]; const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN; if (!Number.isFinite(parsed)) return fallback; return Math.max(min, Math.min(max, parsed)); } async function withTimeoutFallback( promise: Promise, timeoutMs: number, fallbackValue: T, ): Promise { let timeoutId: NodeJS.Timeout | undefined; try { return await Promise.race([ promise, new Promise((resolve) => { timeoutId = setTimeout(() => resolve(fallbackValue), timeoutMs); }), ]); } finally { if (timeoutId) clearTimeout(timeoutId); } } const AGENT_VISION_MAX_IMAGES = 4; const AGENT_VISION_MAX_SIDE = 500; const AGENT_OPERATION_TIMEOUT_MS = getEnvInt( "AGENT_OPERATION_TIMEOUT_MS", 18000, 8000, 120000, ); const AGENT_VISION_FETCH_TIMEOUT_MS = getEnvInt( "AGENT_VISION_FETCH_TIMEOUT_MS", 12000, 3000, 60000, ); function isHttpUrl(value: string) { return /^https?:\/\//i.test(value); } async function fetchWithTimeout(url: string, timeoutMs: number): Promise { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeoutMs); try { return await fetch(url, { signal: controller.signal, cache: "no-store", headers: { "user-agent": "ai-chatbot-auto-agent/1.0", }, }); } finally { clearTimeout(timeoutId); } } async function toAgentVisionDataUrl(imageUrl: string): Promise { try { const response = await fetchWithTimeout(imageUrl, AGENT_VISION_FETCH_TIMEOUT_MS); if (!response.ok) { console.warn( `[Auto Agent] Failed to fetch image for inline vision payload (${response.status}): ${imageUrl}`, ); return null; } const contentType = response.headers.get("content-type") || ""; if (!contentType.startsWith("image/")) { console.warn(`[Auto Agent] Skip non-image content-type for vision payload: ${contentType}`); return null; } const arrayBuffer = await response.arrayBuffer(); const optimized = await sharp(Buffer.from(arrayBuffer)) .rotate() .resize({ width: AGENT_VISION_MAX_SIDE, height: AGENT_VISION_MAX_SIDE, fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 80, mozjpeg: true, }) .toBuffer(); return `data:image/jpeg;base64,${optimized.toString("base64")}`; } catch (error) { console.warn("[Auto Agent] Failed to convert image URL to inline base64:", error); return null; } } async function buildAgentVisionInputs( imageUrls: string[], options?: { preferRemoteUrls?: boolean; }, ) { if (imageUrls.length === 0) return [] as Array<{ type: "image"; image: string }>; if (options?.preferRemoteUrls) { return imageUrls.map((url) => ({ type: "image" as const, image: url, })); } const convertedResults = await Promise.all(imageUrls.map((url) => toAgentVisionDataUrl(url))); const inlineCount = convertedResults.filter(Boolean).length; console.log( `[Auto Agent] Prepared vision images: total=${imageUrls.length}, inline=${inlineCount}, remote=${imageUrls.length - inlineCount}, maxSide=${AGENT_VISION_MAX_SIDE}`, ); return imageUrls.map((originalUrl, index) => ({ type: "image" as const, image: convertedResults[index] || originalUrl, })); } function normalizeAspectRatio(value?: string): string | undefined { if (!value) return undefined; const normalized = value.trim().toLowerCase(); if (normalized === "auto") return "auto"; const raw = value.trim(); const allowed = new Set([ "1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9", ]); return allowed.has(raw) ? raw : undefined; } function normalizeImageSize(value?: string): string | undefined { if (!value) return undefined; const normalized = value.trim().toUpperCase(); if (normalized === "1K" || normalized === "2K" || normalized === "4K") { return normalized; } return undefined; } function normalizeVideoMode(value?: string): AgentRecommendedVideoParams["mode"] | undefined { if (!value) return undefined; if (value === "text-to-video") return "text-to-video"; if (value === "first-frame") return "first-frame"; if (value === "first-last-frame") return "first-last-frame"; if (value === "reference-images") return "reference-images"; return undefined; } function normalizeVideoResolution( value?: string, ): AgentRecommendedVideoParams["resolution"] | undefined { if (!value) return undefined; const normalized = value.toLowerCase(); if (normalized === "480p") return "480p"; if (normalized === "720p") return "720p"; if (normalized === "1080p") return "1080p"; return undefined; } function normalizeVideoRatio(value?: string): AgentRecommendedVideoParams["ratio"] | undefined { if (!value) return undefined; const normalized = value.trim(); const allowed = new Set(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"]); return allowed.has(normalized) ? (normalized as AgentRecommendedVideoParams["ratio"]) : undefined; } function normalizeVideoDuration(value?: number): number | undefined { if (typeof value !== "number" || !Number.isFinite(value)) return undefined; return Math.min(15, Math.max(3, Math.round(value))); } function sanitizeRecommendedVideoParams( params: AgentRecommendedVideoParams | undefined, ): AgentRecommendedVideoParams | undefined { if (!params) return undefined; const safe: AgentRecommendedVideoParams = {}; const mode = normalizeVideoMode(params.mode); const resolution = normalizeVideoResolution(params.resolution); const ratio = normalizeVideoRatio(params.ratio); const duration = normalizeVideoDuration(params.duration); if (mode) safe.mode = mode; if (resolution) safe.resolution = resolution; if (ratio) safe.ratio = ratio; if (duration) safe.duration = duration; if (typeof params.generateAudio === "boolean") safe.generateAudio = params.generateAudio; if (typeof params.cameraMove === "string" && params.cameraMove.trim()) { safe.cameraMove = params.cameraMove.trim().slice(0, 32); } return Object.keys(safe).length > 0 ? safe : undefined; } function inferVideoParamsByText(text: string, hasImage: boolean): AgentRecommendedVideoParams { const normalized = text.toLowerCase(); const durationMatch = normalized.match(/(\d{1,2})\s*秒/); const duration = durationMatch ? normalizeVideoDuration(Number(durationMatch[1])) : undefined; const mode = /首尾帧|首帧.*尾帧|结尾帧|尾帧/.test(normalized) ? "first-last-frame" : hasImage ? "first-frame" : "text-to-video"; const ratio = /竖屏|手机|9:16|竖版/.test(normalized) ? "9:16" : /方形|方图|1:1/.test(normalized) ? "1:1" : /超宽|21:9|电影感/.test(normalized) ? "21:9" : "16:9"; const generateAudio = /配音|旁白|音效|有声|带声音/.test(normalized); return { mode, ratio, duration: duration || 5, resolution: "720p", generateAudio, }; } function buildVideoFinalPrompt(params: { script: string; storyboard: AgentVideoStoryboardShot[]; originalText: string; }): string { const script = params.script.trim(); const storyboard = params.storyboard .slice(0, 8) .map( (shot) => `镜头${shot.shot}(${shot.duration}s): ${shot.prompt || shot.description || shot.title}`.trim() ) .join("\n"); return [ `【用户目标】${params.originalText.trim() || "生成视频"}`, script ? `【脚本】${script}` : "", storyboard ? `【分镜】\n${storyboard}` : "", "【执行要求】镜头衔接自然,人物与主体保持一致,动作清晰,避免突兀跳切。", ] .filter(Boolean) .join("\n\n") .trim(); } function heuristicVideoPlan(state: AutoAgentStateType): AgentVideoPlan { const recommendedVideoParams = inferVideoParamsByText(state.text, state.hasImage); const duration = recommendedVideoParams.duration || 5; const perShot = Math.max(2, Math.round(duration / 3)); const storyboard: AgentVideoStoryboardShot[] = [ { shot: 1, title: "开场建立", duration: perShot, description: "建立主体与环境关系,明确视觉基调。", prompt: `开场镜头,展示主体与环境,围绕「${compactText(state.text, 50)}」建立氛围。`, camera: "稳定推进", }, { shot: 2, title: "动作推进", duration: perShot, description: "主体动作或情绪推进,形成主要叙事段落。", prompt: `中段镜头,突出主体动作与情绪变化,强化故事推进。`, camera: "跟拍或环绕", }, { shot: 3, title: "收束结尾", duration: Math.max(2, duration - perShot * 2), description: "情绪收束并输出最终视觉记忆点。", prompt: "结尾镜头,画面收束,保留主体识别与情绪余韵。", camera: "缓慢拉远", }, ]; const script = `围绕“${compactText(state.text, 80)}”制作一条节奏清晰、主体连续的短视频,包含开场建立、过程推进与结尾收束三段。`; return { script, storyboard, finalPrompt: buildVideoFinalPrompt({ script, storyboard, originalText: state.text, }), recommendedVideoParams, }; } function sanitizeVideoPlan(raw: unknown, state: AutoAgentStateType): AgentVideoPlan { if (!raw || typeof raw !== "object") return heuristicVideoPlan(state); const payload = raw as Record; const recommendedVideoParams = sanitizeRecommendedVideoParams(payload.recommendedVideoParams); const script = typeof payload.script === "string" && payload.script.trim() ? payload.script.trim().slice(0, 400) : `围绕“${compactText(state.text, 80)}”生成视频。`; const storyboardInput = Array.isArray(payload.storyboard) ? payload.storyboard : []; const storyboard: AgentVideoStoryboardShot[] = storyboardInput .map((item: any, idx) => { const shot = Number.isFinite(item?.shot) ? Math.max(1, Math.round(item.shot)) : idx + 1; const duration = normalizeVideoDuration(Number(item?.duration)) || 3; const title = typeof item?.title === "string" && item.title.trim() ? item.title.trim().slice(0, 24) : `镜头${shot}`; const description = typeof item?.description === "string" && item.description.trim() ? item.description.trim().slice(0, 180) : title; const prompt = typeof item?.prompt === "string" && item.prompt.trim() ? item.prompt.trim().slice(0, 260) : description; const camera = typeof item?.camera === "string" && item.camera.trim() ? item.camera.trim().slice(0, 24) : undefined; return { shot, title, duration, description, prompt, camera }; }) .filter((item) => item.prompt); const fallback = heuristicVideoPlan(state); const safeStoryboard = storyboard.length > 0 ? storyboard.slice(0, 8) : fallback.storyboard; const finalPrompt = typeof payload.finalPrompt === "string" && payload.finalPrompt.trim() ? payload.finalPrompt.trim().slice(0, 1400) : buildVideoFinalPrompt({ script, storyboard: safeStoryboard, originalText: state.text, }); const negativePrompt = typeof payload.negativePrompt === "string" && payload.negativePrompt.trim() ? payload.negativePrompt.trim().slice(0, 360) : undefined; return { script, storyboard: safeStoryboard, finalPrompt, negativePrompt, recommendedVideoParams: recommendedVideoParams || fallback.recommendedVideoParams, }; } function sanitizeRecommendedImageParams( params: AgentRecommendedImageParams | undefined, intent?: AnalysisResult["intent"], ): AgentRecommendedImageParams | undefined { if (!params) return undefined; const safe: AgentRecommendedImageParams = {}; const normalizedAspectRatio = normalizeAspectRatio(params.aspectRatio); const normalizedImageSize = normalizeImageSize(params.imageSize); if (normalizedAspectRatio) safe.aspectRatio = normalizedAspectRatio; if (normalizedImageSize) safe.imageSize = normalizedImageSize; if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) { const min = 1; const max = intent === "image_edit" ? 1 : 12; safe.imageCount = Math.min(Math.max(Math.round(params.imageCount), min), max); } if (intent === "image_edit") { safe.imageCount = 1; } if (typeof params.negativePrompt === "string" && params.negativePrompt.trim()) { safe.negativePrompt = params.negativePrompt.trim().slice(0, 500); } if (typeof params.promptExtend === "boolean") { safe.promptExtend = params.promptExtend; } if (intent === "image_edit") { safe.promptExtend = false; } return Object.keys(safe).length > 0 ? safe : undefined; } function heuristicImageParams( text: string, analysis: AnalysisResult, ): AgentRecommendedImageParams | undefined { const normalized = text.toLowerCase(); const params: AgentRecommendedImageParams = { imageSize: "2K", promptExtend: analysis.intent === "image_edit" ? false : true, }; if (analysis.intent === "image_edit") { params.imageCount = 1; } else { const explicitCountMatch = normalized.match(/([1-9]\d?)张/); const explicitCount = explicitCountMatch ? Number(explicitCountMatch[1]) : 0; if (explicitCount >= 1) { params.imageCount = Math.min(explicitCount, 12); } else if (/多张|四张|4张|三张|3张/.test(normalized)) { params.imageCount = /四张|4张/.test(normalized) ? 4 : 3; } else if (/两张|2张|双图/.test(normalized)) { params.imageCount = 2; } else { params.imageCount = 1; } } if (/横图|横版|宽屏|封面|banner|海报/.test(normalized)) { params.aspectRatio = "16:9"; } else if (/竖图|竖版|手机壁纸|全身|海报竖版/.test(normalized)) { params.aspectRatio = "9:16"; } else if (/方图|方形|头像|logo|电商主图/.test(normalized)) { params.aspectRatio = "1:1"; } else { params.aspectRatio = "auto"; } return sanitizeRecommendedImageParams(params, analysis.intent); } function buildStrategicFriendlyMessage( analysis: AnalysisResult, state: AutoAgentStateType, ): Promise { const fallbackQuestion = analysis.clarifyingQuestions?.[0]; const existingReply = analysis.conversationalReply?.trim(); if (analysis.needsClarification) { return Promise.resolve( analysis.conversationalReply || fallbackQuestion || "我先确认一下,你希望我优先调整哪一部分?" ); } if (existingReply) { const isStructuredReply = /###\s*需求理解/.test(existingReply) || /###\s*图片分析/.test(existingReply) || /###\s*执行计划/.test(existingReply) || /【\s*需求理解\s*】/.test(existingReply) || /【\s*图片分析\s*】/.test(existingReply) || /【\s*执行计划\s*】/.test(existingReply); if (isStructuredReply || existingReply.length >= 24) { return Promise.resolve(existingReply); } } return withTimeoutFallback( generateAgentConversation(analysis, state), AGENT_OPERATION_TIMEOUT_MS, existingReply || null, ); } type EnabledModelWithProvider = Awaited< ReturnType >[number]; function isAgentGenerateTextProviderType(providerType: string): boolean { return ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType); } function isLikelyVisionCapableModel(item: EnabledModelWithProvider) { if (!item.model.isEnabled || item.model.isSubModel) return false; return Boolean(item.model.supportsVision) || isAiModelMultimodal(item.model); } function getAgentModelIncompatibleReason(params: { item: EnabledModelWithProvider; requireVision: boolean; }): string | null { const providerType = String(params.item.provider.type || ""); if (!params.item.model.isEnabled || params.item.model.isSubModel) { return "disabled_or_submodel"; } if (!isAgentGenerateTextProviderType(providerType)) { return `unsupported_provider:${providerType || "unknown"}`; } if (params.requireVision && !isLikelyVisionCapableModel(params.item)) { return "vision_capability_missing"; } return null; } function isAgentGenerateTextCompatibleModel(params: { item: EnabledModelWithProvider; requireVision: boolean; }): boolean { return !getAgentModelIncompatibleReason(params); } function shouldPreferRemoteVisionUrls(item?: EnabledModelWithProvider) { if (!item) return false; const providerType = String(item.provider.type || ""); const baseUrl = String((item.provider as { baseUrl?: string | null })?.baseUrl || "").toLowerCase(); const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase(); if ( providerType === "volcano-multimodal" || providerType === "doubao-video" || providerType === "volcano-video" || providerType === "ruxa-video" ) { return true; } if (baseUrl.includes("ark.cn-beijing.volces.com")) { return true; } if (/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText)) { return true; } return false; } function pickQwenVisionModel(models: EnabledModelWithProvider[]) { const visionModels = models.filter((item) => isAgentGenerateTextCompatibleModel({ item, requireVision: true }), ); const zhipuVision = visionModels.find( (item) => item.provider.type === "zhipu" || /(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test( `${item.model.displayName} ${item.model.modelId}`, ), ); if (zhipuVision) return zhipuVision; const qwenByName = visionModels.find( (item) => item.provider.type === "alibaba" && /(qwen|通义|vl)/i.test(`${item.model.displayName} ${item.model.modelId}`) ); if (qwenByName) return qwenByName; const alibabaVision = visionModels.find((item) => item.provider.type === "alibaba"); if (alibabaVision) return alibabaVision; return visionModels[0]; } function pickAgentVisionModel( agentModel: EnabledModelWithProvider | undefined, models: EnabledModelWithProvider[], ) { if ( agentModel && isAgentGenerateTextCompatibleModel({ item: agentModel, requireVision: true }) ) { return agentModel; } return pickQwenVisionModel(models); } function resolvePreferredModel( preferredModelId: string | null | undefined, models: EnabledModelWithProvider[], ) { const normalizedId = String(preferredModelId || "").trim(); if (!normalizedId) return undefined; return models.find( (item) => item.model.id === normalizedId || item.model.modelId === normalizedId, ); } function resolvePrimaryAgentModel(params: { preferredModelId?: string | null; configuredAgentModel?: EnabledModelWithProvider | null; allModels: EnabledModelWithProvider[]; }) { const preferred = resolvePreferredModel(params.preferredModelId, params.allModels); if (preferred) return preferred; return params.configuredAgentModel || undefined; } function pickAgentReasoningModel( agentModel: EnabledModelWithProvider | undefined, models: EnabledModelWithProvider[], ) { if ( agentModel && isAgentGenerateTextCompatibleModel({ item: agentModel, requireVision: false }) ) { return agentModel; } const candidates = models.filter((item) => isAgentGenerateTextCompatibleModel({ item, requireVision: false }), ); const zhipuModel = candidates.find( (item) => item.provider.type === "zhipu" && item.model.isEnabled && !item.model.isSubModel ); if (zhipuModel) return zhipuModel; return candidates[0]; } function parseJsonBlock>(text: string): T | null { const match = text.match(/\{[\s\S]*\}/); if (!match) return null; try { return JSON.parse(match[0]) as T; } catch { return null; } } function resolveAgentCodingModelOverride( selectedModelId: string, configuredAgentModel: | { model?: { id?: string | null; defaultParams?: unknown; capabilities?: unknown; } | null; } | null | undefined, ): string | null { const configuredId = configuredAgentModel?.model?.id || null; if (!configuredId || configuredId !== selectedModelId) { return null; } return getAiModelAgentCodingModelId(configuredAgentModel?.model || {}) || null; } function sanitizeAgentPlan( raw: unknown, fallbackPlan: AgentExecutionPlan, availableTools: AgentProjectToolDefinition[], ): AgentExecutionPlan { if (!raw || typeof raw !== "object") { return fallbackPlan; } const payload = raw as Record; const rawAction = String(payload.nextAction || "").trim() as AgentPlanNextAction; const nextAction: AgentPlanNextAction = [ "respond", "clarify", "analyze_image", "generate_image", "edit_image", "generate_video", ].includes(rawAction) ? rawAction : fallbackPlan.nextAction; const fallbackIntent = fallbackPlan.intent; const rawIntent = String(payload.intent || "").trim() as AnalysisResult["intent"]; const intent: AnalysisResult["intent"] = [ "chat", "vision", "image_edit", "image_gen", "video_gen", ].includes(rawIntent) ? rawIntent : mapActionToIntent(nextAction, fallbackIntent); const rawResponseMode = String(payload.responseMode || "").trim() as AgentPlanResponseMode; const responseMode: AgentPlanResponseMode = [ "direct_answer", "tool_then_answer", "tool_then_execute", "clarify", ].includes(rawResponseMode) ? rawResponseMode : defaultResponseMode(nextAction); const confidenceNumber = Number(payload.confidence); const confidence = Number.isFinite(confidenceNumber) ? Math.max(0.3, Math.min(0.99, confidenceNumber)) : fallbackPlan.confidence; const goal = typeof payload.goal === "string" && payload.goal.trim() ? payload.goal.trim().slice(0, 160) : fallbackPlan.goal; const rawSubIntent = typeof payload.subIntent === "string" && payload.subIntent.trim() ? payload.subIntent.trim().slice(0, 48) : undefined; const rawTargetSource = String(payload.targetSource || "").trim() as AgentTargetResolutionMode; const targetSource: AgentTargetResolutionMode = [ "explicit_selection", "current_upload", "text_reference", "working_memory", "latest_result", "clarification", "none", ].includes(rawTargetSource) ? rawTargetSource : fallbackPlan.targetSource; const sanitizedToolCalls = sanitizePlannedToolCalls(payload.toolCalls, availableTools, nextAction); const rawCapability = String(payload.preferredCapability || "").trim() as AgentPreferredCapability; const derivedAnalysis: AnalysisResult = { intent, subIntent: intent === "image_edit" ? rawSubIntent || fallbackPlan.subIntent : undefined, confidence, }; const preferredCapability: AgentPreferredCapability = [ "respond", "analyze", "generic_edit", "remove_background", "enhance", "extend", "resize", "box_cutout", "generate_image", "generate_video", "publish_canvas", ].includes(rawCapability) ? rawCapability : inferPreferredCapability(derivedAnalysis, sanitizedToolCalls); return { goal, nextAction, intent, subIntent: intent === "image_edit" ? rawSubIntent || fallbackPlan.subIntent : undefined, targetSource, preferredCapability, responseMode, shouldUseHistoryImage: typeof payload.shouldUseHistoryImage === "boolean" ? payload.shouldUseHistoryImage : fallbackPlan.shouldUseHistoryImage, shouldUseStructuredReply: typeof payload.shouldUseStructuredReply === "boolean" ? payload.shouldUseStructuredReply : shouldUseStructuredReply(nextAction), confidence, toolCalls: sanitizedToolCalls, }; } async function buildExecutionPlanWithAgent( state: AutoAgentStateType, analysis: AnalysisResult, availableTools: AgentProjectToolDefinition[], skillRoute?: AgentSkillRouteResult, ): Promise { const fallbackPlan = buildFallbackPlan({ analysis, text: state.text, hasImage: state.hasImage, imageFromHistory: state.imageFromHistory, targetResolutionMode: state.targetResolutionMode, availableTools, }); try { const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); const primaryAgentModel = resolvePrimaryAgentModel({ preferredModelId: state.preferredModelId, configuredAgentModel: agentModel, allModels, }); const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels); if (!plannerModelConfig) { return fallbackPlan; } const plannerModel = await getLanguageModelById( plannerModelConfig.model.id, resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel), ); if (!plannerModel) { return fallbackPlan; } const toolCatalog = availableTools .map((tool) => `- ${tool.name}: ${tool.title}|${tool.description}`) .join("\n"); const historyContext = (state.historyMessages || []) .slice(-6) .map((item) => `${item.role}: ${compactText(item.content || "", 120)}`) .join("\n"); const { text } = await generateText({ model: plannerModel, system: `你是一个多模态 Agent Planner。 你的职责不是直接回答,而是决定当前回合下一步该走哪条能力路径。 只输出 JSON,不要输出解释,不要输出 markdown。 可选 nextAction: - respond:纯对话、策略讨论、解释说明、普通问答 - clarify:信息不足,需要先澄清 - analyze_image:用户要你看图、识图、评估、分析、给建议 - edit_image:基于当前图/历史图做修改、抠图、增强、换背景、调色、尺寸调整、局部重绘 - generate_image:生成新图 - generate_video:生成视频 规则: 1. 用户只是聊天、问图、讨论方案时,绝不能走生图/生视频。 2. 只有用户明确要求“生成/修改/做视频”等执行任务时,才走执行型 action。 3. 如果用户上传了图片但只是问“这是什么/适不适合/帮我分析一下”,应选择 analyze_image 或 respond。 4. 优先保持一次只选择一个主工具;toolCalls 只从提供的工具列表里选。 5. shouldUseStructuredReply 只在 edit_image / generate_image / generate_video 时设为 true。 6. targetSource 要忠实反映当前目标图来源:explicit_selection/current_upload/text_reference/working_memory/latest_result/clarification/none。 7. preferredCapability 要尽量具体,例如 remove_background / enhance / extend / resize / box_cutout / publish_canvas。 8. 必须遵守当前专家路由约束;如果约束要求纯对话,则绝不能输出 generate_image / edit_image / generate_video。 输出格式: { "goal": "一句话概括当前回合目标", "nextAction": "respond|clarify|analyze_image|edit_image|generate_image|generate_video", "intent": "chat|vision|image_edit|image_gen|video_gen", "subIntent": "可选,仅 image_edit 时填写", "targetSource": "explicit_selection|current_upload|text_reference|working_memory|latest_result|clarification|none", "preferredCapability": "respond|analyze|generic_edit|remove_background|enhance|extend|resize|box_cutout|generate_image|generate_video|publish_canvas", "responseMode": "direct_answer|tool_then_answer|tool_then_execute|clarify", "shouldUseHistoryImage": true, "shouldUseStructuredReply": false, "confidence": 0.92, "toolCalls": [ { "name": "analyzeImage", "reason": "为什么选这个工具", "args": {} } ] }`, prompt: [ `用户输入:${state.text || "(空)"}`, `有当前或历史图片:${state.hasImage ? "是" : "否"}`, `图片来自历史:${state.imageFromHistory ? "是" : "否"}`, `目标图来源:${state.targetResolutionMode}(${describeTargetSource(state.targetResolutionMode)})`, `媒体输入来源:${state.mediaSource}`, `当前上传图数量:${state.currentUploadImageCount}`, `历史图数量:${state.historyImageCount}`, `画布选中图数量:${state.selectedImageCount}`, `基础分析:intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`, skillRoute ? `专家路由:${skillRoute.primary.specialistId}|原因:${skillRoute.reason}` : "", skillRoute?.planningHints?.length ? `专家约束:\n${skillRoute.planningHints.map((item, index) => `${index + 1}. ${item}`).join("\n")}` : "", state.threadState?.workflowState ? `当前工作流:type=${state.threadState.workflowState.type}; status=${state.threadState.workflowState.status}; summary=${compactText(state.threadState.workflowState.summary || "", 240)}` : "", state.context ? `画布上下文:${compactText(state.context, 600)}` : "", historyContext ? `最近对话:\n${historyContext}` : "", `可用工具:\n${toolCatalog}`, ] .filter(Boolean) .join("\n\n"), temperature: 0.1, maxTokens: 420, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); const parsed = parseJsonBlock>(text); let planned = sanitizeAgentPlan(parsed, fallbackPlan, availableTools); if (skillRoute) { const { refinePlanWithSpecialist } = await import("./specialists"); for (const specialist of [skillRoute.primary, ...skillRoute.overlays]) { planned = refinePlanWithSpecialist(specialist, { text: state.text, analysis, plan: planned, availableTools, hasImage: state.hasImage, hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0, targetResolutionMode: state.targetResolutionMode, mediaSource: state.mediaSource, selectedImageCount: state.selectedImageCount, threadState: state.threadState, }); } } console.log( `[Auto Agent] Planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})`, ); console.log("[Auto Agent] Execution plan:", planned); return planned; } catch (error) { console.error("[Auto Agent] Planner failed, fallback to heuristic plan:", error); return fallbackPlan; } } async function generateAgentConversation( analysis: AnalysisResult, state: AutoAgentStateType ): Promise { try { const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); const primaryAgentModel = resolvePrimaryAgentModel({ preferredModelId: state.preferredModelId, configuredAgentModel: agentModel, allModels, }); if (primaryAgentModel && state.preferredModelId) { console.log( `[Auto Agent] Preferred analysis model: ${primaryAgentModel.model.displayName} (${primaryAgentModel.provider.type})`, ); } if (primaryAgentModel) { const reasoningIncompatibleReason = getAgentModelIncompatibleReason({ item: primaryAgentModel, requireVision: false, }); if (reasoningIncompatibleReason) { console.warn( `[Auto Agent] Preferred model incompatible for reasoning, skip direct use: ${primaryAgentModel.model.displayName} (${primaryAgentModel.provider.type}), reason=${reasoningIncompatibleReason}`, ); } } const reasoningModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels); if (!reasoningModelConfig) { return analysis.conversationalReply || null; } const reasoningModel = await getLanguageModelById( reasoningModelConfig.model.id, resolveAgentCodingModelOverride(reasoningModelConfig.model.id, agentModel), ); if (!reasoningModel) { return analysis.conversationalReply || null; } const history = state.historyMessages .slice(-6) .map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`) .join("\n"); const validImageUrls = state.imageUrls .filter((url) => isHttpUrl(url)) .slice(0, AGENT_VISION_MAX_IMAGES); const imageInfo = state.hasImage ? `用户本轮可用图片数量:${state.imageUrls.length || 1}。图片来源:${state.imageFromHistory ? "历史消息复用" : "本轮上传"}。` : "用户本轮没有上传图片。"; let visualAnalysis = ""; if (state.hasImage && validImageUrls.length > 0) { try { const visionModelConfig = pickAgentVisionModel(primaryAgentModel, allModels); if (!visionModelConfig) { throw new Error("missing_vision_model"); } const visionModel = await getLanguageModelById(visionModelConfig.model.id); if (!visionModel) { throw new Error("missing_vision_runtime"); } const visionPrompt = `请分析以下参考图片,给出可用于生图/修图执行前决策的结果。 要求: 1) 每张图输出 1 行,格式:图X:主体...;风格...;构图...;光影与色调...;材质/氛围... 2) 只写客观可见信息,不要编造品牌、尺寸、镜头参数。 3) 全部输出控制在 6 行内。`; const preferRemoteVisionUrls = shouldPreferRemoteVisionUrls(visionModelConfig); const visionInputs = await buildAgentVisionInputs(validImageUrls, { preferRemoteUrls: preferRemoteVisionUrls, }); const { text: visionText } = await generateText({ model: visionModel, messages: [ { role: "user" as const, content: [{ type: "text" as const, text: visionPrompt }, ...visionInputs], }, ], temperature: 0.2, maxTokens: 360, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); visualAnalysis = visionText.trim(); console.log( `[Auto Agent] Vision analysis model: ${visionModelConfig.model.displayName} (${visionModelConfig.provider.type}), preferRemote=${preferRemoteVisionUrls}` ); } catch (error) { console.warn("[Auto Agent] Vision analysis failed, continue without visual summary"); visualAnalysis = ""; } } const { buildAgentRoleIdentityInstruction } = await import( "@/lib/ai/runtime/agent-role-profile" ); const identityInstruction = await buildAgentRoleIdentityInstruction({ allowVideoGeneration: true, hasCanvasContext: Boolean(state.context), }); const systemPrompt = `${identityInstruction} 你是一个真实执行中的 AI 生图/修图 Agent。你的核心目标是提升用户一次生成成功率。 你必须按以下 AGENT SKILLS 执行: 1) 需求拆解:把用户目标拆成可执行指令,优先满足用户明确要求。 2) 风险预判:主动识别可能导致失败的歧义(主体数量、视角、风格冲突、文字内容)。 3) 提示词策略:给出可落地的正向约束和必要的负向约束,避免空泛描述。 4) 参数策略:在输出设置中明确建议宽高比、清晰度档位、出图数量(若用户未指定)。 输出要求: - 不要说“看不到图片/无法访问图片”。 - 不暴露内部实现、模型路由、算法细节。 - 输出控制在 8-14 行,标题行不要使用 # 号。 - 有图片时必须按顺序输出以下标题(逐字一致): 【需求理解】 【图片分析】 【执行计划】 【输出设置】 - “图片分析”必须基于视觉分析结果写具体观察,不允许写泛化空话。 - 只有明确拿到原图分辨率时,才能写具体像素尺寸;否则只写宽高比/清晰度目标。`; const prompt = [ `【用户原始需求】${state.text || "(无文本)"}`, `【意图分析】intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`, `【图片信息】${imageInfo}`, visualAnalysis ? `【视觉模型预分析】\n${visualAnalysis}` : "", state.context ? `【画布上下文】${compactText(state.context, 300)}` : "", history ? `【最近对话】\n${history}` : "", state.hasImage ? "请先做本轮图片分析,再输出执行前汇报。" : "请输出本轮执行前汇报。", ] .filter(Boolean) .join("\n\n"); const { text } = await generateText({ model: reasoningModel, system: systemPrompt, prompt, temperature: 0.35, maxTokens: 520, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); const cleaned = text.trim(); if (cleaned.length < 10) { return analysis.conversationalReply || null; } console.log( `[Auto Agent] Conversation model: ${reasoningModelConfig.model.displayName} (${reasoningModelConfig.provider.type})` ); return cleaned; } catch (error) { return analysis.conversationalReply || null; } } async function recommendImageParamsWithAgent( analysis: AnalysisResult, state: AutoAgentStateType, ): Promise { if (analysis.intent !== "image_edit" && analysis.intent !== "image_gen") { return undefined; } try { const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); const primaryAgentModel = resolvePrimaryAgentModel({ preferredModelId: state.preferredModelId, configuredAgentModel: agentModel, allModels, }); const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels); if (!plannerModelConfig) { return heuristicImageParams(state.text, analysis); } const plannerModel = await getLanguageModelById( plannerModelConfig.model.id, resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel), ); if (!plannerModel) { return heuristicImageParams(state.text, analysis); } const history = state.historyMessages .slice(-4) .map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`) .join("\n"); const requestText = [ `用户请求:${state.text || "(无文本)"}`, `识别意图:${analysis.intent}/${analysis.subIntent || "none"}`, `是否有参考图:${state.hasImage ? "是" : "否"}`, history ? `最近对话:\n${history}` : "", "请给出本次最稳妥的生图参数建议。", ] .filter(Boolean) .join("\n\n"); const { text } = await generateText({ model: plannerModel, system: `你是生图参数规划 Agent。目标是提升首图命中率和稳定性。 只输出 JSON,不要输出解释。格式如下: { "aspectRatio": "1:1|2:3|3:2|3:4|4:3|4:5|5:4|9:16|16:9|21:9|auto", "imageSize": "1K|2K|4K", "imageCount": 1, "negativePrompt": "可选,简洁", "promptExtend": true } 规则: - image_edit 场景强制 imageCount=1。 - 若用户未指定比例,优先 auto。 - negativePrompt 仅在明显需要避坑时填写。`, prompt: requestText, temperature: 0.15, maxTokens: 220, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); const parsed = parseJsonBlock(text); const sanitized = sanitizeRecommendedImageParams(parsed || undefined, analysis.intent); if (sanitized) { console.log( `[Auto Agent] Parameter planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})` ); return sanitized; } return heuristicImageParams(state.text, analysis); } catch { return heuristicImageParams(state.text, analysis); } } async function recommendVideoPlanWithAgent( state: AutoAgentStateType, ): Promise { try { const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); const primaryAgentModel = resolvePrimaryAgentModel({ preferredModelId: state.preferredModelId, configuredAgentModel: agentModel, allModels, }); const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels); if (!plannerModelConfig) { return heuristicVideoPlan(state); } const plannerModel = await getLanguageModelById( plannerModelConfig.model.id, resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel), ); if (!plannerModel) { return heuristicVideoPlan(state); } const history = state.historyMessages .slice(-6) .map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`) .join("\n"); const prompt = [ `用户请求:${state.text || "生成视频"}`, `是否有参考图:${state.hasImage ? "是" : "否"}(${state.imageUrls.length} 张)`, history ? `最近对话:\n${history}` : "", "请产出可直接用于视频生成的脚本与分镜。", ] .filter(Boolean) .join("\n\n"); const { text } = await generateText({ model: plannerModel, system: `你是视频生成 Agent 规划器,目标是最大化一次出片成功率。 只输出 JSON,不要输出解释。格式: { "script": "120字内剧情脚本", "recommendedVideoParams": { "mode": "text-to-video|first-frame|first-last-frame|reference-images", "resolution": "480p|720p|1080p", "ratio": "16:9|4:3|1:1|3:4|9:16|21:9|adaptive", "duration": 5, "generateAudio": false, "cameraMove": "none|static|handheld|dynamic-orbit" }, "storyboard": [ { "shot": 1, "title": "镜头标题", "duration": 3, "description": "画面描述", "camera": "运镜方式", "prompt": "该镜头可直接喂给视频模型的提示词" } ], "negativePrompt": "可选", "finalPrompt": "整合后的最终视频提示词" } 要求: - 分镜 2~6 个,时长总计与 duration 接近。 - 有参考图时优先 first-frame 或 first-last-frame。 - 避免抽象空话,必须可执行。`, prompt, temperature: 0.2, maxTokens: 720, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); const parsed = parseJsonBlock>(text); const plan = sanitizeVideoPlan(parsed, state); console.log( `[Auto Agent] Video planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})` ); return plan; } catch { return heuristicVideoPlan(state); } } async function refineRouteWithAgent( state: AutoAgentStateType, baseRoute: RouteResult, ): Promise { if (!state.analysis || (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen")) { return baseRoute; } try { const { db } = await import("@/lib/db"); const { siteSettings } = await import("@/lib/db/schema"); const [settings] = await db.select().from(siteSettings).limit(1); const configuredDefaultModelId = settings?.canvasAutoModeDefaultModelId || null; if (configuredDefaultModelId && baseRoute.id === configuredDefaultModelId) { console.log( `[Auto Agent] Keep admin default model for auto mode: ${baseRoute.displayName}` ); return baseRoute; } const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const { isAiModelAgentAuto } = await import("@/lib/db/ai-model-flags"); const allModelsRaw = await getEnabledAiModelsWithProvider(false); const allModels = allModelsRaw.filter((item) => !isAiModelAgentAuto(item.model)); const imageCandidates = allModels .filter((item) => item.model.supportsImageGen && item.model.isEnabled && !item.model.isSubModel) .slice(0, 12); if (imageCandidates.length <= 1) return baseRoute; const agentModel = await getAgentAiModelWithProvider(); const primaryAgentModel = resolvePrimaryAgentModel({ preferredModelId: state.preferredModelId, configuredAgentModel: agentModel, allModels: allModelsRaw, }); const plannerModelConfig = pickAgentReasoningModel(primaryAgentModel, allModelsRaw); if (!plannerModelConfig) return baseRoute; const plannerModel = await getLanguageModelById( plannerModelConfig.model.id, resolveAgentCodingModelOverride(plannerModelConfig.model.id, agentModel), ); if (!plannerModel) return baseRoute; const candidatesText = imageCandidates .map((item, index) => `${index + 1}. id=${item.model.id}; 名称=${item.model.displayName}; provider=${item.provider.type}`) .join("\n"); const { text } = await generateText({ model: plannerModel, system: `你是生图模型选择 Agent。目标是优先提高一次成功率。 只输出 JSON: { "modelId": "候选里的id", "reason": "一句话中文理由,不超过24字" } 如果无法判断,也必须给出候选中的 modelId。`, prompt: `用户请求:${state.text || "(无文本)"} 意图:${state.analysis.intent}/${state.analysis.subIntent || "none"} 候选模型: ${candidatesText}`, temperature: 0.1, maxTokens: 120, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); const parsed = parseJsonBlock<{ modelId?: string; reason?: string }>(text); if (!parsed?.modelId) return baseRoute; const chosen = imageCandidates.find((item) => item.model.id === parsed.modelId); if (!chosen) return baseRoute; return { id: chosen.model.id, displayName: chosen.model.displayName, providerType: chosen.provider.type, modelId: chosen.model.modelId, reason: `Agent规划: ${(parsed.reason || chosen.model.displayName).slice(0, 40)}`, }; } catch { return baseRoute; } } const analyzeNode = async (state: AutoAgentStateType) => { const { analyzeIntent } = await import("./smart-analyzer"); const analysis = await analyzeIntent({ text: state.text, hasImage: state.hasImage, imageUrls: state.imageUrls, assetIds: state.assetIds, imageFromHistory: state.imageFromHistory, historyMessages: state.historyMessages, context: state.context ?? undefined, preferredModelId: state.preferredModelId ?? undefined, }); return { analysis }; }; const confirmationGateNode = async (state: AutoAgentStateType) => { const workflowDecision = await runHumanConfirmationGate({ text: state.text, threadState: state.threadState, }); if (!workflowDecision.shouldBypassPlanner) { return { workflowDecision, }; } return buildWorkflowGateBypassPayload(state, workflowDecision); }; const shouldBypassPlanner = (state: AutoAgentStateType) => { return state.workflowDecision?.shouldBypassPlanner ? "route" : "analyze"; }; const planNode = async (state: AutoAgentStateType) => { if (!state.analysis) return {}; const baseAvailableTools = await getEnabledAgentToolDefinitionsForContext({ hasImage: state.hasImage, hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0, hasHistoryImage: state.historyImageCount > 0 || state.imageFromHistory, hasCurrentUpload: state.currentUploadImageCount > 0, }); const { filterToolsForSkillRoute, routeAgentSkill } = await import("./skill-router"); const skillRoute = routeAgentSkill({ text: state.text, analysis: state.analysis, hasImage: state.hasImage, hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0, targetResolutionMode: state.targetResolutionMode, mediaSource: state.mediaSource, selectedImageCount: state.selectedImageCount, threadState: state.threadState, }); const availableTools = filterToolsForSkillRoute(baseAvailableTools, skillRoute); console.log("[Auto Agent] Skill route:", { specialist: skillRoute.primary.specialistId, domain: skillRoute.primary.domain, overlays: skillRoute.overlays.map((item) => item.specialistId), allowlist: skillRoute.toolAllowlist, reason: skillRoute.reason, }); const plan = await buildExecutionPlanWithAgent( state, state.analysis, availableTools, skillRoute, ); const alignedAnalysis = alignAnalysisWithPlan(state.analysis, plan); return { availableTools, plan, analysis: alignedAnalysis, }; }; const prepareExecutionNode = async (state: AutoAgentStateType) => { if (!state.analysis || !state.plan) { return {}; } const execution = prepareExecutionFromPlan({ plan: state.plan, availableTools: state.availableTools || [], }); const fallbackQuestion = state.analysis.needsClarification ? state.analysis.clarifyingQuestions?.[0] : undefined; let friendlyMessage = state.analysis.conversationalReply || fallbackQuestion || null; if (state.plan.nextAction === "clarify") { friendlyMessage = friendlyMessage || "我先确认一下你希望我下一步执行什么。"; } else if (execution.shouldUseStructuredReply) { const strategicMessage = await buildStrategicFriendlyMessage(state.analysis, state); friendlyMessage = strategicMessage || state.analysis.conversationalReply || friendlyMessage; } return { execution, friendlyMessage, }; }; const shouldOptimize = (state: AutoAgentStateType) => { if (!state.analysis || !state.execution) return "route"; if (state.analysis.needsClarification) return "route"; return state.execution.shouldOptimizePrompt ? "optimize" : "route"; }; const optimizeNode = async (state: AutoAgentStateType) => { if (!state.analysis || !state.execution) { return {}; } if (state.execution.mode === "generate_video") { const videoPlan = await recommendVideoPlanWithAgent(state); const storyboardPreview = videoPlan.storyboard .slice(0, 4) .map( (shot) => `- 镜头${shot.shot}(${shot.duration}s):${shot.title}|${compactText( shot.description || shot.prompt, 64, )}` ) .join("\n"); const friendlyMessage = [ "### 需求理解", `- ${compactText(state.text || "生成视频", 110)}`, "### 脚本规划", `- ${videoPlan.script}`, "### 分镜规划", storyboardPreview || "- 已生成分镜,执行时自动拼接。", "### 执行说明", "- 已基于上述脚本与分镜优化提示词并提交视频模型执行。", ].join("\n"); return { friendlyMessage, optimizedPrompt: videoPlan.finalPrompt, videoPlan, }; } if (!state.execution.shouldApplyImagePipeline) { return {}; } const { optimizePrompt, optimizePromptWithLLM } = await import("./prompt-optimizer"); const optimizedFromLlm = await withTimeoutFallback< Awaited> | null >( optimizePromptWithLLM({ originalPrompt: state.text, intent: state.analysis.intent, subIntent: state.analysis.subIntent, historyMessages: state.historyMessages, context: state.context || undefined, }), AGENT_OPERATION_TIMEOUT_MS, null, ); const optimized = optimizedFromLlm || optimizePrompt({ originalPrompt: state.text, intent: state.analysis.intent, subIntent: state.analysis.subIntent, }); const mergedParams = sanitizeRecommendedImageParams( { ...optimized.recommendedImageParams, negativePrompt: optimized.negativePrompt && optimized.negativePrompt.trim().length > 0 ? optimized.negativePrompt : optimized.recommendedImageParams?.negativePrompt, }, state.analysis.intent, ) ?? heuristicImageParams(state.text, state.analysis); return { optimizedPrompt: optimized.optimizedPrompt, recommendedImageParams: mergedParams, }; }; const routeNode = async (state: AutoAgentStateType) => { const { routeToModel } = await import("./model-router"); let route = await routeToModel({ intent: state.execution?.routeIntent || state.analysis?.intent || "chat", subIntent: state.execution?.routeSubIntent || state.analysis?.subIntent, hasImage: state.hasImage, }); if (state.execution?.shouldApplyImagePipeline) { route = await refineRouteWithAgent(state, route); } return { route }; }; let compiledGraph: ReturnType | null = null; function buildGraph() { return new StateGraph(AutoAgentState) .addNode("confirmationGate", confirmationGateNode) .addNode("analyze", analyzeNode) .addNode("planner", planNode) .addNode("prepareExecution", prepareExecutionNode) .addNode("optimize", optimizeNode) .addNode("routeModel", routeNode) .addEdge(START, "confirmationGate") .addConditionalEdges("confirmationGate", shouldBypassPlanner, { analyze: "analyze", route: "routeModel", }) .addEdge("analyze", "planner") .addEdge("planner", "prepareExecution") .addConditionalEdges("prepareExecution", shouldOptimize, { optimize: "optimize", route: "routeModel", }) .addEdge("optimize", "routeModel") .addEdge("routeModel", END) .compile(); } function getGraph() { if (!compiledGraph) { compiledGraph = buildGraph(); } return compiledGraph; } export async function runAutoAgentGraph(params: AutoAgentInput): Promise { const graph = getGraph(); const result = await graph.invoke({ text: params.text, hasImage: params.hasImage, imageUrls: params.imageUrls ?? [], assetIds: params.assetIds ?? [], imageFromHistory: params.imageFromHistory ?? false, targetResolutionMode: params.targetResolutionMode ?? "none", mediaSource: params.mediaSource ?? "none", hasCanvasSelection: params.hasCanvasSelection ?? false, selectedImageCount: params.selectedImageCount ?? 0, historyImageCount: params.historyImageCount ?? 0, currentUploadImageCount: params.currentUploadImageCount ?? 0, historyMessages: params.historyMessages ?? [], context: params.context ?? null, preferredModelId: params.preferredModelId ?? null, threadState: normalizeAgentThreadState(params.threadState), analysis: null, availableTools: [], plan: null, execution: null, friendlyMessage: null, optimizedPrompt: null, recommendedImageParams: null, videoPlan: null, route: null, workflowDecision: null, }); return { analysis: result.analysis ?? undefined, plan: result.plan ?? undefined, execution: result.execution ?? undefined, friendlyMessage: result.friendlyMessage ?? undefined, optimizedPrompt: result.optimizedPrompt ?? undefined, recommendedImageParams: result.recommendedImageParams ?? undefined, videoPlan: result.videoPlan ?? undefined, route: result.route ?? undefined, threadState: result.threadState ?? undefined, workflowDecision: result.workflowDecision ?? undefined, }; }