import { Annotation, StateGraph, START, END } from "@langchain/langgraph"; import { generateText } from "ai"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { RouteResult } from "@/lib/ai/model-router"; interface HistoryMessage { role: "user" | "assistant"; content: string; } export interface AutoAgentInput { text: string; hasImage: boolean; imageUrls?: string[]; imageFromHistory?: boolean; historyMessages?: HistoryMessage[]; context?: string; } export interface AutoAgentResult { analysis?: AnalysisResult; friendlyMessage?: string; optimizedPrompt?: string; route?: RouteResult; } const AutoAgentState = Annotation.Root({ text: Annotation, hasImage: Annotation, imageUrls: Annotation, imageFromHistory: Annotation, historyMessages: Annotation, context: Annotation, analysis: Annotation, friendlyMessage: Annotation, optimizedPrompt: Annotation, route: Annotation, }); type AutoAgentStateType = typeof AutoAgentState.State; function compactText(text: string, max = 48) { const normalized = text.replace(/\s+/g, " ").trim(); if (normalized.length <= max) return normalized; return `${normalized.slice(0, max)}...`; } function buildStrategicFriendlyMessage( analysis: AnalysisResult, state: AutoAgentStateType, ): Promise { const fallbackQuestion = analysis.clarifyingQuestions?.[0]; if (analysis.needsClarification) { return Promise.resolve( analysis.conversationalReply || fallbackQuestion || "我先确认一下,你希望我优先调整哪一部分?" ); } return generateAgentConversation(analysis, state); } async function generateAgentConversation( analysis: AnalysisResult, state: AutoAgentStateType ): Promise { try { const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); const canUseAgentModel = Boolean( agentModel && (!state.hasImage || agentModel.model.supportsVision) ); const preferredVisionModel = allModels.find( (m) => m.provider.type === "alibaba" && m.model.supportsVision && m.model.isEnabled ); const candidateModel = (canUseAgentModel ? agentModel : undefined) || (state.hasImage ? preferredVisionModel || allModels.find((m) => m.model.supportsVision && m.model.isEnabled) : allModels.find((m) => m.model.isEnabled && !m.model.isSubModel)); if (!candidateModel) { return analysis.conversationalReply || null; } const model = await getLanguageModelById(candidateModel.model.id); if (!model) { return analysis.conversationalReply || null; } const history = state.historyMessages .slice(-6) .map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`) .join("\n"); const validImageUrls = state.imageUrls .filter((url) => /^https?:\/\//i.test(url)) .slice(0, 4); const imageInfo = state.hasImage ? `用户本轮可用图片数量:${state.imageUrls.length || 1}。图片来源:${state.imageFromHistory ? "历史消息复用" : "本轮上传"}。` : "用户本轮没有上传图片。"; let visualAnalysis = ""; if (state.hasImage && validImageUrls.length > 0) { try { const visionPrompt = `请分析以下参考图片,给出可用于生图/修图执行前决策的结果。 要求: 1) 每张图输出 1 行,格式:图X:主体...;风格...;构图...;光影与色调...;材质/氛围... 2) 只写客观可见信息,不要编造品牌、尺寸、镜头参数。 3) 全部输出控制在 6 行内。`; const visionInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url })); const { text: visionText } = await generateText({ model, messages: [ { role: "user" as const, content: [{ type: "text" as const, text: visionPrompt }, ...visionInputs], }, ], temperature: 0.2, maxTokens: 360, } as any); visualAnalysis = visionText.trim(); } catch (error) { visualAnalysis = ""; } } const systemPrompt = `你是一个真实执行中的 AI 生图/修图 Agent。 请根据用户需求和本轮输入图片,给出“执行前”的分析汇报。 要求: 1) 不要说“我看不到图片”或“无法访问图片”,必须基于给定图片做真实分析。 2) 语气自然、专业,不要模板腔,不要空话。 3) 不暴露内部实现、模型路由、算法细节。 4) 输出 Markdown,控制在 8-14 行。 5) 有图片时,必须按顺序输出以下 Markdown H3 标题(逐字一致): - ### 需求理解 - ### 图片分析 - ### 执行计划 - ### 输出设置 6) “图片分析”要结合每张参考图差异,给出风格/构图/材质/光影等具体观察。 7) 只有明确拿到“原图分辨率”时,才能写具体像素尺寸;否则只写宽高比/清晰度目标,不要编造 335x500 这类预览尺寸。`; const prompt = [ `【用户原始需求】${state.text || "(无文本)"}`, `【意图分析】intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`, `【图片信息】${imageInfo}`, visualAnalysis ? `【视觉模型预分析】\n${visualAnalysis}` : "", state.context ? `【画布上下文】${compactText(state.context, 300)}` : "", history ? `【最近对话】\n${history}` : "", state.hasImage ? "请先做本轮图片分析,再输出执行前汇报。" : "请输出本轮执行前汇报。", ] .filter(Boolean) .join("\n\n"); const imageInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url })); const requestPayload = state.hasImage && imageInputs.length > 0 ? { model, system: systemPrompt, messages: [ { role: "user" as const, content: [ { type: "text" as const, text: prompt }, ...imageInputs, ], }, ], temperature: 0.35, maxTokens: 520, } : { model, system: systemPrompt, prompt, temperature: 0.35, maxTokens: 420, }; const { text } = await generateText(requestPayload as any); const cleaned = text.trim(); if (cleaned.length < 10) { return analysis.conversationalReply || null; } return cleaned; } catch (error) { return analysis.conversationalReply || null; } } const analyzeNode = async (state: AutoAgentStateType) => { const { analyzeIntent } = await import("./smart-analyzer"); const analysis = await analyzeIntent({ text: state.text, hasImage: state.hasImage, imageUrls: state.imageUrls, imageFromHistory: state.imageFromHistory, historyMessages: state.historyMessages, context: state.context ?? undefined, }); const fallbackQuestion = analysis.needsClarification ? analysis.clarifyingQuestions?.[0] : undefined; const strategicMessage = await buildStrategicFriendlyMessage(analysis, state); return { analysis, friendlyMessage: strategicMessage || analysis.conversationalReply || fallbackQuestion || null, }; }; const shouldOptimize = (state: AutoAgentStateType) => { if (!state.analysis) return "route"; if (state.analysis.needsClarification) return "route"; if (state.analysis.intent === "image_gen") { return "optimize"; } if (state.analysis.intent === "image_edit" && state.analysis.subIntent) { return "optimize"; } return "route"; }; const optimizeNode = async (state: AutoAgentStateType) => { if (!state.analysis) { return {}; } if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) { return {}; } if (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen") { return {}; } const { optimizePromptWithLLM } = await import("./prompt-optimizer"); const optimized = await optimizePromptWithLLM({ originalPrompt: state.text, intent: state.analysis.intent, subIntent: state.analysis.subIntent, historyMessages: state.historyMessages, context: state.context ?? undefined, }); return { optimizedPrompt: optimized.optimizedPrompt }; }; const routeNode = async (state: AutoAgentStateType) => { const { routeToModel } = await import("./model-router"); const route = await routeToModel({ intent: state.analysis?.intent || "chat", subIntent: state.analysis?.subIntent, hasImage: state.hasImage, }); return { route }; }; let compiledGraph: ReturnType | null = null; function buildGraph() { return new StateGraph(AutoAgentState) .addNode("analyze", analyzeNode) .addNode("optimize", optimizeNode) .addNode("routeModel", routeNode) .addEdge(START, "analyze") .addConditionalEdges("analyze", shouldOptimize, { optimize: "optimize", route: "routeModel", }) .addEdge("optimize", "routeModel") .addEdge("routeModel", END) .compile(); } function getGraph() { if (!compiledGraph) { compiledGraph = buildGraph(); } return compiledGraph; } export async function runAutoAgentGraph(params: AutoAgentInput): Promise { const graph = getGraph(); const result = await graph.invoke({ text: params.text, hasImage: params.hasImage, imageUrls: params.imageUrls ?? [], imageFromHistory: params.imageFromHistory ?? false, historyMessages: params.historyMessages ?? [], context: params.context ?? null, analysis: null, friendlyMessage: null, optimizedPrompt: null, route: null, }); return { analysis: result.analysis ?? undefined, friendlyMessage: result.friendlyMessage ?? undefined, optimizedPrompt: result.optimizedPrompt ?? undefined, route: result.route ?? undefined, }; }