import { Annotation, StateGraph, START, END } from "@langchain/langgraph"; import { generateText } from "ai"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { RouteResult } from "@/lib/ai/model-router"; interface HistoryMessage { role: "user" | "assistant"; content: string; } export interface AgentRecommendedImageParams { aspectRatio?: string; imageSize?: string; imageCount?: number; negativePrompt?: string; promptExtend?: boolean; } export interface AutoAgentInput { text: string; hasImage: boolean; imageUrls?: string[]; imageFromHistory?: boolean; historyMessages?: HistoryMessage[]; context?: string; } export interface AutoAgentResult { analysis?: AnalysisResult; friendlyMessage?: string; optimizedPrompt?: string; recommendedImageParams?: AgentRecommendedImageParams; route?: RouteResult; } const AutoAgentState = Annotation.Root({ text: Annotation, hasImage: Annotation, imageUrls: Annotation, imageFromHistory: Annotation, historyMessages: Annotation, context: Annotation, analysis: Annotation, friendlyMessage: Annotation, optimizedPrompt: Annotation, recommendedImageParams: Annotation, route: Annotation, }); type AutoAgentStateType = typeof AutoAgentState.State; function compactText(text: string, max = 48) { const normalized = text.replace(/\s+/g, " ").trim(); if (normalized.length <= max) return normalized; return `${normalized.slice(0, max)}...`; } function normalizeAspectRatio(value?: string): string | undefined { if (!value) return undefined; const normalized = value.trim().toLowerCase(); if (normalized === "auto") return "auto"; const raw = value.trim(); const allowed = new Set([ "1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9", ]); return allowed.has(raw) ? raw : undefined; } function normalizeImageSize(value?: string): string | undefined { if (!value) return undefined; const normalized = value.trim().toUpperCase(); if (normalized === "1K" || normalized === "2K" || normalized === "4K") { return normalized; } return undefined; } function sanitizeRecommendedImageParams( params: AgentRecommendedImageParams | undefined, intent?: AnalysisResult["intent"], ): AgentRecommendedImageParams | undefined { if (!params) return undefined; const safe: AgentRecommendedImageParams = {}; const normalizedAspectRatio = normalizeAspectRatio(params.aspectRatio); const normalizedImageSize = normalizeImageSize(params.imageSize); if (normalizedAspectRatio) safe.aspectRatio = normalizedAspectRatio; if (normalizedImageSize) safe.imageSize = normalizedImageSize; if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) { const min = 1; const max = intent === "image_edit" ? 1 : 4; safe.imageCount = Math.min(Math.max(Math.round(params.imageCount), min), max); } if (intent === "image_edit") { safe.imageCount = 1; } if (typeof params.negativePrompt === "string" && params.negativePrompt.trim()) { safe.negativePrompt = params.negativePrompt.trim().slice(0, 500); } if (typeof params.promptExtend === "boolean") { safe.promptExtend = params.promptExtend; } if (intent === "image_edit") { safe.promptExtend = false; } return Object.keys(safe).length > 0 ? safe : undefined; } function heuristicImageParams( text: string, analysis: AnalysisResult, ): AgentRecommendedImageParams | undefined { const normalized = text.toLowerCase(); const params: AgentRecommendedImageParams = { imageSize: "2K", promptExtend: analysis.intent === "image_edit" ? false : true, }; if (analysis.intent === "image_edit") { params.imageCount = 1; } else if (/多张|四张|4张|三张|3张/.test(normalized)) { params.imageCount = /四张|4张/.test(normalized) ? 4 : 3; } else if (/两张|2张|双图/.test(normalized)) { params.imageCount = 2; } else { params.imageCount = 1; } if (/横图|横版|宽屏|封面|banner|海报/.test(normalized)) { params.aspectRatio = "16:9"; } else if (/竖图|竖版|手机壁纸|全身|海报竖版/.test(normalized)) { params.aspectRatio = "9:16"; } else if (/方图|方形|头像|logo|电商主图/.test(normalized)) { params.aspectRatio = "1:1"; } else { params.aspectRatio = "auto"; } return sanitizeRecommendedImageParams(params, analysis.intent); } function buildStrategicFriendlyMessage( analysis: AnalysisResult, state: AutoAgentStateType, ): Promise { const fallbackQuestion = analysis.clarifyingQuestions?.[0]; if (analysis.needsClarification) { return Promise.resolve( analysis.conversationalReply || fallbackQuestion || "我先确认一下,你希望我优先调整哪一部分?" ); } return generateAgentConversation(analysis, state); } type EnabledModelWithProvider = Awaited< ReturnType >[number]; function pickQwenVisionModel(models: EnabledModelWithProvider[]) { const visionModels = models.filter((item) => item.model.supportsVision && item.model.isEnabled); const qwenByName = visionModels.find( (item) => item.provider.type === "alibaba" && /(qwen|通义|vl)/i.test(`${item.model.displayName} ${item.model.modelId}`) ); if (qwenByName) return qwenByName; const alibabaVision = visionModels.find((item) => item.provider.type === "alibaba"); if (alibabaVision) return alibabaVision; return visionModels[0]; } function pickAgentReasoningModel( agentModel: EnabledModelWithProvider | undefined, models: EnabledModelWithProvider[], ) { const isReasoningCompatibleProvider = (providerType: string) => ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType); if (agentModel && isReasoningCompatibleProvider(agentModel.provider.type)) return agentModel; const zhipuModel = models.find( (item) => item.provider.type === "zhipu" && item.model.isEnabled && !item.model.isSubModel && isReasoningCompatibleProvider(item.provider.type) ); if (zhipuModel) return zhipuModel; return models.find( (item) => item.model.isEnabled && !item.model.isSubModel && isReasoningCompatibleProvider(item.provider.type) ); } function parseJsonBlock>(text: string): T | null { const match = text.match(/\{[\s\S]*\}/); if (!match) return null; try { return JSON.parse(match[0]) as T; } catch { return null; } } async function generateAgentConversation( analysis: AnalysisResult, state: AutoAgentStateType ): Promise { try { const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); const reasoningModelConfig = pickAgentReasoningModel(agentModel, allModels); if (!reasoningModelConfig) { return analysis.conversationalReply || null; } const reasoningModel = await getLanguageModelById(reasoningModelConfig.model.id); if (!reasoningModel) { return analysis.conversationalReply || null; } const history = state.historyMessages .slice(-6) .map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`) .join("\n"); const validImageUrls = state.imageUrls .filter((url) => /^https?:\/\//i.test(url)) .slice(0, 4); const imageInfo = state.hasImage ? `用户本轮可用图片数量:${state.imageUrls.length || 1}。图片来源:${state.imageFromHistory ? "历史消息复用" : "本轮上传"}。` : "用户本轮没有上传图片。"; let visualAnalysis = ""; if (state.hasImage && validImageUrls.length > 0) { try { const visionModelConfig = pickQwenVisionModel(allModels); if (!visionModelConfig) { throw new Error("missing_vision_model"); } const visionModel = await getLanguageModelById(visionModelConfig.model.id); if (!visionModel) { throw new Error("missing_vision_runtime"); } const visionPrompt = `请分析以下参考图片,给出可用于生图/修图执行前决策的结果。 要求: 1) 每张图输出 1 行,格式:图X:主体...;风格...;构图...;光影与色调...;材质/氛围... 2) 只写客观可见信息,不要编造品牌、尺寸、镜头参数。 3) 全部输出控制在 6 行内。`; const visionInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url })); const { text: visionText } = await generateText({ model: visionModel, messages: [ { role: "user" as const, content: [{ type: "text" as const, text: visionPrompt }, ...visionInputs], }, ], temperature: 0.2, maxTokens: 360, } as any); visualAnalysis = visionText.trim(); console.log( `[Auto Agent] Vision analysis model: ${visionModelConfig.model.displayName} (${visionModelConfig.provider.type})` ); } catch (error) { console.warn("[Auto Agent] Vision analysis failed, continue without visual summary"); visualAnalysis = ""; } } const systemPrompt = `你是一个真实执行中的 AI 生图/修图 Agent。你的核心目标是提升用户一次生成成功率。 你必须按以下 AGENT SKILLS 执行: 1) 需求拆解:把用户目标拆成可执行指令,优先满足用户明确要求。 2) 风险预判:主动识别可能导致失败的歧义(主体数量、视角、风格冲突、文字内容)。 3) 提示词策略:给出可落地的正向约束和必要的负向约束,避免空泛描述。 4) 参数策略:在输出设置中明确建议宽高比、清晰度档位、出图数量(若用户未指定)。 输出要求: - 不要说“看不到图片/无法访问图片”。 - 不暴露内部实现、模型路由、算法细节。 - 输出 Markdown,控制在 8-14 行。 - 有图片时必须按顺序输出以下 Markdown H3 标题(逐字一致): ### 需求理解 ### 图片分析 ### 执行计划 ### 输出设置 - “图片分析”必须基于视觉分析结果写具体观察,不允许写泛化空话。 - 只有明确拿到原图分辨率时,才能写具体像素尺寸;否则只写宽高比/清晰度目标。`; const prompt = [ `【用户原始需求】${state.text || "(无文本)"}`, `【意图分析】intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`, `【图片信息】${imageInfo}`, visualAnalysis ? `【视觉模型预分析】\n${visualAnalysis}` : "", state.context ? `【画布上下文】${compactText(state.context, 300)}` : "", history ? `【最近对话】\n${history}` : "", state.hasImage ? "请先做本轮图片分析,再输出执行前汇报。" : "请输出本轮执行前汇报。", ] .filter(Boolean) .join("\n\n"); const { text } = await generateText({ model: reasoningModel, system: systemPrompt, prompt, temperature: 0.35, maxTokens: 520, } as any); const cleaned = text.trim(); if (cleaned.length < 10) { return analysis.conversationalReply || null; } console.log( `[Auto Agent] Conversation model: ${reasoningModelConfig.model.displayName} (${reasoningModelConfig.provider.type})` ); return cleaned; } catch (error) { return analysis.conversationalReply || null; } } async function recommendImageParamsWithAgent( analysis: AnalysisResult, state: AutoAgentStateType, ): Promise { if (analysis.intent !== "image_edit" && analysis.intent !== "image_gen") { return undefined; } try { const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); const plannerModelConfig = pickAgentReasoningModel(agentModel, allModels); if (!plannerModelConfig) { return heuristicImageParams(state.text, analysis); } const plannerModel = await getLanguageModelById(plannerModelConfig.model.id); if (!plannerModel) { return heuristicImageParams(state.text, analysis); } const history = state.historyMessages .slice(-4) .map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`) .join("\n"); const requestText = [ `用户请求:${state.text || "(无文本)"}`, `识别意图:${analysis.intent}/${analysis.subIntent || "none"}`, `是否有参考图:${state.hasImage ? "是" : "否"}`, history ? `最近对话:\n${history}` : "", "请给出本次最稳妥的生图参数建议。", ] .filter(Boolean) .join("\n\n"); const { text } = await generateText({ model: plannerModel, system: `你是生图参数规划 Agent。目标是提升首图命中率和稳定性。 只输出 JSON,不要输出解释。格式如下: { "aspectRatio": "1:1|2:3|3:2|3:4|4:3|4:5|5:4|9:16|16:9|21:9|auto", "imageSize": "1K|2K|4K", "imageCount": 1, "negativePrompt": "可选,简洁", "promptExtend": true } 规则: - image_edit 场景强制 imageCount=1。 - 若用户未指定比例,优先 auto。 - negativePrompt 仅在明显需要避坑时填写。`, prompt: requestText, temperature: 0.15, maxTokens: 220, } as any); const parsed = parseJsonBlock(text); const sanitized = sanitizeRecommendedImageParams(parsed || undefined, analysis.intent); if (sanitized) { console.log( `[Auto Agent] Parameter planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})` ); return sanitized; } return heuristicImageParams(state.text, analysis); } catch { return heuristicImageParams(state.text, analysis); } } async function refineRouteWithAgent( state: AutoAgentStateType, baseRoute: RouteResult, ): Promise { if (!state.analysis || (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen")) { return baseRoute; } try { const { db } = await import("@/lib/db"); const { siteSettings } = await import("@/lib/db/schema"); const [settings] = await db.select().from(siteSettings).limit(1); const configuredDefaultModelId = settings?.canvasAutoModeDefaultModelId || null; if (configuredDefaultModelId && baseRoute.id === configuredDefaultModelId) { console.log( `[Auto Agent] Keep admin default model for auto mode: ${baseRoute.displayName}` ); return baseRoute; } const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); const { isAiModelAgentAuto } = await import("@/lib/db/ai-model-flags"); const allModelsRaw = await getEnabledAiModelsWithProvider(false); const allModels = allModelsRaw.filter((item) => !isAiModelAgentAuto(item.model)); const imageCandidates = allModels .filter((item) => item.model.supportsImageGen && item.model.isEnabled && !item.model.isSubModel) .slice(0, 12); if (imageCandidates.length <= 1) return baseRoute; const agentModel = await getAgentAiModelWithProvider(); const plannerModelConfig = pickAgentReasoningModel(agentModel, allModelsRaw); if (!plannerModelConfig) return baseRoute; const plannerModel = await getLanguageModelById(plannerModelConfig.model.id); if (!plannerModel) return baseRoute; const candidatesText = imageCandidates .map((item, index) => `${index + 1}. id=${item.model.id}; 名称=${item.model.displayName}; provider=${item.provider.type}`) .join("\n"); const { text } = await generateText({ model: plannerModel, system: `你是生图模型选择 Agent。目标是优先提高一次成功率。 只输出 JSON: { "modelId": "候选里的id", "reason": "一句话中文理由,不超过24字" } 如果无法判断,也必须给出候选中的 modelId。`, prompt: `用户请求:${state.text || "(无文本)"} 意图:${state.analysis.intent}/${state.analysis.subIntent || "none"} 候选模型: ${candidatesText}`, temperature: 0.1, maxTokens: 120, } as any); const parsed = parseJsonBlock<{ modelId?: string; reason?: string }>(text); if (!parsed?.modelId) return baseRoute; const chosen = imageCandidates.find((item) => item.model.id === parsed.modelId); if (!chosen) return baseRoute; return { id: chosen.model.id, displayName: chosen.model.displayName, providerType: chosen.provider.type, modelId: chosen.model.modelId, reason: `Agent规划: ${(parsed.reason || chosen.model.displayName).slice(0, 40)}`, }; } catch { return baseRoute; } } const analyzeNode = async (state: AutoAgentStateType) => { const { analyzeIntent } = await import("./smart-analyzer"); const analysis = await analyzeIntent({ text: state.text, hasImage: state.hasImage, imageUrls: state.imageUrls, imageFromHistory: state.imageFromHistory, historyMessages: state.historyMessages, context: state.context ?? undefined, }); const fallbackQuestion = analysis.needsClarification ? analysis.clarifyingQuestions?.[0] : undefined; const strategicMessage = await buildStrategicFriendlyMessage(analysis, state); return { analysis, friendlyMessage: strategicMessage || analysis.conversationalReply || fallbackQuestion || null, }; }; const shouldOptimize = (state: AutoAgentStateType) => { if (!state.analysis) return "route"; if (state.analysis.needsClarification) return "route"; if (state.analysis.intent === "image_gen") { return "optimize"; } if (state.analysis.intent === "image_edit" && state.analysis.subIntent) { return "optimize"; } return "route"; }; const optimizeNode = async (state: AutoAgentStateType) => { if (!state.analysis) { return {}; } if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) { const recommendedImageParams = await recommendImageParamsWithAgent(state.analysis, state); return { recommendedImageParams }; } if (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen") { return {}; } const { optimizePromptWithLLM } = await import("./prompt-optimizer"); const optimized = await optimizePromptWithLLM({ originalPrompt: state.text, intent: state.analysis.intent, subIntent: state.analysis.subIntent, historyMessages: state.historyMessages, context: state.context ?? undefined, }); const mergedParams = sanitizeRecommendedImageParams( { ...optimized.recommendedImageParams, negativePrompt: optimized.negativePrompt && optimized.negativePrompt.trim().length > 0 ? optimized.negativePrompt : optimized.recommendedImageParams?.negativePrompt, }, state.analysis.intent, ) ?? (await recommendImageParamsWithAgent(state.analysis, state)); return { optimizedPrompt: optimized.optimizedPrompt, recommendedImageParams: mergedParams, }; }; const routeNode = async (state: AutoAgentStateType) => { const { routeToModel } = await import("./model-router"); const baseRoute = await routeToModel({ intent: state.analysis?.intent || "chat", subIntent: state.analysis?.subIntent, hasImage: state.hasImage, }); const route = await refineRouteWithAgent(state, baseRoute); return { route }; }; let compiledGraph: ReturnType | null = null; function buildGraph() { return new StateGraph(AutoAgentState) .addNode("analyze", analyzeNode) .addNode("optimize", optimizeNode) .addNode("routeModel", routeNode) .addEdge(START, "analyze") .addConditionalEdges("analyze", shouldOptimize, { optimize: "optimize", route: "routeModel", }) .addEdge("optimize", "routeModel") .addEdge("routeModel", END) .compile(); } function getGraph() { if (!compiledGraph) { compiledGraph = buildGraph(); } return compiledGraph; } export async function runAutoAgentGraph(params: AutoAgentInput): Promise { const graph = getGraph(); const result = await graph.invoke({ text: params.text, hasImage: params.hasImage, imageUrls: params.imageUrls ?? [], imageFromHistory: params.imageFromHistory ?? false, historyMessages: params.historyMessages ?? [], context: params.context ?? null, analysis: null, friendlyMessage: null, optimizedPrompt: null, recommendedImageParams: null, route: null, }); return { analysis: result.analysis ?? undefined, friendlyMessage: result.friendlyMessage ?? undefined, optimizedPrompt: result.optimizedPrompt ?? undefined, recommendedImageParams: result.recommendedImageParams ?? undefined, route: result.route ?? undefined, }; }