308 lines
9.7 KiB
TypeScript
308 lines
9.7 KiB
TypeScript
import { Annotation, StateGraph, START, END } from "@langchain/langgraph";
|
||
import { generateText } from "ai";
|
||
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
|
||
import type { RouteResult } from "@/lib/ai/model-router";
|
||
|
||
interface HistoryMessage {
|
||
role: "user" | "assistant";
|
||
content: string;
|
||
}
|
||
|
||
export interface AutoAgentInput {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
imageFromHistory?: boolean;
|
||
historyMessages?: HistoryMessage[];
|
||
context?: string;
|
||
}
|
||
|
||
export interface AutoAgentResult {
|
||
analysis?: AnalysisResult;
|
||
friendlyMessage?: string;
|
||
optimizedPrompt?: string;
|
||
route?: RouteResult;
|
||
}
|
||
|
||
const AutoAgentState = Annotation.Root({
|
||
text: Annotation<string>,
|
||
hasImage: Annotation<boolean>,
|
||
imageUrls: Annotation<string[]>,
|
||
imageFromHistory: Annotation<boolean>,
|
||
historyMessages: Annotation<HistoryMessage[]>,
|
||
context: Annotation<string | null>,
|
||
analysis: Annotation<AnalysisResult | null>,
|
||
friendlyMessage: Annotation<string | null>,
|
||
optimizedPrompt: Annotation<string | null>,
|
||
route: Annotation<RouteResult | null>,
|
||
});
|
||
|
||
type AutoAgentStateType = typeof AutoAgentState.State;
|
||
|
||
function compactText(text: string, max = 48) {
|
||
const normalized = text.replace(/\s+/g, " ").trim();
|
||
if (normalized.length <= max) return normalized;
|
||
return `${normalized.slice(0, max)}...`;
|
||
}
|
||
|
||
function buildStrategicFriendlyMessage(
|
||
analysis: AnalysisResult,
|
||
state: AutoAgentStateType,
|
||
): Promise<string | null> {
|
||
const fallbackQuestion = analysis.clarifyingQuestions?.[0];
|
||
|
||
if (analysis.needsClarification) {
|
||
return Promise.resolve(
|
||
analysis.conversationalReply || fallbackQuestion || "我先确认一下，你希望我优先调整哪一部分？"
|
||
);
|
||
}
|
||
|
||
return generateAgentConversation(analysis, state);
|
||
}
|
||
|
||
async function generateAgentConversation(
|
||
analysis: AnalysisResult,
|
||
state: AutoAgentStateType
|
||
): Promise<string | null> {
|
||
try {
|
||
const { getEnabledAiModelsWithProvider } = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
|
||
const preferredVisionModel = allModels.find(
|
||
(m) => m.provider.type === "alibaba" && m.model.supportsVision && m.model.isEnabled
|
||
);
|
||
const candidateModel = state.hasImage
|
||
? preferredVisionModel || allModels.find((m) => m.model.supportsVision && m.model.isEnabled)
|
||
: allModels.find((m) => m.model.isEnabled && !m.model.isSubModel);
|
||
|
||
if (!candidateModel) {
|
||
return analysis.conversationalReply || null;
|
||
}
|
||
|
||
const model = await getLanguageModelById(candidateModel.model.id);
|
||
if (!model) {
|
||
return analysis.conversationalReply || null;
|
||
}
|
||
|
||
const history = state.historyMessages
|
||
.slice(-6)
|
||
.map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`)
|
||
.join("\n");
|
||
|
||
const validImageUrls = state.imageUrls
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
.slice(0, 4);
|
||
|
||
const imageInfo = state.hasImage
|
||
? `用户本轮可用图片数量：${state.imageUrls.length || 1}。图片来源：${state.imageFromHistory ? "历史消息复用" : "本轮上传"}。`
|
||
: "用户本轮没有上传图片。";
|
||
|
||
let visualAnalysis = "";
|
||
if (state.hasImage && validImageUrls.length > 0) {
|
||
try {
|
||
const visionPrompt = `请分析以下参考图片，给出可用于生图/修图执行前决策的结果。
|
||
要求：
|
||
1) 每张图输出 1 行，格式：图X：主体...；风格...；构图...；光影与色调...；材质/氛围...
|
||
2) 只写客观可见信息，不要编造品牌、尺寸、镜头参数。
|
||
3) 全部输出控制在 6 行内。`;
|
||
|
||
const visionInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url }));
|
||
const { text: visionText } = await generateText({
|
||
model,
|
||
messages: [
|
||
{
|
||
role: "user" as const,
|
||
content: [{ type: "text" as const, text: visionPrompt }, ...visionInputs],
|
||
},
|
||
],
|
||
temperature: 0.2,
|
||
maxTokens: 360,
|
||
} as any);
|
||
|
||
visualAnalysis = visionText.trim();
|
||
} catch (error) {
|
||
visualAnalysis = "";
|
||
}
|
||
}
|
||
|
||
const systemPrompt = `你是一个真实执行中的 AI 生图/修图 Agent。
|
||
请根据用户需求和本轮输入图片，给出“执行前”的分析汇报。
|
||
|
||
要求：
|
||
1) 不要说“我看不到图片”或“无法访问图片”，必须基于给定图片做真实分析。
|
||
2) 语气自然、专业，不要模板腔，不要空话。
|
||
3) 不暴露内部实现、模型路由、算法细节。
|
||
4) 输出 Markdown，控制在 8-14 行。
|
||
5) 有图片时，必须按顺序输出以下 Markdown H3 标题（逐字一致）：
|
||
- ### 需求理解
|
||
- ### 图片分析
|
||
- ### 执行计划
|
||
- ### 输出设置
|
||
6) “图片分析”要结合每张参考图差异，给出风格/构图/材质/光影等具体观察。`;
|
||
|
||
const prompt = [
|
||
`【用户原始需求】${state.text || "（无文本）"}`,
|
||
`【意图分析】intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`,
|
||
`【图片信息】${imageInfo}`,
|
||
visualAnalysis ? `【视觉模型预分析】\n${visualAnalysis}` : "",
|
||
state.context ? `【画布上下文】${compactText(state.context, 300)}` : "",
|
||
history ? `【最近对话】\n${history}` : "",
|
||
state.hasImage
|
||
? "请先做本轮图片分析，再输出执行前汇报。"
|
||
: "请输出本轮执行前汇报。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n");
|
||
|
||
const imageInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url }));
|
||
|
||
const requestPayload = state.hasImage && imageInputs.length > 0
|
||
? {
|
||
model,
|
||
system: systemPrompt,
|
||
messages: [
|
||
{
|
||
role: "user" as const,
|
||
content: [
|
||
{ type: "text" as const, text: prompt },
|
||
...imageInputs,
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.35,
|
||
maxTokens: 520,
|
||
}
|
||
: {
|
||
model,
|
||
system: systemPrompt,
|
||
prompt,
|
||
temperature: 0.35,
|
||
maxTokens: 420,
|
||
};
|
||
|
||
const { text } = await generateText(requestPayload as any);
|
||
|
||
const cleaned = text.trim();
|
||
if (cleaned.length < 10) {
|
||
return analysis.conversationalReply || null;
|
||
}
|
||
return cleaned;
|
||
} catch (error) {
|
||
return analysis.conversationalReply || null;
|
||
}
|
||
}
|
||
|
||
const analyzeNode = async (state: AutoAgentStateType) => {
|
||
const { analyzeIntent } = await import("./smart-analyzer");
|
||
const analysis = await analyzeIntent({
|
||
text: state.text,
|
||
hasImage: state.hasImage,
|
||
imageUrls: state.imageUrls,
|
||
imageFromHistory: state.imageFromHistory,
|
||
historyMessages: state.historyMessages,
|
||
context: state.context ?? undefined,
|
||
});
|
||
|
||
const fallbackQuestion = analysis.needsClarification
|
||
? analysis.clarifyingQuestions?.[0]
|
||
: undefined;
|
||
|
||
const strategicMessage = await buildStrategicFriendlyMessage(analysis, state);
|
||
|
||
return {
|
||
analysis,
|
||
friendlyMessage: strategicMessage || analysis.conversationalReply || fallbackQuestion || null,
|
||
};
|
||
};
|
||
|
||
const shouldOptimize = (state: AutoAgentStateType) => {
|
||
if (!state.analysis) return "route";
|
||
if (state.analysis.needsClarification) return "route";
|
||
if (state.analysis.intent === "image_gen") {
|
||
return "optimize";
|
||
}
|
||
if (state.analysis.intent === "image_edit" && state.analysis.subIntent) {
|
||
return "optimize";
|
||
}
|
||
return "route";
|
||
};
|
||
|
||
const optimizeNode = async (state: AutoAgentStateType) => {
|
||
if (!state.analysis) {
|
||
return {};
|
||
}
|
||
if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) {
|
||
return {};
|
||
}
|
||
if (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen") {
|
||
return {};
|
||
}
|
||
const { optimizePromptWithLLM } = await import("./prompt-optimizer");
|
||
const optimized = await optimizePromptWithLLM({
|
||
originalPrompt: state.text,
|
||
intent: state.analysis.intent,
|
||
subIntent: state.analysis.subIntent,
|
||
historyMessages: state.historyMessages,
|
||
context: state.context ?? undefined,
|
||
});
|
||
return { optimizedPrompt: optimized.optimizedPrompt };
|
||
};
|
||
|
||
const routeNode = async (state: AutoAgentStateType) => {
|
||
const { routeToModel } = await import("./model-router");
|
||
const route = await routeToModel({
|
||
intent: state.analysis?.intent || "chat",
|
||
subIntent: state.analysis?.subIntent,
|
||
hasImage: state.hasImage,
|
||
});
|
||
return { route };
|
||
};
|
||
|
||
let compiledGraph: ReturnType<typeof buildGraph> | null = null;
|
||
|
||
function buildGraph() {
|
||
return new StateGraph(AutoAgentState)
|
||
.addNode("analyze", analyzeNode)
|
||
.addNode("optimize", optimizeNode)
|
||
.addNode("routeModel", routeNode)
|
||
.addEdge(START, "analyze")
|
||
.addConditionalEdges("analyze", shouldOptimize, {
|
||
optimize: "optimize",
|
||
route: "routeModel",
|
||
})
|
||
.addEdge("optimize", "routeModel")
|
||
.addEdge("routeModel", END)
|
||
.compile();
|
||
}
|
||
|
||
function getGraph() {
|
||
if (!compiledGraph) {
|
||
compiledGraph = buildGraph();
|
||
}
|
||
return compiledGraph;
|
||
}
|
||
|
||
export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAgentResult> {
|
||
const graph = getGraph();
|
||
const result = await graph.invoke({
|
||
text: params.text,
|
||
hasImage: params.hasImage,
|
||
imageUrls: params.imageUrls ?? [],
|
||
imageFromHistory: params.imageFromHistory ?? false,
|
||
historyMessages: params.historyMessages ?? [],
|
||
context: params.context ?? null,
|
||
analysis: null,
|
||
friendlyMessage: null,
|
||
optimizedPrompt: null,
|
||
route: null,
|
||
});
|
||
|
||
return {
|
||
analysis: result.analysis ?? undefined,
|
||
friendlyMessage: result.friendlyMessage ?? undefined,
|
||
optimizedPrompt: result.optimizedPrompt ?? undefined,
|
||
route: result.route ?? undefined,
|
||
};
|
||
}
|