fix: restore agent routing and isolate canvas context
This commit is contained in:
Your Name
parent
77f1401333
commit
d4e6fb3fc8
@ -143,6 +143,7 @@ import {
|
||||
recordProviderHealthSuccess,
|
||||
shouldAvoidProvider,
|
||||
} from "@/lib/ai/runtime/provider-health";
|
||||
import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
|
||||
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
|
||||
@ -6747,6 +6748,17 @@ export async function POST(request: Request) {
|
||||
resolvedRuntimeModelLookupId = actualModelId;
|
||||
}
|
||||
|
||||
const effectiveConversationSystemPrompt = shouldRunAgentConversation
|
||||
? `${await buildAgentRoleIdentityInstruction({
|
||||
allowVideoGeneration: allowAgentVideoGeneration,
|
||||
hasCanvasContext: Boolean(
|
||||
effectiveContextPayload?.source === "canvas" ||
|
||||
effectiveContextPayload?.selectedNodeIds?.length ||
|
||||
effectiveContextPayload?.canvasImageNodes?.length,
|
||||
),
|
||||
})}\n\n${systemPrompt({ selectedChatModel: finalModelId, requestHints })}`
|
||||
: systemPrompt({ selectedChatModel: finalModelId, requestHints });
|
||||
|
||||
const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor(
|
||||
resolvedRuntimeModelLookupId,
|
||||
).catch(() => null);
|
||||
@ -9506,7 +9518,7 @@ export async function POST(request: Request) {
|
||||
|
||||
const result = streamText({
|
||||
model: languageModel,
|
||||
system: systemPrompt({ selectedChatModel: finalModelId, requestHints }),
|
||||
system: effectiveConversationSystemPrompt,
|
||||
messages: convertedMessages,
|
||||
stopWhen: stepCountIs(5),
|
||||
experimental_activeTools:
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useState, useCallback, use, useRef, useMemo } from "react";
|
||||
import { useEffect, useLayoutEffect, useState, useCallback, use, useRef, useMemo } from "react";
|
||||
import { useRouter, useParams, useSearchParams } from "next/navigation";
|
||||
import useSWR from "swr";
|
||||
import {
|
||||
@ -248,7 +248,7 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
}, [isAgentMode, updateCanvasSearchParams]);
|
||||
|
||||
// 加载画布数据
|
||||
useEffect(() => {
|
||||
useLayoutEffect(() => {
|
||||
console.log('[Canvas] Loading project:', id);
|
||||
|
||||
const loadProject = async () => {
|
||||
|
||||
@ -4792,7 +4792,7 @@ export function CanvasChatPanel({
|
||||
|
||||
return {
|
||||
source: "canvas" as const,
|
||||
projectId: rawChatId || undefined,
|
||||
projectId: canvasId || undefined,
|
||||
selectedNodeIds: selectedImageNodes.map((node) => node.id),
|
||||
selectedAssetIds,
|
||||
focusPolicy: selectedImageNodes.length > 0 ? ("selected" as const) : ("auto" as const),
|
||||
@ -4818,7 +4818,7 @@ export function CanvasChatPanel({
|
||||
isSelected: selectedIds.has(node.id),
|
||||
})),
|
||||
};
|
||||
}, [nodes, rawChatId, selectedImageNodes]);
|
||||
}, [canvasId, nodes, selectedImageNodes]);
|
||||
|
||||
const updateAttachmentAtVideoSlot = useCallback(
|
||||
(slotIndex: number, attachment: PickedObjectAttachment | null) => {
|
||||
|
||||
@ -1528,7 +1528,16 @@ async function generateAgentConversation(
|
||||
}
|
||||
}
|
||||
|
||||
const systemPrompt = `你是一个真实执行中的 AI 生图/修图 Agent。你的核心目标是提升用户一次生成成功率。
|
||||
const { buildAgentRoleIdentityInstruction } = await import(
|
||||
"@/lib/ai/runtime/agent-role-profile"
|
||||
);
|
||||
const identityInstruction = await buildAgentRoleIdentityInstruction({
|
||||
allowVideoGeneration: true,
|
||||
hasCanvasContext: Boolean(state.context),
|
||||
});
|
||||
const systemPrompt = `${identityInstruction}
|
||||
|
||||
你是一个真实执行中的 AI 生图/修图 Agent。你的核心目标是提升用户一次生成成功率。
|
||||
|
||||
你必须按以下 AGENT SKILLS 执行：
|
||||
1) 需求拆解：把用户目标拆成可执行指令，优先满足用户明确要求。
|
||||
|
||||
34
lib/ai/runtime/agent-role-profile.ts
Normal file
34
lib/ai/runtime/agent-role-profile.ts
Normal file
@ -0,0 +1,34 @@
|
||||
import "server-only";
|
||||
|
||||
import { getSiteSettings } from "@/lib/db/queries";
|
||||
|
||||
export async function buildAgentRoleIdentityInstruction(params?: {
|
||||
siteName?: string;
|
||||
allowVideoGeneration?: boolean;
|
||||
hasCanvasContext?: boolean;
|
||||
}) {
|
||||
const resolvedSiteName =
|
||||
String(params?.siteName || "").trim() || String((await getSiteSettings()).siteName || "").trim() || "本站";
|
||||
const agentName = `${resolvedSiteName}视觉创作助手`;
|
||||
const capabilityLines = [
|
||||
"1. 图片分析：识别主体、风格、构图、光线、材质，并给出可执行观察。",
|
||||
"2. 提示词整理与优化：根据图片或用户描述，整理适合生图/修图/视频的提示词。",
|
||||
"3. 图片编辑与生成：修图、替换、去元素、换背景、增强、扩图、改尺寸、生图。",
|
||||
params?.allowVideoGeneration
|
||||
? "4. 视频与分镜：生成视频、整理分镜草案、确认镜头、修改镜头。"
|
||||
: "4. 视频与分镜：可以先分析视频需求、整理分镜草案；真正执行视频生成可能需要用户切换到视频模式或满足当前执行条件。",
|
||||
params?.hasCanvasContext
|
||||
? "5. 画布协作：结合无限画布里的当前选中图、当前上传图、最近结果继续工作。"
|
||||
: "5. 会话协作：结合当前上传图、当前需求和最近有效上下文继续工作。",
|
||||
];
|
||||
|
||||
return [
|
||||
`你是 ${resolvedSiteName} 产品中的视觉创作 Agent，对外名称是“${agentName}”。`,
|
||||
"你的职责是帮助用户完成视觉创作相关工作，而不是把自己介绍成通用百科、学习辅导、历史人文或生活杂谈助手。",
|
||||
"当用户问“你是谁 / 你能做什么 / 你有哪些能力”时，只能介绍你在本产品里的真实视觉能力边界。",
|
||||
"不要说自己是“字节跳动官方豆包”或“字节跳动开发的AI豆包”，也不要冒充任何外部品牌官方身份。如果用户明确追问底层模型，只能解释为“当前 Agent 默认由系统配置的模型驱动”，不能把底层模型名字说成产品品牌名。",
|
||||
"不要夸大、不要编造不存在的能力，不要把自己说成能处理所有领域问题。",
|
||||
"你可以对外介绍的核心能力只有这些：",
|
||||
...capabilityLines,
|
||||
].join("\n");
|
||||
}
|
||||
@ -55,7 +55,7 @@ function isLikelyImageExecutionRequest(text: string): boolean {
|
||||
const normalized = normalizeCompactText(text);
|
||||
if (!normalized) return false;
|
||||
|
||||
return /生图|生成图片|出图|画一张|帮我画|做图|做一张图|重新生成|再来一张|执行生成|开始生成|生成出来/.test(
|
||||
return /生图|生成图片|出图|画一张|帮我画|做图|做一张图|重新生成|再来一张|执行生成|开始生成|生成出来|生成一只|生成一个|生成一张|生成一幅|生成一版|帮我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)|给我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)/.test(
|
||||
normalized,
|
||||
);
|
||||
}
|
||||
|
||||
@ -94,7 +94,7 @@ function isExplicitVideoGenerationRequest(text: string): boolean {
|
||||
function isExplicitImageGenerationRequest(text: string): boolean {
|
||||
const normalizedText = normalizeCompactText(text);
|
||||
if (isLikelyAdvisoryOrPlanningRequest(text)) return false;
|
||||
return /重新生成|再来一张|生成新图|新建图片|创作一张|生成一张|画一张|给我画|帮我画|出一张|做一张图|做一张海报|创建一张图片|参考.*生成|按这个风格生成|以这张图为灵感生成|仿照.*生成/.test(
|
||||
return /重新生成|再来一张|生成新图|新建图片|创作一张|生成一张|画一张|给我画|帮我画|出一张|做一张图|做一张海报|创建一张图片|参考.*生成|按这个风格生成|以这张图为灵感生成|仿照.*生成|生成一只|生成一个|生成一幅|生成一版|生成一套|生成一组|帮我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)|给我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)/.test(
|
||||
normalizedText,
|
||||
);
|
||||
}
|
||||
@ -161,6 +161,22 @@ function applyDirectExecutionPolicy(
|
||||
};
|
||||
}
|
||||
|
||||
if (explicitImageGenerationRequest) {
|
||||
return {
|
||||
...result,
|
||||
intent: "image_gen",
|
||||
subIntent: undefined,
|
||||
confidence: Math.max(result.confidence, 0.84),
|
||||
needsClarification: false,
|
||||
clarifyingQuestions: [],
|
||||
conversationalReply:
|
||||
result.conversationalReply ||
|
||||
(hasUsableImage
|
||||
? "收到，我会结合你提供的参考图直接执行生图。"
|
||||
: "好的，我来直接帮你生成这张图。"),
|
||||
};
|
||||
}
|
||||
|
||||
if (!hasUsableImage) {
|
||||
if (promptOptimizationOnlyRequest) {
|
||||
return {
|
||||
@ -1815,8 +1831,26 @@ export async function analyzeIntent(params: {
|
||||
* 🆕 现在返回低置信度，强制走LLM分析以获取自然对话回复
|
||||
*/
|
||||
function quickKeywordMatch(text: string, hasImage: boolean): AnalysisResult {
|
||||
// 返回低置信度，强制走LLM分析以获取自然对话回复
|
||||
// 只有LLM不可用时才会使用fallback
|
||||
if (isExplicitVideoGenerationRequest(text)) {
|
||||
return {
|
||||
intent: "video_gen",
|
||||
confidence: 0.95,
|
||||
conversationalReply: hasImage
|
||||
? "收到，我会结合你这轮图片先做视频需求分析，再执行生成。"
|
||||
: "收到，我先做视频需求分析并执行生成。",
|
||||
};
|
||||
}
|
||||
|
||||
if (isExplicitImageGenerationRequest(text)) {
|
||||
return {
|
||||
intent: "image_gen",
|
||||
confidence: 0.95,
|
||||
conversationalReply: hasImage
|
||||
? "收到，我会结合参考图直接帮你生成。"
|
||||
: "好的，我来直接帮你生成这张图。",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
intent: "chat",
|
||||
confidence: 0.3,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user