From 77531aa7b5612f164c2ad7799d5f7194dab9c53c Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 14:23:38 +0800 Subject: [PATCH] improve agent planning brief --- app/(chat)/api/chat/_lib/agent-output.ts | 2 + app/(chat)/api/chat/_lib/image-media.ts | 17 +- app/(chat)/api/chat/route.ts | 2 +- components/canvas/CanvasChatPanel.tsx | 2 + components/elements/response.tsx | 2 + components/message.tsx | 2 +- lib/ai/auto-agent-graph.ts | 44 +++- lib/ai/runtime/agent-planning-brief.ts | 302 +++++++++++++++++++++++ tests/routes/canvas-chat-helpers.test.ts | 38 +++ 9 files changed, 403 insertions(+), 8 deletions(-) create mode 100644 lib/ai/runtime/agent-planning-brief.ts diff --git a/app/(chat)/api/chat/_lib/agent-output.ts b/app/(chat)/api/chat/_lib/agent-output.ts index 9b04ebc34..9d9334c47 100644 --- a/app/(chat)/api/chat/_lib/agent-output.ts +++ b/app/(chat)/api/chat/_lib/agent-output.ts @@ -52,6 +52,8 @@ export function normalizeAgentReportHeadingLine(line: string): string { .trim(); const knownTitles = [ + "意图分析", + "任务规划", "需求理解", "图片分析", "执行计划", diff --git a/app/(chat)/api/chat/_lib/image-media.ts b/app/(chat)/api/chat/_lib/image-media.ts index 5b5a8bfac..844f6aa92 100644 --- a/app/(chat)/api/chat/_lib/image-media.ts +++ b/app/(chat)/api/chat/_lib/image-media.ts @@ -138,14 +138,14 @@ export function buildAutoModeFallbackMessage(params: { : "基于当前上传图片处理。"; return [ - "### 需求理解", + "### 意图分析", `- ${demand}`, "### 图片分析", params.imageSummary ? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}` : `- ${sourceLine}`, "- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。", - "### 执行计划", + "### 任务规划", "- 先完成本轮核心改动，再检查边缘与光影一致性。", "- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。", "### 输出设置", @@ -154,9 +154,9 @@ export function buildAutoModeFallbackMessage(params: { } return [ - "### 需求理解", + "### 意图分析", `- ${demand}`, - "### 执行计划", + "### 任务规划", "- 先锁定主体与场景，再细化风格、材质与光影。", "### 输出设置", "- 默认单图输出，优先稳定命中。", @@ -222,7 +222,7 @@ export function buildPromptOnlyReply(params: { export function hasStructuredAgentSections(message?: string): boolean { if (!message) return false; - return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( + return /(?:###\s*意图分析|【\s*意图分析\s*】|###\s*任务规划|【\s*任务规划\s*】|###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( message, ); } @@ -253,6 +253,13 @@ export function ensureStructuredAgentFriendlyMessage(params: { return fallback; } + if (fallback.includes("### 任务规划\n")) { + return fallback.replace( + "### 任务规划\n", + `### 任务规划\n- 补充说明：${normalizedSupplement}\n`, + ); + } + return fallback.replace( "### 执行计划\n", `### 执行计划\n- 补充说明：${normalizedSupplement}\n`, diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index fcbfd0b3d..93fd9f99c 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -545,7 +545,7 @@ function writeCanvasOperationShadowChunk(params: { } const AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN = - /(?:^|\n)\s*(?:#{1,6}\s*)?(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)(?:\s|[:：]|$)|【\s*(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)\s*】|提示词|自动优化|模型|提交生成|回写|流程|步骤/; + /(?:^|\n)\s*(?:#{1,6}\s*)?(?:意图分析|任务规划|需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)(?:\s|[:：]|$)|【\s*(?:意图分析|任务规划|需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)\s*】|提示词|自动优化|模型|提交生成|回写|流程|步骤/; function buildAgentExecutionPrefaceText(params: { friendlyMessage?: string | null; diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index cc450b929..b8bd5e92b 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -791,6 +791,8 @@ function buildPreviewImageUrl(url: string, width: number): string { const CANVAS_IMAGE_ANALYSIS_TITLE = "图片分析"; const CANVAS_IMAGE_DONE_SUMMARY = "已完成本次生成，如需微调可继续告诉我。"; const CANVAS_AGENT_SECTION_TITLES = new Set([ + "意图分析", + "任务规划", "需求理解", CANVAS_IMAGE_ANALYSIS_TITLE, "执行计划", diff --git a/components/elements/response.tsx b/components/elements/response.tsx index 5ce92c208..b908c4b7e 100644 --- a/components/elements/response.tsx +++ b/components/elements/response.tsx @@ -47,6 +47,8 @@ type ResponseProps = ComponentProps & { const IMAGE_ANALYSIS_TITLE = "图片分析"; const AGENT_SECTION_TITLES = new Set([ + "意图分析", + "任务规划", "需求理解", IMAGE_ANALYSIS_TITLE, "执行计划", diff --git a/components/message.tsx b/components/message.tsx index 4ff575742..6aa4d0c1b 100644 --- a/components/message.tsx +++ b/components/message.tsx @@ -174,7 +174,7 @@ function filterAgentRoutingNoise(text: string): string { const IMAGE_DONE_SUMMARY = "已完成本次生成，如需微调可继续告诉我。"; const ASSISTANT_TYPEWRITER_SECTION_REGEX = - /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】|###\s*输出设置|【\s*输出设置\s*】)/; + /(?:###\s*意图分析|【\s*意图分析\s*】|###\s*任务规划|【\s*任务规划\s*】|###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】|###\s*输出设置|【\s*输出设置\s*】)/; const ASSISTANT_TYPEWRITER_STORAGE_KEY = "assistant-typewriter-completed-keys-v1"; const MODEL_BADGE_LINE_REGEX = /^\s*[◈◇◆🔹⬡✦•·]\s*([^\n]+?)\s*$/im; diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index bf3b9fd64..ed723f6c3 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -19,6 +19,10 @@ import { normalizeAgentThreadState, type AgentThreadState, } from "@/lib/ai/runtime/agent-thread-state"; +import { + buildAgentPlanningBriefMarkdown, + isAgentPlanningBriefMarkdown, +} from "@/lib/ai/runtime/agent-planning-brief"; import { getAiModelAgentCodingModelId, isAiModelMultimodal, @@ -1153,17 +1157,55 @@ function buildStrategicFriendlyMessage( if (existingReply) { const isStructuredReply = + isAgentPlanningBriefMarkdown(existingReply) || + /###\s*意图分析/.test(existingReply) || + /###\s*任务规划/.test(existingReply) || /###\s*需求理解/.test(existingReply) || /###\s*图片分析/.test(existingReply) || /###\s*执行计划/.test(existingReply) || + /【\s*意图分析\s*】/.test(existingReply) || + /【\s*任务规划\s*】/.test(existingReply) || /【\s*需求理解\s*】/.test(existingReply) || /【\s*图片分析\s*】/.test(existingReply) || /【\s*执行计划\s*】/.test(existingReply); - if (isStructuredReply || existingReply.length >= 24) { + if (isStructuredReply) { return Promise.resolve(existingReply); } } + if ( + analysis.intent === "image_edit" || + analysis.intent === "image_gen" || + analysis.intent === "video_gen" + ) { + const imageParams = heuristicImageParams(state.text, analysis); + const planningBrief = buildAgentPlanningBriefMarkdown({ + userText: state.text, + intent: analysis.intent, + subIntent: analysis.subIntent, + nextAction: state.plan?.nextAction, + preferredCapability: state.plan?.preferredCapability, + targetSource: state.plan?.targetSource || state.targetResolutionMode, + hasImage: state.hasImage, + imageFromHistory: state.imageFromHistory, + selectedImageCount: state.selectedImageCount, + currentUploadImageCount: state.currentUploadImageCount, + historyImageCount: state.historyImageCount, + imageCount: imageParams?.imageCount, + aspectRatio: imageParams?.aspectRatio, + imageSize: imageParams?.imageSize, + planGoal: state.plan?.goal, + }); + if (process.env.AGENT_LLM_PLANNING_BRIEF_ENABLED !== "true") { + return Promise.resolve(planningBrief); + } + return withTimeoutFallback( + generateAgentConversation(analysis, state), + AGENT_OPERATION_TIMEOUT_MS, + planningBrief, + ); + } + return withTimeoutFallback( generateAgentConversation(analysis, state), AGENT_OPERATION_TIMEOUT_MS, diff --git a/lib/ai/runtime/agent-planning-brief.ts b/lib/ai/runtime/agent-planning-brief.ts new file mode 100644 index 000000000..943a6db17 --- /dev/null +++ b/lib/ai/runtime/agent-planning-brief.ts @@ -0,0 +1,302 @@ +export type AgentPlanningBriefIntent = + | "image_edit" + | "vision" + | "image_gen" + | "video_gen" + | "chat"; + +export type AgentPlanningBriefAction = + | "respond" + | "clarify" + | "analyze_image" + | "generate_image" + | "edit_image" + | "generate_video"; + +export type AgentPlanningBriefTargetSource = + | "explicit_selection" + | "current_upload" + | "text_reference" + | "working_memory" + | "latest_result" + | "clarification" + | "none"; + +export type AgentPlanningBriefCapability = + | "respond" + | "analyze" + | "generic_edit" + | "remove_background" + | "remove_watermark" + | "enhance" + | "extend" + | "resize" + | "angle_variant" + | "local_edit" + | "box_cutout" + | "generate_image" + | "generate_video" + | "publish_canvas"; + +export interface AgentPlanningBriefInput { + userText: string; + intent: AgentPlanningBriefIntent; + subIntent?: string; + nextAction?: AgentPlanningBriefAction; + preferredCapability?: AgentPlanningBriefCapability; + targetSource?: AgentPlanningBriefTargetSource; + hasImage?: boolean; + imageFromHistory?: boolean; + selectedImageCount?: number; + currentUploadImageCount?: number; + historyImageCount?: number; + imageCount?: number; + aspectRatio?: string; + imageSize?: string; + planGoal?: string; +} + +export interface AgentTaskGroup { + label: string; + count?: number; + description: string; +} + +const CHINESE_NUMBER_VALUES: Record = { + 一: 1, + 二: 2, + 两: 2, + 三: 3, + 四: 4, + 五: 5, + 六: 6, + 七: 7, + 八: 8, + 九: 9, + 十: 10, +}; + +function compactLine(value: string, maxLength: number) { + const normalized = String(value || "").replace(/\s+/g, " ").trim(); + if (normalized.length <= maxLength) return normalized; + return `${normalized.slice(0, maxLength)}...`; +} + +function parseCountToken(token?: string | null) { + const value = String(token || "").trim(); + if (!value) return undefined; + if (/^\d+$/.test(value)) { + const numeric = Number(value); + return Number.isFinite(numeric) && numeric > 0 ? Math.round(numeric) : undefined; + } + return CHINESE_NUMBER_VALUES[value]; +} + +function countFromText(text: string) { + const match = String(text || "").match(/([1-9]\d?|一|二|两|三|四|五|六|七|八|九|十)\s*张/); + return parseCountToken(match?.[1]); +} + +function normalizeGroupDescription(description: string) { + return String(description || "") + .replace(/^[为是:：，,\s]+/, "") + .replace(/[。；;，,]\s*$/, "") + .trim(); +} + +export function extractAgentTaskGroups(text: string): AgentTaskGroup[] { + const normalized = String(text || "").replace(/\s+/g, " ").trim(); + if (!normalized) return []; + + const groups: AgentTaskGroup[] = []; + const groupPattern = + /第([一二两三四五六七八九十]|\d{1,2})组(?:为|是|:|：)?([\s\S]*?)(?=(?:第[一二两三四五六七八九十\d]{1,2}组)|[。；;]|$)/g; + let match: RegExpExecArray | null; + while ((match = groupPattern.exec(normalized)) !== null) { + const labelToken = match[1]; + const description = normalizeGroupDescription(match[2] || ""); + if (!description) continue; + const labelNumber = parseCountToken(labelToken); + groups.push({ + label: `第${labelNumber || labelToken}组`, + count: countFromText(description), + description: compactLine(description, 96), + }); + } + + return groups.slice(0, 8); +} + +function getTargetSourceLabel(source?: AgentPlanningBriefTargetSource) { + switch (source) { + case "explicit_selection": + return "画布选中图片"; + case "current_upload": + return "当前上传图片"; + case "text_reference": + return "文本指定图片"; + case "working_memory": + return "会话工作记忆图片"; + case "latest_result": + return "最近生成结果"; + case "clarification": + return "待确认目标"; + case "none": + default: + return "无需现有图片"; + } +} + +function getCapabilityLabel(capability?: AgentPlanningBriefCapability, subIntent?: string) { + switch (capability || subIntent) { + case "remove_background": + return "主体抠图/去背景"; + case "remove_watermark": + return "去水印"; + case "enhance": + return "清晰度增强"; + case "extend": + return "扩图/画幅延展"; + case "resize": + return "尺寸调整"; + case "angle_variant": + case "generate_angle_variant": + return "多视角变体"; + case "local_edit": + case "inpaint_region": + return "局部编辑"; + case "box_cutout": + return "框选抠图"; + case "generate_video": + return "视频生成"; + case "generate_image": + return "图片生成"; + case "analyze": + return "图片分析"; + default: + return "图片编辑"; + } +} + +function inferTotalCount(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) { + if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) { + return Math.max(1, Math.round(params.imageCount)); + } + const groupedTotal = groups.reduce((sum, group) => sum + (group.count || 0), 0); + return groupedTotal > 0 ? groupedTotal : undefined; +} + +function buildIntentLine(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) { + const target = getTargetSourceLabel(params.targetSource); + const capability = getCapabilityLabel(params.preferredCapability, params.subIntent); + const totalCount = inferTotalCount(params, groups); + const demand = compactLine(params.planGoal || params.userText || "", 120); + + if (params.intent === "image_gen") { + const countText = totalCount ? `，共 ${totalCount} 张` : ""; + const groupText = groups.length > 0 ? `，拆成 ${groups.length} 组` : ""; + const referenceText = params.hasImage ? "，并保持参考图主体核心特征一致" : ""; + return `用户需要生成新图片${groupText}${countText}${referenceText}；核心目标是：${demand || "完成本轮生图任务"}。`; + } + + if (params.intent === "image_edit") { + return `用户需要基于${target}执行${capability}；核心目标是：${demand || "完成本轮图片处理"}。`; + } + + if (params.intent === "video_gen") { + return `用户需要生成视频；先拆解镜头、主体连续性和运动节奏，再提交视频生成。`; + } + + if (params.intent === "vision") { + return `用户需要理解或分析图片内容；先观察主体、场景、构图和潜在问题，再给出结论。`; + } + + return `用户当前主要是在对话或咨询；先回答问题，不主动触发生图或修图工具。`; +} + +function buildImageGenerationPlan(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) { + if (groups.length > 0) { + return groups.map((group, index) => { + const countText = group.count ? `${group.count}张` : "一组"; + return `${index + 1}. ${group.label}：生成${countText}${group.description.replace(/^\d+\s*张/, "")}，保持主体识别一致。`; + }); + } + + const count = inferTotalCount(params, groups) || 1; + return [ + `1. 锁定主体、场景、风格和构图约束，整理成可执行提示词。`, + `2. 生成 ${count} 张候选图，优先保证主体一致性和画面完成度。`, + `3. 输出结果并保留可继续追改的上下文。`, + ]; +} + +function buildImageEditPlan(params: AgentPlanningBriefInput) { + const target = getTargetSourceLabel(params.targetSource); + const capability = getCapabilityLabel(params.preferredCapability, params.subIntent); + return [ + `1. 先锁定${target}作为本轮处理目标，避免误处理其他图片。`, + `2. 调用${capability}能力执行核心改动，同时尽量保留主体结构和画质。`, + `3. 输出处理结果，并保留原图用于对比或继续微调。`, + ]; +} + +function buildGenericPlan(params: AgentPlanningBriefInput) { + if (params.intent === "video_gen") { + return [ + "1. 拆解视频主体、场景、动作节奏和镜头关系。", + "2. 生成脚本与分镜，并整理可直接提交的视频提示词。", + "3. 按规划提交视频生成，返回结果后支持继续调整。", + ]; + } + if (params.intent === "vision") { + return [ + "1. 先识别图片主体、背景、构图和关键信息。", + "2. 结合用户问题输出判断、建议或可执行修改方向。", + ]; + } + return ["1. 先直接回答用户问题；如果需要执行工具，再根据明确指令进入下一步。"]; +} + +function buildOutputSettings(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) { + const lines: string[] = []; + const totalCount = inferTotalCount(params, groups); + if (params.intent === "image_gen" || params.intent === "image_edit") { + lines.push(`- 输出数量：${params.intent === "image_edit" ? "1 张处理结果" : `${totalCount || 1} 张`}`); + lines.push(`- 图片比例：${params.aspectRatio && params.aspectRatio !== "auto" ? params.aspectRatio : "按需求自适应"}`); + lines.push(`- 清晰度：${params.imageSize || (params.intent === "image_edit" ? "4K优先" : "2K优先")}`); + if (params.intent === "image_edit") { + lines.push(`- 目标来源：${getTargetSourceLabel(params.targetSource)}`); + } + } else if (params.intent === "video_gen") { + lines.push("- 输出形式：视频结果 + 可继续追改的分镜上下文"); + } else { + lines.push("- 输出形式：直接回答，不主动触发生成工具"); + } + return lines; +} + +export function buildAgentPlanningBriefMarkdown(params: AgentPlanningBriefInput) { + const groups = extractAgentTaskGroups(params.userText); + const planLines = + params.intent === "image_gen" + ? buildImageGenerationPlan(params, groups) + : params.intent === "image_edit" + ? buildImageEditPlan(params) + : buildGenericPlan(params); + const outputSettings = buildOutputSettings(params, groups); + + return [ + "### 意图分析", + `- ${buildIntentLine(params, groups)}`, + "### 任务规划", + ...planLines.map((line) => (line.startsWith("- ") ? line : `- ${line}`)), + "### 输出设置", + ...outputSettings, + ].join("\n"); +} + +export function isAgentPlanningBriefMarkdown(text: string) { + const normalized = String(text || ""); + return /###\s*意图分析|【\s*意图分析\s*】/.test(normalized) && + /###\s*任务规划|【\s*任务规划\s*】/.test(normalized); +} diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index ee23944a0..d2175bb3d 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -54,6 +54,11 @@ import { } from "@/lib/ai/runtime/canvas-agent-kernel"; import { isExplicitImageGenerationRequest } from "@/lib/ai/smart-analyzer"; import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent"; +import { + buildAgentPlanningBriefMarkdown, + extractAgentTaskGroups, + isAgentPlanningBriefMarkdown, +} from "@/lib/ai/runtime/agent-planning-brief"; import { buildRecentDialogueMemory, isRecentDialogueRecallRequest, @@ -385,6 +390,39 @@ test.describe("canvas chat pure helpers", () => { expect(inferAgentImageCountFromText("再出30张", 12)).toBe(12); }); + test("builds market-grade planning brief for grouped image generation requests", () => { + const request = + "基于上传的潮玩角色图片，生成3组共12张图片，要求保持角色核心特征一致。第一组为4张动作表情夸张的半身表情包；第二组为4张不同场景的角色活动照；第三组为4张服装发型差异巨大的造型照，所有图片比例保持3:4。"; + + const groups = extractAgentTaskGroups(request); + expect(groups).toHaveLength(3); + expect(groups.map((group) => group.count)).toEqual([4, 4, 4]); + expect(groups[0].description).toContain("半身表情包"); + expect(groups[1].description).toContain("不同场景"); + expect(groups[2].description).toContain("服装发型"); + + const markdown = buildAgentPlanningBriefMarkdown({ + userText: request, + intent: "image_gen", + nextAction: "generate_image", + preferredCapability: "generate_image", + targetSource: "current_upload", + hasImage: true, + imageCount: 12, + aspectRatio: "3:4", + imageSize: "2K", + }); + + expect(isAgentPlanningBriefMarkdown(markdown)).toBe(true); + expect(markdown).toContain("### 意图分析"); + expect(markdown).toContain("拆成 3 组，共 12 张"); + expect(markdown).toContain("### 任务规划"); + expect(markdown).toContain("第1组：生成4张"); + expect(markdown).toContain("第2组：生成4张"); + expect(markdown).toContain("第3组：生成4张"); + expect(markdown).toContain("图片比例：3:4"); + }); + test("keeps canvas message empty-history access scoped and explicit", () => { const canvasChatId = "11111111-1111-4111-8111-111111111111"; const agentScopedChatId = "11111111-1111-4111-8111-a93e17a60001";