improve agent planning brief

This commit is contained in:
134 2026-07-08 14:23:38 +08:00
parent 6c89df0f8b
commit 77531aa7b5
9 changed files with 403 additions and 8 deletions

View File

@ -52,6 +52,8 @@ export function normalizeAgentReportHeadingLine(line: string): string {
.trim();
const knownTitles = [
"意图分析",
"任务规划",
"需求理解",
"图片分析",
"执行计划",

View File

@ -138,14 +138,14 @@ export function buildAutoModeFallbackMessage(params: {
: "基于当前上传图片处理。";
return [
"### 需求理解",
"### 意图分析",
`- ${demand}`,
"### 图片分析",
params.imageSummary
? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}`
: `- ${sourceLine}`,
"- 我会优先保持主体、构图和整体风格一致,只处理你指定的改动。",
"### 执行计划",
"### 任务规划",
"- 先完成本轮核心改动,再检查边缘与光影一致性。",
"- 若结果偏差,我会按“保留A,修改B”的方式继续迭代。",
"### 输出设置",
@ -154,9 +154,9 @@ export function buildAutoModeFallbackMessage(params: {
}
return [
"### 需求理解",
"### 意图分析",
`- ${demand}`,
"### 执行计划",
"### 任务规划",
"- 先锁定主体与场景,再细化风格、材质与光影。",
"### 输出设置",
"- 默认单图输出,优先稳定命中。",
@ -222,7 +222,7 @@ export function buildPromptOnlyReply(params: {
export function hasStructuredAgentSections(message?: string): boolean {
if (!message) return false;
return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
return /(?:###\s*意图分析|【\s*意图分析\s*】|###\s*任务规划|【\s*任务规划\s*】|###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
message,
);
}
@ -253,6 +253,13 @@ export function ensureStructuredAgentFriendlyMessage(params: {
return fallback;
}
if (fallback.includes("### 任务规划\n")) {
return fallback.replace(
"### 任务规划\n",
`### 任务规划\n- 补充说明:${normalizedSupplement}\n`,
);
}
return fallback.replace(
"### 执行计划\n",
`### 执行计划\n- 补充说明:${normalizedSupplement}\n`,

View File

@ -545,7 +545,7 @@ function writeCanvasOperationShadowChunk(params: {
}
const AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN =
/(?:^|\n)\s*(?:#{1,6}\s*)?(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)(?:\s|[::]|$)|【\s*(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)\s*】|提示词|自动优化|模型|提交生成|回写|流程|步骤/;
/(?:^|\n)\s*(?:#{1,6}\s*)?(?:意图分析|任务规划|需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)(?:\s|[::]|$)|【\s*(?:意图分析|任务规划|需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)\s*】|提示词|自动优化|模型|提交生成|回写|流程|步骤/;
function buildAgentExecutionPrefaceText(params: {
friendlyMessage?: string | null;

View File

@ -791,6 +791,8 @@ function buildPreviewImageUrl(url: string, width: number): string {
const CANVAS_IMAGE_ANALYSIS_TITLE = "图片分析";
const CANVAS_IMAGE_DONE_SUMMARY = "已完成本次生成,如需微调可继续告诉我。";
const CANVAS_AGENT_SECTION_TITLES = new Set([
"意图分析",
"任务规划",
"需求理解",
CANVAS_IMAGE_ANALYSIS_TITLE,
"执行计划",

View File

@ -47,6 +47,8 @@ type ResponseProps = ComponentProps<typeof Streamdown> & {
const IMAGE_ANALYSIS_TITLE = "图片分析";
const AGENT_SECTION_TITLES = new Set([
"意图分析",
"任务规划",
"需求理解",
IMAGE_ANALYSIS_TITLE,
"执行计划",

View File

@ -174,7 +174,7 @@ function filterAgentRoutingNoise(text: string): string {
const IMAGE_DONE_SUMMARY = "已完成本次生成,如需微调可继续告诉我。";
const ASSISTANT_TYPEWRITER_SECTION_REGEX =
/(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】|###\s*输出设置|【\s*输出设置\s*】)/;
/(?:###\s*意图分析|【\s*意图分析\s*】|###\s*任务规划|【\s*任务规划\s*】|###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】|###\s*输出设置|【\s*输出设置\s*】)/;
const ASSISTANT_TYPEWRITER_STORAGE_KEY = "assistant-typewriter-completed-keys-v1";
const MODEL_BADGE_LINE_REGEX =
/^\s*[◈◇◆🔹⬡✦•·]\s*([^\n]+?)\s*$/im;

View File

@ -19,6 +19,10 @@ import {
normalizeAgentThreadState,
type AgentThreadState,
} from "@/lib/ai/runtime/agent-thread-state";
import {
buildAgentPlanningBriefMarkdown,
isAgentPlanningBriefMarkdown,
} from "@/lib/ai/runtime/agent-planning-brief";
import {
getAiModelAgentCodingModelId,
isAiModelMultimodal,
@ -1153,17 +1157,55 @@ function buildStrategicFriendlyMessage(
if (existingReply) {
const isStructuredReply =
isAgentPlanningBriefMarkdown(existingReply) ||
/###\s*意图分析/.test(existingReply) ||
/###\s*任务规划/.test(existingReply) ||
/###\s*需求理解/.test(existingReply) ||
/###\s*图片分析/.test(existingReply) ||
/###\s*执行计划/.test(existingReply) ||
/【\s*意图分析\s*】/.test(existingReply) ||
/【\s*任务规划\s*】/.test(existingReply) ||
/【\s*需求理解\s*】/.test(existingReply) ||
/【\s*图片分析\s*】/.test(existingReply) ||
/【\s*执行计划\s*】/.test(existingReply);
if (isStructuredReply || existingReply.length >= 24) {
if (isStructuredReply) {
return Promise.resolve(existingReply);
}
}
if (
analysis.intent === "image_edit" ||
analysis.intent === "image_gen" ||
analysis.intent === "video_gen"
) {
const imageParams = heuristicImageParams(state.text, analysis);
const planningBrief = buildAgentPlanningBriefMarkdown({
userText: state.text,
intent: analysis.intent,
subIntent: analysis.subIntent,
nextAction: state.plan?.nextAction,
preferredCapability: state.plan?.preferredCapability,
targetSource: state.plan?.targetSource || state.targetResolutionMode,
hasImage: state.hasImage,
imageFromHistory: state.imageFromHistory,
selectedImageCount: state.selectedImageCount,
currentUploadImageCount: state.currentUploadImageCount,
historyImageCount: state.historyImageCount,
imageCount: imageParams?.imageCount,
aspectRatio: imageParams?.aspectRatio,
imageSize: imageParams?.imageSize,
planGoal: state.plan?.goal,
});
if (process.env.AGENT_LLM_PLANNING_BRIEF_ENABLED !== "true") {
return Promise.resolve(planningBrief);
}
return withTimeoutFallback(
generateAgentConversation(analysis, state),
AGENT_OPERATION_TIMEOUT_MS,
planningBrief,
);
}
return withTimeoutFallback(
generateAgentConversation(analysis, state),
AGENT_OPERATION_TIMEOUT_MS,

View File

@ -0,0 +1,302 @@
export type AgentPlanningBriefIntent =
| "image_edit"
| "vision"
| "image_gen"
| "video_gen"
| "chat";
export type AgentPlanningBriefAction =
| "respond"
| "clarify"
| "analyze_image"
| "generate_image"
| "edit_image"
| "generate_video";
export type AgentPlanningBriefTargetSource =
| "explicit_selection"
| "current_upload"
| "text_reference"
| "working_memory"
| "latest_result"
| "clarification"
| "none";
export type AgentPlanningBriefCapability =
| "respond"
| "analyze"
| "generic_edit"
| "remove_background"
| "remove_watermark"
| "enhance"
| "extend"
| "resize"
| "angle_variant"
| "local_edit"
| "box_cutout"
| "generate_image"
| "generate_video"
| "publish_canvas";
export interface AgentPlanningBriefInput {
userText: string;
intent: AgentPlanningBriefIntent;
subIntent?: string;
nextAction?: AgentPlanningBriefAction;
preferredCapability?: AgentPlanningBriefCapability;
targetSource?: AgentPlanningBriefTargetSource;
hasImage?: boolean;
imageFromHistory?: boolean;
selectedImageCount?: number;
currentUploadImageCount?: number;
historyImageCount?: number;
imageCount?: number;
aspectRatio?: string;
imageSize?: string;
planGoal?: string;
}
export interface AgentTaskGroup {
label: string;
count?: number;
description: string;
}
const CHINESE_NUMBER_VALUES: Record<string, number> = {
一: 1,
二: 2,
两: 2,
三: 3,
四: 4,
五: 5,
六: 6,
七: 7,
八: 8,
九: 9,
十: 10,
};
function compactLine(value: string, maxLength: number) {
const normalized = String(value || "").replace(/\s+/g, " ").trim();
if (normalized.length <= maxLength) return normalized;
return `${normalized.slice(0, maxLength)}...`;
}
function parseCountToken(token?: string | null) {
const value = String(token || "").trim();
if (!value) return undefined;
if (/^\d+$/.test(value)) {
const numeric = Number(value);
return Number.isFinite(numeric) && numeric > 0 ? Math.round(numeric) : undefined;
}
return CHINESE_NUMBER_VALUES[value];
}
function countFromText(text: string) {
const match = String(text || "").match(/([1-9]\d?|一|二|两|三|四|五|六|七|八|九|十)\s*张/);
return parseCountToken(match?.[1]);
}
function normalizeGroupDescription(description: string) {
return String(description || "")
.replace(/^[为是::,,\s]+/, "")
.replace(/[。;;,,]\s*$/, "")
.trim();
}
export function extractAgentTaskGroups(text: string): AgentTaskGroup[] {
const normalized = String(text || "").replace(/\s+/g, " ").trim();
if (!normalized) return [];
const groups: AgentTaskGroup[] = [];
const groupPattern =
/第([一二两三四五六七八九十]|\d{1,2})组(?:为|是|:|:)?([\s\S]*?)(?=(?:第[一二两三四五六七八九十\d]{1,2}组)|[。;;]|$)/g;
let match: RegExpExecArray | null;
while ((match = groupPattern.exec(normalized)) !== null) {
const labelToken = match[1];
const description = normalizeGroupDescription(match[2] || "");
if (!description) continue;
const labelNumber = parseCountToken(labelToken);
groups.push({
label: `第${labelNumber || labelToken}组`,
count: countFromText(description),
description: compactLine(description, 96),
});
}
return groups.slice(0, 8);
}
function getTargetSourceLabel(source?: AgentPlanningBriefTargetSource) {
switch (source) {
case "explicit_selection":
return "画布选中图片";
case "current_upload":
return "当前上传图片";
case "text_reference":
return "文本指定图片";
case "working_memory":
return "会话工作记忆图片";
case "latest_result":
return "最近生成结果";
case "clarification":
return "待确认目标";
case "none":
default:
return "无需现有图片";
}
}
function getCapabilityLabel(capability?: AgentPlanningBriefCapability, subIntent?: string) {
switch (capability || subIntent) {
case "remove_background":
return "主体抠图/去背景";
case "remove_watermark":
return "去水印";
case "enhance":
return "清晰度增强";
case "extend":
return "扩图/画幅延展";
case "resize":
return "尺寸调整";
case "angle_variant":
case "generate_angle_variant":
return "多视角变体";
case "local_edit":
case "inpaint_region":
return "局部编辑";
case "box_cutout":
return "框选抠图";
case "generate_video":
return "视频生成";
case "generate_image":
return "图片生成";
case "analyze":
return "图片分析";
default:
return "图片编辑";
}
}
function inferTotalCount(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) {
if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) {
return Math.max(1, Math.round(params.imageCount));
}
const groupedTotal = groups.reduce((sum, group) => sum + (group.count || 0), 0);
return groupedTotal > 0 ? groupedTotal : undefined;
}
function buildIntentLine(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) {
const target = getTargetSourceLabel(params.targetSource);
const capability = getCapabilityLabel(params.preferredCapability, params.subIntent);
const totalCount = inferTotalCount(params, groups);
const demand = compactLine(params.planGoal || params.userText || "", 120);
if (params.intent === "image_gen") {
const countText = totalCount ? `,共 ${totalCount} 张` : "";
const groupText = groups.length > 0 ? `,拆成 ${groups.length} 组` : "";
const referenceText = params.hasImage ? ",并保持参考图主体核心特征一致" : "";
return `用户需要生成新图片${groupText}${countText}${referenceText};核心目标是:${demand || "完成本轮生图任务"}。`;
}
if (params.intent === "image_edit") {
return `用户需要基于${target}执行${capability};核心目标是:${demand || "完成本轮图片处理"}。`;
}
if (params.intent === "video_gen") {
return `用户需要生成视频;先拆解镜头、主体连续性和运动节奏,再提交视频生成。`;
}
if (params.intent === "vision") {
return `用户需要理解或分析图片内容;先观察主体、场景、构图和潜在问题,再给出结论。`;
}
return `用户当前主要是在对话或咨询;先回答问题,不主动触发生图或修图工具。`;
}
function buildImageGenerationPlan(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) {
if (groups.length > 0) {
return groups.map((group, index) => {
const countText = group.count ? `${group.count}张` : "一组";
return `${index + 1}. ${group.label}:生成${countText}${group.description.replace(/^\d+\s*张/, "")},保持主体识别一致。`;
});
}
const count = inferTotalCount(params, groups) || 1;
return [
`1. 锁定主体、场景、风格和构图约束,整理成可执行提示词。`,
`2. 生成 ${count} 张候选图,优先保证主体一致性和画面完成度。`,
`3. 输出结果并保留可继续追改的上下文。`,
];
}
function buildImageEditPlan(params: AgentPlanningBriefInput) {
const target = getTargetSourceLabel(params.targetSource);
const capability = getCapabilityLabel(params.preferredCapability, params.subIntent);
return [
`1. 先锁定${target}作为本轮处理目标,避免误处理其他图片。`,
`2. 调用${capability}能力执行核心改动,同时尽量保留主体结构和画质。`,
`3. 输出处理结果,并保留原图用于对比或继续微调。`,
];
}
function buildGenericPlan(params: AgentPlanningBriefInput) {
if (params.intent === "video_gen") {
return [
"1. 拆解视频主体、场景、动作节奏和镜头关系。",
"2. 生成脚本与分镜,并整理可直接提交的视频提示词。",
"3. 按规划提交视频生成,返回结果后支持继续调整。",
];
}
if (params.intent === "vision") {
return [
"1. 先识别图片主体、背景、构图和关键信息。",
"2. 结合用户问题输出判断、建议或可执行修改方向。",
];
}
return ["1. 先直接回答用户问题;如果需要执行工具,再根据明确指令进入下一步。"];
}
function buildOutputSettings(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) {
const lines: string[] = [];
const totalCount = inferTotalCount(params, groups);
if (params.intent === "image_gen" || params.intent === "image_edit") {
lines.push(`- 输出数量:${params.intent === "image_edit" ? "1 张处理结果" : `${totalCount || 1} 张`}`);
lines.push(`- 图片比例:${params.aspectRatio && params.aspectRatio !== "auto" ? params.aspectRatio : "按需求自适应"}`);
lines.push(`- 清晰度:${params.imageSize || (params.intent === "image_edit" ? "4K优先" : "2K优先")}`);
if (params.intent === "image_edit") {
lines.push(`- 目标来源:${getTargetSourceLabel(params.targetSource)}`);
}
} else if (params.intent === "video_gen") {
lines.push("- 输出形式:视频结果 + 可继续追改的分镜上下文");
} else {
lines.push("- 输出形式:直接回答,不主动触发生成工具");
}
return lines;
}
export function buildAgentPlanningBriefMarkdown(params: AgentPlanningBriefInput) {
const groups = extractAgentTaskGroups(params.userText);
const planLines =
params.intent === "image_gen"
? buildImageGenerationPlan(params, groups)
: params.intent === "image_edit"
? buildImageEditPlan(params)
: buildGenericPlan(params);
const outputSettings = buildOutputSettings(params, groups);
return [
"### 意图分析",
`- ${buildIntentLine(params, groups)}`,
"### 任务规划",
...planLines.map((line) => (line.startsWith("- ") ? line : `- ${line}`)),
"### 输出设置",
...outputSettings,
].join("\n");
}
export function isAgentPlanningBriefMarkdown(text: string) {
const normalized = String(text || "");
return /###\s*意图分析|【\s*意图分析\s*】/.test(normalized) &&
/###\s*任务规划|【\s*任务规划\s*】/.test(normalized);
}

View File

@ -54,6 +54,11 @@ import {
} from "@/lib/ai/runtime/canvas-agent-kernel";
import { isExplicitImageGenerationRequest } from "@/lib/ai/smart-analyzer";
import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent";
import {
buildAgentPlanningBriefMarkdown,
extractAgentTaskGroups,
isAgentPlanningBriefMarkdown,
} from "@/lib/ai/runtime/agent-planning-brief";
import {
buildRecentDialogueMemory,
isRecentDialogueRecallRequest,
@ -385,6 +390,39 @@ test.describe("canvas chat pure helpers", () => {
expect(inferAgentImageCountFromText("再出30张", 12)).toBe(12);
});
test("builds market-grade planning brief for grouped image generation requests", () => {
const request =
"基于上传的潮玩角色图片,生成3组共12张图片,要求保持角色核心特征一致。第一组为4张动作表情夸张的半身表情包;第二组为4张不同场景的角色活动照;第三组为4张服装发型差异巨大的造型照,所有图片比例保持3:4。";
const groups = extractAgentTaskGroups(request);
expect(groups).toHaveLength(3);
expect(groups.map((group) => group.count)).toEqual([4, 4, 4]);
expect(groups[0].description).toContain("半身表情包");
expect(groups[1].description).toContain("不同场景");
expect(groups[2].description).toContain("服装发型");
const markdown = buildAgentPlanningBriefMarkdown({
userText: request,
intent: "image_gen",
nextAction: "generate_image",
preferredCapability: "generate_image",
targetSource: "current_upload",
hasImage: true,
imageCount: 12,
aspectRatio: "3:4",
imageSize: "2K",
});
expect(isAgentPlanningBriefMarkdown(markdown)).toBe(true);
expect(markdown).toContain("### 意图分析");
expect(markdown).toContain("拆成 3 组,共 12 张");
expect(markdown).toContain("### 任务规划");
expect(markdown).toContain("第1组:生成4张");
expect(markdown).toContain("第2组:生成4张");
expect(markdown).toContain("第3组:生成4张");
expect(markdown).toContain("图片比例:3:4");
});
test("keeps canvas message empty-history access scoped and explicit", () => {
const canvasChatId = "11111111-1111-4111-8111-111111111111";
const agentScopedChatId = "11111111-1111-4111-8111-a93e17a60001";