From 67d834f0dd2e243b0337bb0ac9947a80e602d030 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 13 Mar 2026 11:35:07 +0800 Subject: [PATCH] Refine agent text mode workflow --- app/(chat)/api/chat/route.ts | 35 ++++++++++++-- lib/ai/prompt-memory.ts | 92 ++++++++++++++++++++++++++++++++++++ lib/db/queries.ts | 14 +++++- 3 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 lib/ai/prompt-memory.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 9cc24944f..ea56c72bb 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -98,6 +98,11 @@ import type { } from "@/lib/ai/context-memory/types"; import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; +import { + buildPromptMemoryUpdate, + extractPromptMemoryFromWorkingState, + isPromptCarryForwardRequest, +} from "@/lib/ai/prompt-memory"; import { uploadFile } from "@/lib/storage/service"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline"; @@ -3059,6 +3064,7 @@ export async function POST(request: Request) { | Awaited> | null = null; let storyboardPreferenceMemory = null; + let reusablePromptMemory = null; let smartAutoHistoryMessagesFromDb: DBMessage[] = []; let preferredAgentModelId: string | undefined; let pendingGridConfirmation: @@ -3186,6 +3192,7 @@ export async function POST(request: Request) { return null; }); storyboardPreferenceMemory = extractStoryboardPreferenceMemory(requestWorkingState); + reusablePromptMemory = extractPromptMemoryFromWorkingState(requestWorkingState); pendingGridConfirmation = extractPendingGridConfirmationFromWorkingState(requestWorkingState); @@ -3234,9 +3241,22 @@ export async function POST(request: Request) { chatId: id, }); } - confirmedPromptForGeneration = extractConfirmedPromptFromWorkingState(requestWorkingState); + confirmedPromptForGeneration = + extractConfirmedPromptFromWorkingState(requestWorkingState) || + reusablePromptMemory?.optimizedPrompt; + const hasImageInputForPromptCarryForward = + message.parts.some((part: any) => + part?.type === "file" || part?.type === "image" || part?.type === "image_url" + ); + const isPromptCarryForwardExecution = + Boolean(confirmedPromptForGeneration) && + isPromptCarryForwardRequest({ + text: requestTextContent, + hasImage: hasImageInputForPromptCarryForward, + }); forcePromptDrivenImageGeneration = - isConfirmedPromptGenerationRequest(requestTextContent) && + (isConfirmedPromptGenerationRequest(requestTextContent) || + isPromptCarryForwardExecution) && Boolean(confirmedPromptForGeneration); if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ @@ -3457,7 +3477,10 @@ export async function POST(request: Request) { availableTools: [], }; smartAutoFriendlyMessage = - smartAutoFriendlyMessage || "收到，我会按你刚确认的提示词直接生图。"; + smartAutoFriendlyMessage || + (isPromptCarryForwardExecution + ? "收到，我会使用上一轮优化后的提示词，结合你这轮图片继续执行。" + : "收到，我会按你刚确认的提示词直接生图。"); } if (smartAutoPlan) { @@ -4689,6 +4712,12 @@ export async function POST(request: Request) { resolutionMode: "prompt_only", confidence: 90, optimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000), + promptMemory: buildPromptMemoryUpdate({ + optimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000), + sourceIntent: "image_gen", + sourceText: requestTextContent.slice(0, 500), + reusable: true, + }), imageSummary: promptOnlyImageSummary.slice(0, 1200), sourceImageUrls: promptOnlySourceImageUrls, }, diff --git a/lib/ai/prompt-memory.ts b/lib/ai/prompt-memory.ts new file mode 100644 index 000000000..0bfa9b0ad --- /dev/null +++ b/lib/ai/prompt-memory.ts @@ -0,0 +1,92 @@ +import "server-only"; + +export interface PromptMemoryEntry { + optimizedPrompt: string; + sourceIntent?: string; + sourceText?: string; + reusable?: boolean; +} + +function normalizeText(value: unknown, max = 2000): string | undefined { + const normalized = String(value || "").trim(); + if (!normalized) return undefined; + return normalized.slice(0, max); +} + +export function extractPromptMemoryFromWorkingState( + workingState?: { + lastTurnSummary?: unknown; + } | null, +): PromptMemoryEntry | null { + const summary = workingState?.lastTurnSummary; + if (!summary || typeof summary !== "object") return null; + const summaryRecord = summary as Record; + const memory = summaryRecord.promptMemory; + if (memory && typeof memory === "object") { + const memoryRecord = memory as Record; + const optimizedPrompt = normalizeText(memoryRecord.optimizedPrompt, 4000); + if (!optimizedPrompt) return null; + return { + optimizedPrompt, + sourceIntent: normalizeText(memoryRecord.sourceIntent, 64), + sourceText: normalizeText(memoryRecord.sourceText, 500), + reusable: + typeof memoryRecord.reusable === "boolean" ? memoryRecord.reusable : true, + }; + } + + const resolutionMode = normalizeText(summaryRecord.resolutionMode, 64); + const optimizedPrompt = normalizeText(summaryRecord.optimizedPrompt, 4000); + if (resolutionMode === "prompt_only" && optimizedPrompt) { + return { + optimizedPrompt, + sourceIntent: "image_gen", + sourceText: normalizeText(summaryRecord.text, 500), + reusable: true, + }; + } + return null; +} + +export function buildPromptMemoryUpdate(params: { + optimizedPrompt: string; + sourceIntent?: string; + sourceText?: string; + reusable?: boolean; +}): PromptMemoryEntry { + return { + optimizedPrompt: normalizeText(params.optimizedPrompt, 4000) || "", + sourceIntent: normalizeText(params.sourceIntent, 64), + sourceText: normalizeText(params.sourceText, 500), + reusable: params.reusable !== false, + }; +} + +export function isPromptCarryForwardRequest(params: { + text: string; + hasImage: boolean; +}): boolean { + const normalized = String(params.text || "").toLowerCase().replace(/\s+/g, ""); + if (!normalized) return false; + + const refersPreviousPrompt = + /根据(上一个|上一句|刚才|上面|之前|那个)?提示词|按(上一个|上一句|刚才|上面|之前|那个)?提示词|用(上一个|上一句|刚才|上面|之前|那个)?提示词|用刚才输出的提示词|按刚才输出的提示词|按上面的提示词|沿用上个提示词|继续用那个提示词/.test( + normalized, + ); + + if (!refersPreviousPrompt) return false; + + const wantsExecution = + /生图|生成图片|出图|画出来|做图|继续生成|继续做|重新做|重绘|优化这张图|优化这个图|处理这张图|处理这个图|按这个做|照这个做|按此生成/.test( + normalized, + ); + + if (wantsExecution) return true; + + if (params.hasImage) { + return /优化|处理|重绘|参考这张图|结合这张图|基于这张图/.test(normalized); + } + + return false; +} + diff --git a/lib/db/queries.ts b/lib/db/queries.ts index 4cbe2267a..a4bd06c0b 100644 --- a/lib/db/queries.ts +++ b/lib/db/queries.ts @@ -2517,7 +2517,12 @@ export async function createAgentPreset(params: { slots: params.slots, }); - const requiredUploadCount = validatedSlots.some((slot) => slot.requireUpload !== false) ? 3 : 0; + const requiredUploadCount = + params.type === "text" + ? 0 + : validatedSlots.some((slot) => slot.requireUpload !== false) + ? 3 + : 0; const avatarAssetId = String(params.avatarAssetId || "").trim() || null; if (avatarAssetId) { await validateOwnedReferenceAssets({ @@ -2705,7 +2710,12 @@ export async function updateAgentPreset(params: { userId: params.userId, slots: normalizedSlotsInput, }); - const requiredUploadCount = validatedSlots.some((slot) => slot.requireUpload !== false) ? 3 : 0; + const requiredUploadCount = + nextType === "text" + ? 0 + : validatedSlots.some((slot) => slot.requireUpload !== false) + ? 3 + : 0; const nextAvatarAssetId = params.avatarAssetId !== undefined ? String(params.avatarAssetId || "").trim() || null