diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index ea56c72bb..581f34056 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -90,6 +90,16 @@ import { message as messageTable, } from "@/lib/db/schema"; import { generateCanvasImage } from "@/lib/canvas/generate-image"; +import { + buildGridConfirmationMessage, + detectRequestedStoryboardGrid, + extractPendingGridConfirmationFromWorkingState, + isGridConfirmationReply, + type PendingGridConfirmation, + resolveConversationAction, +} from "@/lib/ai/conversation-action-resolver"; +import { composeConversationMemory } from "@/lib/ai/conversation-memory-composer"; +import { buildConversationThreadState } from "@/lib/ai/conversation-thread-state-builder"; import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input"; import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution"; import type { @@ -98,10 +108,13 @@ import type { } from "@/lib/ai/context-memory/types"; import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; +import { + buildImageWorkflowMemoryUpdate, + extractImageWorkflowMemory, +} from "@/lib/ai/image-workflow-memory"; import { buildPromptMemoryUpdate, extractPromptMemoryFromWorkingState, - isPromptCarryForwardRequest, } from "@/lib/ai/prompt-memory"; import { uploadFile } from "@/lib/storage/service"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; @@ -123,7 +136,6 @@ import { markVideoStoryboardDraftForRevision, cancelVideoStoryboardTaskByUser, } from "@/lib/video-agent/task-store"; -import { buildAgentThreadStateFromStoryboardTask } from "@/lib/video-agent/agent-thread-state"; import type { StoryboardModelCandidate } from "@/lib/video-agent/types"; import type { ChatMessage } from "@/lib/types"; import type { AppUsage } from "@/lib/usage"; @@ -1792,102 +1804,6 @@ function extractConfirmedPromptFromWorkingState( return optimizedPrompt; } -type PendingGridConfirmation = { - gridType: "grid_2x2" | "grid_3x2" | "grid_3x3"; - shotCount: 4 | 6 | 9; - label: string; - originalPrompt: string; -}; - -function detectRequestedStoryboardGrid(text: string): PendingGridConfirmation | null { - const normalized = String(text || "").toLowerCase().replace(/\s+/g, ""); - if (!normalized) return null; - - if (/4宫格|四宫格|2x2|2×2/.test(normalized)) { - return { - gridType: "grid_2x2", - shotCount: 4, - label: "4宫格", - originalPrompt: String(text || "").trim(), - }; - } - - if (/6宫格|六宫格|3x2|3×2|2x3|2×3/.test(normalized)) { - return { - gridType: "grid_3x2", - shotCount: 6, - label: "6宫格", - originalPrompt: String(text || "").trim(), - }; - } - - if (/9宫格|九宫格|3x3|3×3/.test(normalized)) { - return { - gridType: "grid_3x3", - shotCount: 9, - label: "9宫格", - originalPrompt: String(text || "").trim(), - }; - } - - return null; -} - -function isGridConfirmationReply(text: string): boolean { - const normalized = String(text || "").toLowerCase().replace(/\s+/g, ""); - if (!normalized) return false; - return /确认|就按这个|开始生成|可以开始了|没问题|对的|就这样/.test(normalized); -} - -function buildGridConfirmationMessage(grid: PendingGridConfirmation): string { - return [ - `我看到你希望按 ${grid.label} 来生成分镜图。`, - "为了确保后续自动切片稳定，我会按标准网格、统一格线、固定顺序来出图。", - `请确认是否按 ${grid.label} 标准版式开始生成。`, - "", - "如果确认，请回复：", - `- 确认按${grid.label}生成`, - "", - "如果要修改，也可以直接回复：", - "- 改成 9 宫格", - "- 改成 6 宫格", - "- 改成 4 宫格", - ].join("\n"); -} - -function extractPendingGridConfirmationFromWorkingState( - workingState?: { - lastTurnSummary?: unknown; - } | null, -): PendingGridConfirmation | null { - const summary = workingState?.lastTurnSummary; - if (!summary || typeof summary !== "object") return null; - const summaryRecord = summary as Record; - const pending = summaryRecord.pendingGridConfirmation; - if (!pending || typeof pending !== "object") return null; - const record = pending as Record; - const gridType = String(record.gridType || "").trim(); - const shotCount = Number(record.shotCount || 0); - const label = String(record.label || "").trim(); - const originalPrompt = String(record.originalPrompt || "").trim(); - - if ( - !originalPrompt || - !label || - !["grid_2x2", "grid_3x2", "grid_3x3"].includes(gridType) || - ![4, 6, 9].includes(shotCount) - ) { - return null; - } - - return { - gridType: gridType as PendingGridConfirmation["gridType"], - shotCount: shotCount as PendingGridConfirmation["shotCount"], - label, - originalPrompt, - }; -} - function hasStructuredAgentSections(message?: string): boolean { if (!message) return false; return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( @@ -2655,6 +2571,15 @@ async function saveGeneratedImages( if (savedOriginalUrls.length > 0) { try { + const existingWorkingState = await getConversationWorkingStateByChatId({ + chatId, + }).catch(() => null); + const existingSummary = + existingWorkingState?.lastTurnSummary && + typeof existingWorkingState.lastTurnSummary === "object" + ? (existingWorkingState.lastTurnSummary as Record) + : {}; + const existingImageWorkflowMemory = extractImageWorkflowMemory(existingWorkingState); await upsertConversationWorkingState({ userId, chatId, @@ -2665,6 +2590,17 @@ async function saveGeneratedImages( lastResolvedTargetAssetIds: savedAssetIds, lastResolvedTargetImageUrls: savedOriginalUrls, lastMessageId: messageId, + lastTurnSummary: { + ...existingSummary, + imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ + existing: existingImageWorkflowMemory, + lastIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen", + lastUserPrompt: prompt, + lastRequestedImageCount: savedOriginalUrls.length, + lastSourceImageUrls: options?.sourceImageUrls || null, + lastResultImageUrls: savedOriginalUrls, + }), + }, }); } catch (memoryError) { console.warn("[Gallery] Failed to update working state with generated images:", memoryError); @@ -3064,6 +3000,7 @@ export async function POST(request: Request) { | Awaited> | null = null; let storyboardPreferenceMemory = null; + let imageWorkflowMemory = null; let reusablePromptMemory = null; let smartAutoHistoryMessagesFromDb: DBMessage[] = []; let preferredAgentModelId: string | undefined; @@ -3192,29 +3129,42 @@ export async function POST(request: Request) { return null; }); storyboardPreferenceMemory = extractStoryboardPreferenceMemory(requestWorkingState); + imageWorkflowMemory = extractImageWorkflowMemory(requestWorkingState); reusablePromptMemory = extractPromptMemoryFromWorkingState(requestWorkingState); pendingGridConfirmation = extractPendingGridConfirmationFromWorkingState(requestWorkingState); + const hasImageInputForConversationAction = message.parts.some((part: any) => + part?.type === "file" || part?.type === "image" || part?.type === "image_url" + ); + const preloadedConfirmedPrompt = + extractConfirmedPromptFromWorkingState(requestWorkingState) || + reusablePromptMemory?.optimizedPrompt; + const resolvedConversationAction = resolveConversationAction({ + text: requestTextContent, + pendingGridConfirmation, + hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id), + imageWorkflowMemory, + confirmedPrompt: preloadedConfirmedPrompt, + hasImageInputForPromptCarryForward: hasImageInputForConversationAction, + }); - if ( - pendingGridConfirmation && - (isGridConfirmationReply(requestTextContent) || - detectRequestedStoryboardGrid(requestTextContent) !== null) && - !shouldRunAgentConversation - ) { + if (resolvedConversationAction.forceAgentConversation && !shouldRunAgentConversation) { shouldRunAgentConversation = true; - shouldKeepSelectedModel = true; - console.log( - `[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}` - ); - } - - if (activeStoryboardTaskForChat?.task?.id && !shouldRunAgentConversation) { - shouldRunAgentConversation = true; - shouldKeepSelectedModel = true; - console.log( - `[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}` - ); + shouldKeepSelectedModel = resolvedConversationAction.shouldKeepSelectedModel; + if (resolvedConversationAction.kind === "grid_confirmation_followup" && pendingGridConfirmation) { + console.log( + `[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}` + ); + } else if ( + resolvedConversationAction.kind === "active_storyboard_followup" && + activeStoryboardTaskForChat?.task?.id + ) { + console.log( + `[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}` + ); + } else if (resolvedConversationAction.kind === "image_edit_followup") { + console.log("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow"); + } } // ========== Smart Auto Mode Logic ========== @@ -3226,9 +3176,7 @@ export async function POST(request: Request) { let confirmedPromptForGeneration: string | undefined; let forcePromptDrivenImageGeneration = false; const smartAutoInputText = - pendingGridConfirmation && isGridConfirmationReply(requestTextContent) - ? pendingGridConfirmation.originalPrompt - : requestTextContent; + resolvedConversationAction.effectiveInputText || requestTextContent; try { // 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb） @@ -3241,22 +3189,11 @@ export async function POST(request: Request) { chatId: id, }); } - confirmedPromptForGeneration = - extractConfirmedPromptFromWorkingState(requestWorkingState) || - reusablePromptMemory?.optimizedPrompt; - const hasImageInputForPromptCarryForward = - message.parts.some((part: any) => - part?.type === "file" || part?.type === "image" || part?.type === "image_url" - ); + confirmedPromptForGeneration = preloadedConfirmedPrompt; const isPromptCarryForwardExecution = - Boolean(confirmedPromptForGeneration) && - isPromptCarryForwardRequest({ - text: requestTextContent, - hasImage: hasImageInputForPromptCarryForward, - }); + resolvedConversationAction.kind === "prompt_carry_forward"; forcePromptDrivenImageGeneration = - (isConfirmedPromptGenerationRequest(requestTextContent) || - isPromptCarryForwardExecution) && + resolvedConversationAction.usePromptCarryForward && Boolean(confirmedPromptForGeneration); if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ @@ -3350,6 +3287,37 @@ export async function POST(request: Request) { CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS ); } + const baseThreadState = buildConversationThreadState({ + activeStoryboardTask: activeStoryboardTaskForChat, + storyboardPreferenceMemory, + promptMemory: reusablePromptMemory, + imageWorkflowMemory, + }); + const composedConversationMemory = composeConversationMemory({ + autoContextText: autoContextText || undefined, + resolvedMediaInput: requestResolvedMediaInput, + targetResolution: requestTargetResolution, + promptMemory: reusablePromptMemory, + imageWorkflowMemory, + storyboardPreferenceMemory, + threadState: baseThreadState, + }); + const composedThreadState = { + workflowState: baseThreadState?.workflowState || null, + memoryHints: Array.from( + new Set([ + ...(baseThreadState?.memoryHints || []), + ...composedConversationMemory.memoryHints, + ]) + ), + contextSummary: + composedConversationMemory.contextSummary || + baseThreadState?.contextSummary, + metadata: { + ...(baseThreadState?.metadata || {}), + ...composedConversationMemory.metadata, + }, + }; const autoGraphResult = await withOperationTimeout( runAutoAgentGraph({ text: smartAutoInputText, @@ -3373,14 +3341,10 @@ export async function POST(request: Request) { requestResolvedMediaInput.historyUserImageUrls.length, currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length, historyMessages, - context: autoContextText || undefined, + context: + composedConversationMemory.contextSummary || autoContextText || undefined, preferredModelId: preferredAgentModelId, - threadState: buildAgentThreadStateFromStoryboardTask( - activeStoryboardTaskForChat, - { - storyboardPreferenceMemory, - } - ), + threadState: composedThreadState, }), "runAutoAgentGraph", autoGraphTimeoutMs, @@ -4718,6 +4682,14 @@ export async function POST(request: Request) { sourceText: requestTextContent.slice(0, 500), reusable: true, }), + imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ + existing: imageWorkflowMemory, + lastIntent: "vision", + lastOptimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000), + lastUserPrompt: requestTextContent.slice(0, 500), + lastRequestedImageCount: 1, + lastSourceImageUrls: promptOnlySourceImageUrls, + }), imageSummary: promptOnlyImageSummary.slice(0, 1200), sourceImageUrls: promptOnlySourceImageUrls, }, @@ -6214,6 +6186,12 @@ export async function POST(request: Request) { try { const resultAssetIds = resultAssetId ? [resultAssetId] : null; const resultImageUrls = resultOriginalUrl ? [resultOriginalUrl] : null; + const existingImageWorkflowMemory = extractImageWorkflowMemory(requestWorkingState); + const existingSummary = + requestWorkingState?.lastTurnSummary && + typeof requestWorkingState.lastTurnSummary === "object" + ? (requestWorkingState.lastTurnSummary as Record) + : {}; await upsertConversationWorkingState({ userId: session.user.id, chatId: id, @@ -6227,6 +6205,17 @@ export async function POST(request: Request) { lastResolvedTargetAssetIds: resultAssetIds, lastResolvedTargetImageUrls: resultImageUrls, lastMessageId: assistantMessageId, + lastTurnSummary: { + ...existingSummary, + imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ + existing: existingImageWorkflowMemory, + lastIntent: "image_edit", + lastUserPrompt: requestTextContent.slice(0, 500), + lastRequestedImageCount: resultImageUrls?.length || 1, + lastSourceImageUrls: directAgentToolTargetImageUrls, + lastResultImageUrls: resultImageUrls, + }), + }, }); } catch (memoryError) { console.warn( diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 95b63e924..df3725134 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -3523,6 +3523,11 @@ export function CanvasChatPanel({ } const latestNodes = useCanvasStore.getState().nodes; + const suppressedMediaKeys = new Set( + (useCanvasStore.getState().getCanvasState().suppressedMediaKeys || []) + .map((item) => String(item || "").trim()) + .filter(Boolean) + ); const existingImageUrlKeys = new Set( latestNodes .filter((node) => node.type === "image" && (node.data as any)?.src) @@ -3572,6 +3577,12 @@ export function CanvasChatPanel({ continue; } const messageImageKey = messageId ? `${messageId}::${sourceImageKey}` : ""; + if ( + suppressedMediaKeys.has(`image:${messageImageKey}`) || + suppressedMediaKeys.has(`image-url:${url}`) + ) { + continue; + } if (messageImageKey && existingMessageImageKeys.has(messageImageKey)) { continue; } @@ -3642,6 +3653,9 @@ export function CanvasChatPanel({ if (!key || existingVideoUrlKeys.has(key)) { continue; } + if (suppressedMediaKeys.has(`video:${url}`)) { + continue; + } existingVideoUrlKeys.add(key); const currentNodes = useCanvasStore.getState().nodes; const existingNodes = currentNodes.length + addedImageCount + addedVideoCount; @@ -4344,6 +4358,11 @@ export function CanvasChatPanel({ let cancelled = false; const processNewAssistantMessages = async () => { + const suppressedMediaKeys = new Set( + (useCanvasStore.getState().getCanvasState().suppressedMediaKeys || []) + .map((item) => String(item || "").trim()) + .filter(Boolean) + ); const existingImageUrlKeys = new Set( nodes .filter((n) => n.type === "image" && (n.data as any)?.src) @@ -4465,6 +4484,12 @@ export function CanvasChatPanel({ return acc; } const messageImageKey = `${message.id}::${sourceImageKey}`; + if ( + suppressedMediaKeys.has(`image:${messageImageKey}`) || + suppressedMediaKeys.has(`image-url:${url}`) + ) { + return acc; + } if (existingMessageImageKeys.has(messageImageKey)) { return acc; } @@ -4556,6 +4581,7 @@ export function CanvasChatPanel({ const newVideoUrls = videoUrls.filter((url) => { const key = normalizeMediaUrl(url); if (!key || existingVideoUrlKeys.has(key)) return false; + if (suppressedMediaKeys.has(`video:${url}`)) return false; existingVideoUrlKeys.add(key); return true; }); diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index 0ceddb024..91f29719c 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -861,7 +861,7 @@ function sanitizeRecommendedImageParams( if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) { const min = 1; - const max = intent === "image_edit" ? 1 : 4; + const max = intent === "image_edit" ? 1 : 12; safe.imageCount = Math.min(Math.max(Math.round(params.imageCount), min), max); } if (intent === "image_edit") { @@ -894,12 +894,18 @@ function heuristicImageParams( if (analysis.intent === "image_edit") { params.imageCount = 1; - } else if (/多张|四张|4张|三张|3张/.test(normalized)) { - params.imageCount = /四张|4张/.test(normalized) ? 4 : 3; - } else if (/两张|2张|双图/.test(normalized)) { - params.imageCount = 2; } else { - params.imageCount = 1; + const explicitCountMatch = normalized.match(/([1-9]\d?)张/); + const explicitCount = explicitCountMatch ? Number(explicitCountMatch[1]) : 0; + if (explicitCount >= 1) { + params.imageCount = Math.min(explicitCount, 12); + } else if (/多张|四张|4张|三张|3张/.test(normalized)) { + params.imageCount = /四张|4张/.test(normalized) ? 4 : 3; + } else if (/两张|2张|双图/.test(normalized)) { + params.imageCount = 2; + } else { + params.imageCount = 1; + } } if (/横图|横版|宽屏|封面|banner|海报/.test(normalized)) { diff --git a/lib/ai/conversation-action-resolver.ts b/lib/ai/conversation-action-resolver.ts new file mode 100644 index 000000000..a5955fea0 --- /dev/null +++ b/lib/ai/conversation-action-resolver.ts @@ -0,0 +1,201 @@ +import "server-only"; + +import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory"; +import { mentionsImageEditFollowup } from "@/lib/ai/image-edit-subagent"; +import { isPromptCarryForwardRequest } from "@/lib/ai/prompt-memory"; + +export interface PendingGridConfirmation { + gridType: "grid_2x2" | "grid_3x2" | "grid_3x3"; + shotCount: 4 | 6 | 9; + label: string; + originalPrompt: string; +} + +export type ConversationActionKind = + | "default" + | "grid_confirmation_followup" + | "active_storyboard_followup" + | "image_edit_followup" + | "prompt_carry_forward"; + +export interface ConversationActionResolution { + kind: ConversationActionKind; + forceAgentConversation: boolean; + shouldKeepSelectedModel: boolean; + effectiveInputText?: string; + usePromptCarryForward: boolean; +} + +export function isConfirmedPromptGenerationRequest(text: string): boolean { + const normalized = String(text || "").toLowerCase().replace(/\s+/g, ""); + if (!normalized) return false; + + const refersPrompt = + /根据这个提示词|按这个提示词|就按这个提示词|按刚才的提示词|按上面的提示词|用这个提示词|用刚才的提示词|按该提示词/.test( + normalized, + ); + const wantsImageGeneration = + /生图|生成图片|出图|画出来|生成一张|来一张|开始生成|按此生成|按这个生成/.test(normalized); + + return refersPrompt && wantsImageGeneration; +} + +export function detectRequestedStoryboardGrid(text: string): PendingGridConfirmation | null { + const normalized = String(text || "").toLowerCase().replace(/\s+/g, ""); + if (!normalized) return null; + + if (/4宫格|四宫格|2x2|2×2/.test(normalized)) { + return { + gridType: "grid_2x2", + shotCount: 4, + label: "4宫格", + originalPrompt: String(text || "").trim(), + }; + } + + if (/6宫格|六宫格|3x2|3×2|2x3|2×3/.test(normalized)) { + return { + gridType: "grid_3x2", + shotCount: 6, + label: "6宫格", + originalPrompt: String(text || "").trim(), + }; + } + + if (/9宫格|九宫格|3x3|3×3/.test(normalized)) { + return { + gridType: "grid_3x3", + shotCount: 9, + label: "9宫格", + originalPrompt: String(text || "").trim(), + }; + } + + return null; +} + +export function isGridConfirmationReply(text: string): boolean { + const normalized = String(text || "").toLowerCase().replace(/\s+/g, ""); + if (!normalized) return false; + return /确认|就按这个|开始生成|可以开始了|没问题|对的|就这样/.test(normalized); +} + +export function extractPendingGridConfirmationFromWorkingState( + workingState?: { + lastTurnSummary?: unknown; + } | null, +): PendingGridConfirmation | null { + const summary = workingState?.lastTurnSummary; + if (!summary || typeof summary !== "object") return null; + const summaryRecord = summary as Record; + const pending = summaryRecord.pendingGridConfirmation; + if (!pending || typeof pending !== "object") return null; + const record = pending as Record; + const gridType = String(record.gridType || "").trim(); + const shotCount = Number(record.shotCount || 0); + const label = String(record.label || "").trim(); + const originalPrompt = String(record.originalPrompt || "").trim(); + + if ( + !originalPrompt || + !label || + !["grid_2x2", "grid_3x2", "grid_3x3"].includes(gridType) || + ![4, 6, 9].includes(shotCount) + ) { + return null; + } + + return { + gridType: gridType as PendingGridConfirmation["gridType"], + shotCount: shotCount as PendingGridConfirmation["shotCount"], + label, + originalPrompt, + }; +} + +export function buildGridConfirmationMessage(grid: PendingGridConfirmation): string { + return [ + `我看到你希望按 ${grid.label} 来生成分镜图。`, + "为了确保后续自动切片稳定，我会按标准网格、统一格线、固定顺序来出图。", + `请确认是否按 ${grid.label} 标准版式开始生成。`, + "", + "如果确认，请回复：", + `- 确认按${grid.label}生成`, + "", + "如果要修改，也可以直接回复：", + "- 改成 9 宫格", + "- 改成 6 宫格", + "- 改成 4 宫格", + ].join("\n"); +} + +export function resolveConversationAction(params: { + text: string; + pendingGridConfirmation?: PendingGridConfirmation | null; + hasActiveStoryboardTask: boolean; + imageWorkflowMemory?: ImageWorkflowMemory | null; + confirmedPrompt?: string; + hasImageInputForPromptCarryForward: boolean; +}) { + const requestedGrid = detectRequestedStoryboardGrid(params.text); + const shouldContinuePendingGrid = + Boolean(params.pendingGridConfirmation) && + (isGridConfirmationReply(params.text) || requestedGrid !== null); + + if (shouldContinuePendingGrid) { + return { + kind: "grid_confirmation_followup", + forceAgentConversation: true, + shouldKeepSelectedModel: true, + effectiveInputText: params.pendingGridConfirmation?.originalPrompt, + usePromptCarryForward: false, + } satisfies ConversationActionResolution; + } + + if (params.hasActiveStoryboardTask) { + return { + kind: "active_storyboard_followup", + forceAgentConversation: true, + shouldKeepSelectedModel: true, + usePromptCarryForward: false, + } satisfies ConversationActionResolution; + } + + const hasImageEditFollowup = + params.imageWorkflowMemory?.lastIntent === "image_edit" && + (mentionsImageEditFollowup(params.text) || + Boolean(params.imageWorkflowMemory?.lastResultImageUrls?.length)); + + if (hasImageEditFollowup) { + return { + kind: "image_edit_followup", + forceAgentConversation: true, + shouldKeepSelectedModel: true, + usePromptCarryForward: false, + } satisfies ConversationActionResolution; + } + + const usePromptCarryForward = + Boolean(params.confirmedPrompt) && + (isConfirmedPromptGenerationRequest(params.text) || + isPromptCarryForwardRequest({ + text: params.text, + hasImage: params.hasImageInputForPromptCarryForward, + })); + + if (usePromptCarryForward) { + return { + kind: "prompt_carry_forward", + forceAgentConversation: false, + shouldKeepSelectedModel: false, + usePromptCarryForward: true, + } satisfies ConversationActionResolution; + } + + return { + kind: "default", + forceAgentConversation: false, + shouldKeepSelectedModel: false, + usePromptCarryForward: false, + } satisfies ConversationActionResolution; +} diff --git a/lib/ai/conversation-memory-composer.ts b/lib/ai/conversation-memory-composer.ts new file mode 100644 index 000000000..406c8f84c --- /dev/null +++ b/lib/ai/conversation-memory-composer.ts @@ -0,0 +1,204 @@ +import "server-only"; + +import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; +import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory"; +import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory"; +import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory"; +import type { + ResolvedChatMediaInput, + ResolvedTargetAssets, +} from "@/lib/ai/context-memory/types"; + +export interface ComposedConversationMemory { + contextSummary: string; + memoryHints: string[]; + metadata: Record; +} + +function normalizeText(value: unknown, max = 400): string | undefined { + const normalized = String(value || "").trim(); + if (!normalized) return undefined; + return normalized.slice(0, max); +} + +function uniqueStrings(values: Array) { + return Array.from( + new Set( + values + .map((item) => normalizeText(item, 240)) + .filter((item): item is string => Boolean(item)) + ) + ); +} + +function summarizeTargetResolution(target?: ResolvedTargetAssets | null) { + if (!target) return { lines: [] as string[], metadata: {} as Record }; + const lines: string[] = []; + lines.push( + `当前目标解析：mode=${target.resolutionMode}; confidence=${target.confidence}; fromHistory=${ + target.fromHistory ? "yes" : "no" + }` + ); + if (target.targetImageUrls.length > 0) { + lines.push(`当前命中的目标图片数：${target.targetImageUrls.length}`); + } + if (target.reason) { + lines.push(`目标解析原因：${target.reason}`); + } + return { + lines, + metadata: { + targetResolutionMode: target.resolutionMode, + targetResolutionConfidence: target.confidence, + targetImageCount: target.targetImageUrls.length, + targetReason: target.reason, + }, + }; +} + +function summarizeResolvedMedia(media?: ResolvedChatMediaInput | null) { + if (!media) return { lines: [] as string[], metadata: {} as Record }; + return { + lines: [ + `当前媒体输入：source=${media.source}; hasImage=${media.hasImage ? "yes" : "no"}`, + media.currentMessageImageUrls.length > 0 + ? `本轮上传图片数：${media.currentMessageImageUrls.length}` + : "", + media.selectedImageUrls.length > 0 + ? `当前选中图片数：${media.selectedImageUrls.length}` + : "", + media.historyAssistantImageUrls.length + media.historyUserImageUrls.length > 0 + ? `历史可参考图片数：${ + media.historyAssistantImageUrls.length + media.historyUserImageUrls.length + }` + : "", + ].filter(Boolean), + metadata: { + mediaSource: media.source, + hasImage: media.hasImage, + currentImageCount: media.currentMessageImageUrls.length, + selectedImageCount: media.selectedImageUrls.length, + historyImageCount: + media.historyAssistantImageUrls.length + media.historyUserImageUrls.length, + }, + }; +} + +function summarizePromptMemory(promptMemory?: PromptMemoryEntry | null) { + if (!promptMemory?.optimizedPrompt) { + return { lines: [] as string[], hints: [] as string[], metadata: {} as Record }; + } + const preview = promptMemory.optimizedPrompt.replace(/\s+/g, " ").trim().slice(0, 220); + return { + lines: [`上一轮可复用提示词：${preview}`], + hints: [ + "如果用户明确要求沿用上一轮提示词继续做图，优先把这段提示词作为本轮生成基础。", + ], + metadata: { + promptMemory, + }, + }; +} + +function summarizeImageWorkflow(memory?: ImageWorkflowMemory | null) { + if (!memory) { + return { lines: [] as string[], hints: [] as string[], metadata: {} as Record }; + } + const lines = [ + memory.lastOptimizedPrompt + ? `上一轮优化提示词摘要：${memory.lastOptimizedPrompt.replace(/\s+/g, " ").slice(0, 180)}` + : "", + memory.lastRequestedImageCount + ? `上一轮目标出图数：${memory.lastRequestedImageCount} 张` + : "", + memory.lastAspectRatio ? `上一轮画幅：${memory.lastAspectRatio}` : "", + memory.lastImageSize ? `上一轮清晰度：${memory.lastImageSize}` : "", + memory.lastSourceImageUrls?.length + ? `上一轮参考/目标图数量：${memory.lastSourceImageUrls.length}` + : "", + memory.lastResultImageUrls?.length + ? `上一轮生成结果数量：${memory.lastResultImageUrls.length}` + : "", + ].filter(Boolean); + const hints = memory.lastOptimizedPrompt + ? ["如果用户要求沿用上一轮提示词继续做图，应优先复用这段提示词。"] + : []; + return { + lines, + hints, + metadata: { + imageWorkflowMemory: memory, + }, + }; +} + +function summarizeStoryboardPreference(memory?: StoryboardPreferenceMemory | null) { + if (!memory) { + return { lines: [] as string[], hints: [] as string[], metadata: {} as Record }; + } + const lines = [ + memory.preferredGridType ? `用户偏好宫格：${memory.preferredGridType}` : "", + memory.preferredShotCount ? `用户偏好镜头数：${memory.preferredShotCount}` : "", + memory.prefersConfirmation === true ? "用户偏好先确认再执行" : "", + memory.lastUsedGridType && memory.lastUsedShotCount + ? `最近一次分镜任务：${memory.lastUsedGridType} / ${memory.lastUsedShotCount} 段` + : "", + ].filter(Boolean); + const hints = memory.prefersConfirmation + ? ["如果当前任务涉及分镜或高成本执行，优先先确认再推进。"] + : []; + return { + lines, + hints, + metadata: { + storyboardPreferenceMemory: memory, + }, + }; +} + +export function composeConversationMemory(params: { + autoContextText?: string; + resolvedMediaInput?: ResolvedChatMediaInput | null; + targetResolution?: ResolvedTargetAssets | null; + promptMemory?: PromptMemoryEntry | null; + imageWorkflowMemory?: ImageWorkflowMemory | null; + storyboardPreferenceMemory?: StoryboardPreferenceMemory | null; + threadState?: AgentThreadState | null; +}) { + const targetSummary = summarizeTargetResolution(params.targetResolution); + const mediaSummary = summarizeResolvedMedia(params.resolvedMediaInput); + const promptSummary = summarizePromptMemory(params.promptMemory); + const imageWorkflowSummary = summarizeImageWorkflow(params.imageWorkflowMemory); + const storyboardSummary = summarizeStoryboardPreference(params.storyboardPreferenceMemory); + + const contextSummary = uniqueStrings([ + params.autoContextText, + ...mediaSummary.lines, + ...targetSummary.lines, + ...promptSummary.lines, + ...imageWorkflowSummary.lines, + ...storyboardSummary.lines, + ]).join("\n"); + + const memoryHints = uniqueStrings([ + ...(params.threadState?.memoryHints || []), + ...promptSummary.hints, + ...imageWorkflowSummary.hints, + ...storyboardSummary.hints, + ]); + + const metadata: Record = { + ...(params.threadState?.metadata || {}), + ...mediaSummary.metadata, + ...targetSummary.metadata, + ...promptSummary.metadata, + ...imageWorkflowSummary.metadata, + ...storyboardSummary.metadata, + }; + + return { + contextSummary, + memoryHints, + metadata, + } satisfies ComposedConversationMemory; +} diff --git a/lib/ai/conversation-thread-state-builder.ts b/lib/ai/conversation-thread-state-builder.ts new file mode 100644 index 000000000..7313e58eb --- /dev/null +++ b/lib/ai/conversation-thread-state-builder.ts @@ -0,0 +1,133 @@ +import "server-only"; + +import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; +import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory"; +import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory"; +import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory"; +import { buildAgentThreadStateFromStoryboardTask } from "@/lib/video-agent/agent-thread-state"; + +type StoryboardTaskDetail = Parameters[0]; + +function normalizeText(value: unknown, max = 240): string | undefined { + const normalized = String(value || "").trim(); + if (!normalized) return undefined; + return normalized.slice(0, max); +} + +export function buildConversationThreadState(params: { + activeStoryboardTask?: StoryboardTaskDetail | null; + storyboardPreferenceMemory?: StoryboardPreferenceMemory | null; + promptMemory?: PromptMemoryEntry | null; + imageWorkflowMemory?: ImageWorkflowMemory | null; +}): AgentThreadState | null { + const storyboardState = buildAgentThreadStateFromStoryboardTask( + params.activeStoryboardTask, + { + storyboardPreferenceMemory: params.storyboardPreferenceMemory, + promptMemory: params.promptMemory, + }, + ); + + if (params.activeStoryboardTask?.task?.id) { + return { + workflowState: storyboardState?.workflowState || null, + memoryHints: [ + ...(storyboardState?.memoryHints || []), + params.imageWorkflowMemory?.lastOptimizedPrompt + ? "当前会话里也保留了最近一次图片工作流记忆，可辅助理解用户是在继续哪一轮生图。" + : "", + ].filter(Boolean), + contextSummary: + storyboardState?.contextSummary || + normalizeText(params.imageWorkflowMemory?.lastUserPrompt, 320), + metadata: { + ...(storyboardState?.metadata || {}), + imageWorkflowMemory: params.imageWorkflowMemory || undefined, + }, + }; + } + + const hasImageFollowup = + Boolean(params.promptMemory?.optimizedPrompt) || + Boolean(params.imageWorkflowMemory?.lastOptimizedPrompt) || + Boolean(params.imageWorkflowMemory?.lastResultImageUrls?.length) || + Boolean(params.imageWorkflowMemory?.lastSourceImageUrls?.length); + + if (!hasImageFollowup && !params.storyboardPreferenceMemory) { + return null; + } + + const preferredCount = params.imageWorkflowMemory?.lastRequestedImageCount; + const preferredRatio = params.imageWorkflowMemory?.lastAspectRatio; + const preferredSize = params.imageWorkflowMemory?.lastImageSize; + const sourceCount = params.imageWorkflowMemory?.lastSourceImageUrls?.length || 0; + const resultCount = params.imageWorkflowMemory?.lastResultImageUrls?.length || 0; + const imageFollowupType = + params.imageWorkflowMemory?.lastIntent === "image_edit" + ? "image_edit_followup" + : "image_generation_followup"; + const imageFollowupTitle = + imageFollowupType === "image_edit_followup" + ? "连续修图工作流" + : "连续生图工作流"; + + return { + workflowState: hasImageFollowup + ? { + workflowId: "image-generation-followup", + type: imageFollowupType, + status: "draft", + title: imageFollowupTitle, + summary: [ + params.promptMemory?.optimizedPrompt || params.imageWorkflowMemory?.lastOptimizedPrompt + ? "当前会话中存在上一轮可复用提示词。" + : "", + preferredCount ? `上一轮目标出图数：${preferredCount} 张。` : "", + preferredRatio ? `上一轮画幅：${preferredRatio}。` : "", + preferredSize ? `上一轮清晰度：${preferredSize}。` : "", + sourceCount > 0 ? `上一轮参考图数：${sourceCount}。` : "", + resultCount > 0 ? `上一轮结果图数：${resultCount}。` : "", + ] + .filter(Boolean) + .join(" "), + confirmPhrases: ["按上一个提示词生成", "继续生成", "就按这个来"], + revisionHints: ["换一版", "保持提示词不变再来一组", "沿用上一个提示词"], + pendingItems: [], + metadata: { + promptMemory: params.promptMemory || undefined, + imageWorkflowMemory: params.imageWorkflowMemory || undefined, + storyboardPreferenceMemory: params.storyboardPreferenceMemory || undefined, + }, + } + : null, + memoryHints: [ + params.promptMemory?.optimizedPrompt + ? "当前会话存在上一轮可复用提示词；如果用户明确要求沿用它继续生成，应优先复用。" + : "", + params.imageWorkflowMemory?.lastResultImageUrls?.length + ? "当前会话保留了上一轮生成结果图，可用于理解“继续做这张/沿用上一版”的指代。" + : "", + params.imageWorkflowMemory?.lastSourceImageUrls?.length + ? "当前会话保留了上一轮参考图/目标图，可用于连续图生图理解。" + : "", + params.storyboardPreferenceMemory + ? "当前会话也存在 storyboard 偏好记忆，后续如果转入分镜任务可继续复用。" + : "", + ].filter(Boolean), + contextSummary: [ + params.promptMemory?.optimizedPrompt + ? `上一轮提示词摘要：${normalizeText(params.promptMemory.optimizedPrompt, 180)}` + : "", + params.imageWorkflowMemory?.lastUserPrompt + ? `上一轮用户原始需求：${normalizeText(params.imageWorkflowMemory.lastUserPrompt, 180)}` + : "", + ] + .filter(Boolean) + .join("\n"), + metadata: { + promptMemory: params.promptMemory || undefined, + imageWorkflowMemory: params.imageWorkflowMemory || undefined, + storyboardPreferenceMemory: params.storyboardPreferenceMemory || undefined, + }, + }; +} diff --git a/lib/ai/image-edit-subagent.ts b/lib/ai/image-edit-subagent.ts new file mode 100644 index 000000000..5ddf80d3d --- /dev/null +++ b/lib/ai/image-edit-subagent.ts @@ -0,0 +1,142 @@ +import "server-only"; + +import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; +import type { AgentExecutionPlan, AgentPlannedToolCall } from "@/lib/ai/auto-agent-graph"; +import type { AgentSpecialistRoute } from "@/lib/ai/specialists"; +import type { AgentProjectToolDefinition } from "@/lib/ai/tools-registry"; +import { getImageWorkflowPlanningHints } from "@/lib/ai/image-workflow-memory"; + +function normalizeCompactText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +export function mentionsImageEditFollowup(text: string): boolean { + return /继续改|继续调整|再改一下|再调一下|上一版|上一张结果|这张继续改|保留.*不变|只改|继续优化这张图|继续修这张图/.test( + normalizeCompactText(text) + ); +} + +function hasImageEditWorkflow(threadState?: AgentThreadState | null) { + return threadState?.workflowState?.type === "image_edit_followup"; +} + +export function buildImageEditSpecialistRoute(params: { + text: string; + threadState?: AgentThreadState | null; + hasImage: boolean; +}): AgentSpecialistRoute | null { + const normalized = normalizeCompactText(params.text); + const workflowHints = getImageWorkflowPlanningHints(params.threadState); + const hasWorkflow = hasImageEditWorkflow(params.threadState); + const mentionsFollowupEdit = mentionsImageEditFollowup(params.text); + + if (!hasWorkflow && !mentionsFollowupEdit && workflowHints.length === 0) { + return null; + } + + const planningHints = [ + hasWorkflow + ? "当前回合处于连续修图 workflow，应优先理解为对上一轮结果图或目标图继续编辑。" + : "", + mentionsFollowupEdit + ? "用户当前表达的是继续修图或局部重调，不要误解成重新生图。" + : "", + params.hasImage + ? "如果本轮带了图片，应优先把它理解为当前修图输入或替换后的参考图。" + : "", + ...workflowHints, + ].filter(Boolean); + + return { + domain: "image_edit_memory", + specialistId: "image-edit-subagent", + reason: hasWorkflow ? "当前回合命中连续修图工作流" : "当前回合命中连续修图语义", + planningHints, + toolAllowlist: [ + "respondDirect", + "editImage", + "removeBackground", + "enhanceImage", + "extendImage", + "resizeImage", + "boxCutout", + "useSelectedCanvasImage", + "useLatestResultImage", + "useOriginalUploadImage", + "publishToCanvas", + ], + }; +} + +function hasTool( + availableTools: AgentProjectToolDefinition[], + name: AgentPlannedToolCall["name"], +) { + return availableTools.some((tool) => tool.name === name); +} + +function upsertTool( + toolCalls: AgentPlannedToolCall[], + name: AgentPlannedToolCall["name"], + reason: string, + options?: { prepend?: boolean }, +) { + const existing = toolCalls.find((tool) => tool.name === name); + if (existing) return toolCalls; + const nextTool = { name, reason } satisfies AgentPlannedToolCall; + return options?.prepend ? [nextTool, ...toolCalls] : [...toolCalls, nextTool]; +} + +export function refinePlanWithImageEditSubagent(params: { + text: string; + plan: AgentExecutionPlan; + availableTools: AgentProjectToolDefinition[]; + threadState?: AgentThreadState | null; + hasImage: boolean; +}): AgentExecutionPlan { + const hasWorkflow = hasImageEditWorkflow(params.threadState); + const hasImageWorkflowHints = getImageWorkflowPlanningHints(params.threadState).length > 0; + const mentionsFollowupEdit = mentionsImageEditFollowup(params.text); + + if (!hasWorkflow && !hasImageWorkflowHints && !mentionsFollowupEdit) { + return params.plan; + } + + let toolCalls = [...params.plan.toolCalls]; + if (hasTool(params.availableTools, "useLatestResultImage")) { + toolCalls = upsertTool( + toolCalls, + "useLatestResultImage", + "image-edit 子代理优先绑定上一轮结果图作为连续修图目标", + { prepend: true } + ); + } + if (hasTool(params.availableTools, "editImage")) { + toolCalls = upsertTool( + toolCalls, + "editImage", + "image-edit 子代理要求优先走连续修图执行链路", + { prepend: true } + ); + } + if (hasTool(params.availableTools, "publishToCanvas")) { + toolCalls = upsertTool( + toolCalls, + "publishToCanvas", + "image-edit 子代理建议把修图结果同步回画布" + ); + } + + return { + ...params.plan, + nextAction: "edit_image", + intent: "image_edit", + responseMode: "tool_then_execute", + preferredCapability: + params.plan.preferredCapability === "respond" + ? "generic_edit" + : params.plan.preferredCapability, + shouldUseStructuredReply: true, + toolCalls, + }; +} diff --git a/lib/ai/image-generation-subagent.ts b/lib/ai/image-generation-subagent.ts new file mode 100644 index 000000000..6aa53f3b2 --- /dev/null +++ b/lib/ai/image-generation-subagent.ts @@ -0,0 +1,131 @@ +import "server-only"; + +import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; +import type { AgentExecutionPlan, AgentPlannedToolCall } from "@/lib/ai/auto-agent-graph"; +import type { AgentSpecialistRoute } from "@/lib/ai/specialists"; +import type { AgentProjectToolDefinition } from "@/lib/ai/tools-registry"; +import { + getImageWorkflowPlanningHints, +} from "@/lib/ai/image-workflow-memory"; +import { + getPromptMemoryPlanningHints, + isPromptCarryForwardRequest, +} from "@/lib/ai/prompt-memory"; + +function normalizeCompactText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +export function buildImageGenerationSpecialistRoute(params: { + text: string; + threadState?: AgentThreadState | null; + hasImage: boolean; +}): AgentSpecialistRoute | null { + const normalized = normalizeCompactText(params.text); + const promptHints = getPromptMemoryPlanningHints(params.threadState); + const workflowHints = getImageWorkflowPlanningHints(params.threadState); + const hasPromptMemory = promptHints.length > 0; + const hasImageWorkflowMemory = workflowHints.length > 0; + const carryForward = isPromptCarryForwardRequest({ + text: params.text, + hasImage: params.hasImage, + }); + const mentionsPromptExecution = + /提示词|prompt|关键词|咒语/.test(normalized) && + /生成|生图|出图|继续做|继续生成|执行/.test(normalized); + + if (!hasPromptMemory && !hasImageWorkflowMemory && !carryForward && !mentionsPromptExecution) { + return null; + } + + const planningHints = [ + carryForward + ? "用户当前要求沿用上一轮优化提示词继续执行，应优先理解为连续生图执行。" + : "", + hasPromptMemory + ? "当前会话里存在上一轮可复用提示词，如果用户在说“上一个提示词/刚才那个 prompt”，优先复用这段提示词。" + : "", + hasImageWorkflowMemory + ? "当前会话里存在图片工作流记忆，优先参考上一轮参考图、结果图、数量和画幅偏好。" + : "", + params.hasImage + ? "如果当前回合上传了新图片，并要求沿用上一轮提示词，应理解为“上一轮提示词 + 当前图片条件”的联合执行。" + : "", + ...promptHints, + ...workflowHints, + ].filter(Boolean); + + return { + domain: "image_generation_memory", + specialistId: "image-generation-subagent", + reason: carryForward ? "当前回合命中提示词接力生图" : "当前回合命中提示词相关的连续生图任务", + planningHints, + toolAllowlist: [ + "respondDirect", + "generateImage", + "useSelectedCanvasImage", + "useLatestResultImage", + "useOriginalUploadImage", + "publishToCanvas", + ], + }; +} + +function hasTool( + availableTools: AgentProjectToolDefinition[], + name: AgentPlannedToolCall["name"], +) { + return availableTools.some((tool) => tool.name === name); +} + +function upsertTool( + toolCalls: AgentPlannedToolCall[], + name: AgentPlannedToolCall["name"], + reason: string, + options?: { prepend?: boolean }, +) { + const existing = toolCalls.find((tool) => tool.name === name); + if (existing) return toolCalls; + const nextTool = { name, reason } satisfies AgentPlannedToolCall; + return options?.prepend ? [nextTool, ...toolCalls] : [...toolCalls, nextTool]; +} + +export function refinePlanWithImageGenerationSubagent(params: { + text: string; + plan: AgentExecutionPlan; + availableTools: AgentProjectToolDefinition[]; + threadState?: AgentThreadState | null; + hasImage: boolean; +}): AgentExecutionPlan { + const carryForward = isPromptCarryForwardRequest({ + text: params.text, + hasImage: params.hasImage, + }); + const hasPromptMemory = getPromptMemoryPlanningHints(params.threadState).length > 0; + const hasImageWorkflowMemory = getImageWorkflowPlanningHints(params.threadState).length > 0; + if (!carryForward && !hasPromptMemory && !hasImageWorkflowMemory) { + return params.plan; + } + + let toolCalls = [...params.plan.toolCalls]; + if (hasTool(params.availableTools, "generateImage")) { + toolCalls = [{ name: "generateImage", reason: "image-generation 子代理要求优先走连续生图执行链路" }]; + } + if (hasTool(params.availableTools, "publishToCanvas")) { + toolCalls = upsertTool( + toolCalls, + "publishToCanvas", + "image-generation 子代理建议把连续生图结果同步到画布" + ); + } + + return { + ...params.plan, + nextAction: "generate_image", + intent: "image_gen", + responseMode: "tool_then_execute", + preferredCapability: "generate_image", + shouldUseStructuredReply: true, + toolCalls, + }; +} diff --git a/lib/ai/image-workflow-memory.ts b/lib/ai/image-workflow-memory.ts new file mode 100644 index 000000000..b234a88a9 --- /dev/null +++ b/lib/ai/image-workflow-memory.ts @@ -0,0 +1,159 @@ +import "server-only"; + +import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; + +export interface ImageWorkflowMemory { + lastIntent?: "image_gen" | "image_edit" | "vision"; + lastOptimizedPrompt?: string; + lastUserPrompt?: string; + lastRequestedImageCount?: number; + lastAspectRatio?: string; + lastImageSize?: string; + lastSourceImageUrls?: string[]; + lastResultImageUrls?: string[]; +} + +function normalizeText(value: unknown, max = 2000): string | undefined { + const normalized = String(value || "").trim(); + if (!normalized) return undefined; + return normalized.slice(0, max); +} + +function normalizeImageUrls(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + const urls = Array.from( + new Set( + value + .map((item) => String(item || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)) + ) + ); + return urls.length > 0 ? urls : undefined; +} + +function normalizeImageCount(value: unknown): number | undefined { + const parsed = Number(value); + if (!Number.isFinite(parsed)) return undefined; + const rounded = Math.round(parsed); + if (rounded < 1 || rounded > 12) return undefined; + return rounded; +} + +export function extractImageWorkflowMemory( + workingState?: { + lastTurnSummary?: unknown; + lastResolvedTargetImageUrls?: string[] | null; + lastGeneratedImageUrls?: string[] | null; + } | null, +): ImageWorkflowMemory | null { + const summary = workingState?.lastTurnSummary; + let memory: ImageWorkflowMemory | null = null; + + if (summary && typeof summary === "object") { + const summaryRecord = summary as Record; + const imageWorkflowMemory = summaryRecord.imageWorkflowMemory; + if (imageWorkflowMemory && typeof imageWorkflowMemory === "object") { + const record = imageWorkflowMemory as Record; + memory = { + lastIntent: + record.lastIntent === "image_gen" || + record.lastIntent === "image_edit" || + record.lastIntent === "vision" + ? record.lastIntent + : undefined, + lastOptimizedPrompt: normalizeText(record.lastOptimizedPrompt, 4000), + lastUserPrompt: normalizeText(record.lastUserPrompt, 1000), + lastRequestedImageCount: normalizeImageCount(record.lastRequestedImageCount), + lastAspectRatio: normalizeText(record.lastAspectRatio, 32), + lastImageSize: normalizeText(record.lastImageSize, 32), + lastSourceImageUrls: normalizeImageUrls(record.lastSourceImageUrls), + lastResultImageUrls: normalizeImageUrls(record.lastResultImageUrls), + }; + } + } + + const fallbackTargetUrls = normalizeImageUrls(workingState?.lastResolvedTargetImageUrls); + const fallbackGeneratedUrls = normalizeImageUrls(workingState?.lastGeneratedImageUrls); + if (!memory) { + if (!fallbackTargetUrls && !fallbackGeneratedUrls) return null; + memory = {}; + } + + if (!memory.lastSourceImageUrls && fallbackTargetUrls) { + memory.lastSourceImageUrls = fallbackTargetUrls; + } + if (!memory.lastResultImageUrls && fallbackGeneratedUrls) { + memory.lastResultImageUrls = fallbackGeneratedUrls; + } + + return memory; +} + +export function buildImageWorkflowMemoryUpdate(params: { + existing?: ImageWorkflowMemory | null; + lastIntent?: "image_gen" | "image_edit" | "vision"; + lastOptimizedPrompt?: string; + lastUserPrompt?: string; + lastRequestedImageCount?: number; + lastAspectRatio?: string; + lastImageSize?: string; + lastSourceImageUrls?: string[] | null; + lastResultImageUrls?: string[] | null; +}) { + return { + lastIntent: params.lastIntent ?? params.existing?.lastIntent, + lastOptimizedPrompt: + normalizeText(params.lastOptimizedPrompt, 4000) ?? + params.existing?.lastOptimizedPrompt, + lastUserPrompt: + normalizeText(params.lastUserPrompt, 1000) ?? params.existing?.lastUserPrompt, + lastRequestedImageCount: + normalizeImageCount(params.lastRequestedImageCount) ?? + params.existing?.lastRequestedImageCount, + lastAspectRatio: + normalizeText(params.lastAspectRatio, 32) ?? params.existing?.lastAspectRatio, + lastImageSize: + normalizeText(params.lastImageSize, 32) ?? params.existing?.lastImageSize, + lastSourceImageUrls: + normalizeImageUrls(params.lastSourceImageUrls) ?? + params.existing?.lastSourceImageUrls, + lastResultImageUrls: + normalizeImageUrls(params.lastResultImageUrls) ?? + params.existing?.lastResultImageUrls, + } satisfies ImageWorkflowMemory; +} + +export function getImageWorkflowPlanningHints( + threadState?: AgentThreadState | null, +): string[] { + const metadata = threadState?.metadata; + const memory = + metadata && typeof metadata === "object" && !Array.isArray(metadata) + ? ((metadata.imageWorkflowMemory as ImageWorkflowMemory | undefined) || undefined) + : undefined; + if (!memory) return []; + + const hints: string[] = []; + if (memory.lastOptimizedPrompt) { + hints.push( + `上一轮可复用提示词摘要：${memory.lastOptimizedPrompt.replace(/\s+/g, " ").slice(0, 180)}` + ); + } + if (memory.lastRequestedImageCount) { + hints.push(`用户上一轮期望出图数量：${memory.lastRequestedImageCount} 张。`); + } + if (memory.lastAspectRatio) { + hints.push(`用户上一轮常用画幅：${memory.lastAspectRatio}。`); + } + if (memory.lastImageSize) { + hints.push(`用户上一轮常用清晰度：${memory.lastImageSize}。`); + } + if (memory.lastSourceImageUrls?.length) { + hints.push(`上一轮存在 ${memory.lastSourceImageUrls.length} 张参考/目标图。`); + } + if (memory.lastResultImageUrls?.length) { + hints.push(`上一轮生成结果数：${memory.lastResultImageUrls.length} 张。`); + } + return hints; +} + diff --git a/lib/ai/prompt-memory.ts b/lib/ai/prompt-memory.ts index 0bfa9b0ad..7ce96258f 100644 --- a/lib/ai/prompt-memory.ts +++ b/lib/ai/prompt-memory.ts @@ -1,5 +1,7 @@ import "server-only"; +import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; + export interface PromptMemoryEntry { optimizedPrompt: string; sourceIntent?: string; @@ -48,6 +50,24 @@ export function extractPromptMemoryFromWorkingState( return null; } +export function extractConfirmedPromptFromWorkingState( + workingState?: { + lastTurnSummary?: unknown; + } | null, +): string | undefined { + const summary = workingState?.lastTurnSummary; + if (!summary || typeof summary !== "object") return undefined; + const summaryRecord = summary as Record; + + const resolutionMode = + typeof summaryRecord.resolutionMode === "string" ? summaryRecord.resolutionMode.trim() : ""; + const optimizedPrompt = + typeof summaryRecord.optimizedPrompt === "string" ? summaryRecord.optimizedPrompt.trim() : ""; + + if (resolutionMode !== "prompt_only" || !optimizedPrompt) return undefined; + return optimizedPrompt; +} + export function buildPromptMemoryUpdate(params: { optimizedPrompt: string; sourceIntent?: string; @@ -90,3 +110,24 @@ export function isPromptCarryForwardRequest(params: { return false; } +export function getPromptMemoryPlanningHints( + threadState?: AgentThreadState | null, +): string[] { + const metadata = threadState?.metadata; + const memory = + metadata && typeof metadata === "object" && !Array.isArray(metadata) + ? ((metadata.promptMemory as PromptMemoryEntry | undefined) || undefined) + : undefined; + + if (!memory?.optimizedPrompt) return []; + + const preview = memory.optimizedPrompt.replace(/\s+/g, " ").trim().slice(0, 180); + const hints = [ + "当前会话中存在上一轮可复用的优化提示词；如果用户明确要求沿用、根据上一个提示词继续做图，应优先复用这段提示词。", + `上一轮优化提示词摘要：${preview}`, + ]; + if (memory.sourceText) { + hints.push(`该提示词来源于用户上一轮需求：${memory.sourceText}`); + } + return hints; +} diff --git a/lib/ai/prompt-optimizer.ts b/lib/ai/prompt-optimizer.ts index 78b7ec201..f288bb9fb 100644 --- a/lib/ai/prompt-optimizer.ts +++ b/lib/ai/prompt-optimizer.ts @@ -394,7 +394,7 @@ function sanitizeSuggestedParams(input?: { const normalizedSize = input.imageSize?.trim().toUpperCase() || undefined; const normalizedCount = typeof input.imageCount === "number" && Number.isFinite(input.imageCount) - ? Math.min(Math.max(Math.round(input.imageCount), 1), 4) + ? Math.min(Math.max(Math.round(input.imageCount), 1), 12) : undefined; const output: NonNullable = {}; if (normalizedAspectRatio && allowedAspectRatios.has(normalizedAspectRatio)) { diff --git a/lib/ai/runtime/agent-workflow-state.ts b/lib/ai/runtime/agent-workflow-state.ts index 4c915cdc2..7ce07d610 100644 --- a/lib/ai/runtime/agent-workflow-state.ts +++ b/lib/ai/runtime/agent-workflow-state.ts @@ -1,5 +1,7 @@ export type AgentWorkflowType = | "storyboard_video" + | "image_generation_followup" + | "image_edit_followup" | "high_cost_video" | "image_batch" | "custom"; @@ -111,4 +113,3 @@ export function isAwaitingUserConfirmationWorkflow( ): boolean { return workflowState?.status === "awaiting_user_confirmation"; } - diff --git a/lib/ai/skill-router.ts b/lib/ai/skill-router.ts index 2bac2f326..d8bf361ea 100644 --- a/lib/ai/skill-router.ts +++ b/lib/ai/skill-router.ts @@ -6,6 +6,8 @@ import type { AgentSpecialistContext, AgentSpecialistRoute, } from "@/lib/ai/specialists"; +import { buildImageEditSpecialistRoute } from "@/lib/ai/image-edit-subagent"; +import { buildImageGenerationSpecialistRoute } from "@/lib/ai/image-generation-subagent"; import { buildStoryboardSpecialistRoute } from "@/lib/video-agent/storyboard-subagent"; export interface AgentSkillRouteResult { @@ -215,9 +217,22 @@ export function routeAgentSkill( text: context.text, threadState: context.threadState, }); - const mergedOverlays = storyboardOverlay - ? [...overlays, storyboardOverlay] - : overlays; + const imageEditOverlay = buildImageEditSpecialistRoute({ + text: context.text, + threadState: context.threadState, + hasImage: context.hasImage, + }); + const imageGenerationOverlay = buildImageGenerationSpecialistRoute({ + text: context.text, + threadState: context.threadState, + hasImage: context.hasImage, + }); + const mergedOverlays = [ + ...overlays, + ...(imageEditOverlay ? [imageEditOverlay] : []), + ...(imageGenerationOverlay ? [imageGenerationOverlay] : []), + ...(storyboardOverlay ? [storyboardOverlay] : []), + ]; const toolAllowlist = dedupeToolNames([ ...(primary.toolAllowlist || []), diff --git a/lib/ai/specialists/index.ts b/lib/ai/specialists/index.ts index 73c839074..7c51828d0 100644 --- a/lib/ai/specialists/index.ts +++ b/lib/ai/specialists/index.ts @@ -1,5 +1,7 @@ import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; +import { refinePlanWithImageEditSubagent } from "@/lib/ai/image-edit-subagent"; +import { refinePlanWithImageGenerationSubagent } from "@/lib/ai/image-generation-subagent"; import { refinePlanWithStoryboardSubagent } from "@/lib/video-agent/storyboard-subagent"; import type { AgentExecutionPlan, @@ -18,7 +20,9 @@ export type AgentSpecialistDomain = | "conversation" | "vision" | "image_edit" + | "image_edit_memory" | "image_generate" + | "image_generation_memory" | "video_generate" | "storyboard" | "canvas"; @@ -27,7 +31,9 @@ export type AgentSpecialistId = | "conversation-coordinator" | "vision-analyst" | "image-edit-specialist" + | "image-edit-subagent" | "image-generate-specialist" + | "image-generation-subagent" | "video-specialist" | "storyboard-subagent" | "canvas-specialist"; @@ -235,6 +241,16 @@ export function refinePlanWithSpecialist( }); } + if (specialist.domain === "image_edit_memory") { + return refinePlanWithImageEditSubagent({ + text, + plan, + availableTools, + threadState: input.threadState, + hasImage: input.hasImage, + }); + } + if (specialist.domain === "image_generate") { if (hasTool(availableTools, "generateImage")) { toolCalls = [{ name: "generateImage", reason: "生图专家执行图片生成" }]; @@ -250,6 +266,16 @@ export function refinePlanWithSpecialist( }); } + if (specialist.domain === "image_generation_memory") { + return refinePlanWithImageGenerationSubagent({ + text, + plan, + availableTools, + threadState: input.threadState, + hasImage: input.hasImage, + }); + } + if (specialist.domain === "video_generate") { if (hasTool(availableTools, "generateVideo")) { toolCalls = [{ name: "generateVideo", reason: "视频专家执行视频生成" }]; diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index c47da6947..871d37124 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -71,6 +71,22 @@ function parseImagesFromMarkdown(markdown: string): ParsedImage[] { return parsed; } +function getProviderBatchLimit(providerType: string) { + switch (providerType) { + case "ruxa-image": + case "openai-image": + case "midjourney": + return 1; + case "gemini": + return 2; + case "alibaba-image": + case "fal-image": + return 6; + default: + return 2; + } +} + export async function generateCanvasImage( request: CanvasGenerateImageRequest ): Promise { @@ -92,7 +108,7 @@ export async function generateCanvasImage( : "2K"; const imageCount = typeof request.imageCount === "number" && Number.isFinite(request.imageCount) - ? Math.max(1, Math.min(8, Math.floor(request.imageCount))) + ? Math.max(1, Math.min(12, Math.floor(request.imageCount))) : 1; const referenceImage = typeof request.referenceImage === "string" ? request.referenceImage : undefined; @@ -191,11 +207,13 @@ export async function generateCanvasImage( const imageParams = { aspectRatio, imageSize, - imageCount, + imageCount: 1, negativePrompt, }; - let imageMarkdown = ""; + let executeGenerateBatch: + | ((batchImageCount: number) => Promise) + | null = null; if (provider.type === "gemini") { const geminiProvider = createGeminiImage({ @@ -203,61 +221,128 @@ export async function generateCanvasImage( baseURL: provider.baseUrl, }); const modelInstance = geminiProvider.chat(selectedModel.model.modelId); - const result = await modelInstance.doGenerate({ messages, imageParams }); - imageMarkdown = result.text || ""; + executeGenerateBatch = async (batchImageCount) => { + const result = await modelInstance.doGenerate({ + messages, + imageParams: { + ...imageParams, + imageCount: batchImageCount, + }, + }); + return result.text || ""; + }; } else if (provider.type === "alibaba-image") { const alibabaProvider = createAlibabaImage({ apiKey: provider.apiKey, baseURL: provider.baseUrl, }); const modelInstance = alibabaProvider.chat(selectedModel.model.modelId); - const result = await modelInstance.doGenerate({ messages, imageParams }); - imageMarkdown = result.text || ""; + executeGenerateBatch = async (batchImageCount) => { + const result = await modelInstance.doGenerate({ + messages, + imageParams: { + ...imageParams, + imageCount: batchImageCount, + }, + }); + return result.text || ""; + }; } else if (provider.type === "ruxa-image") { const ruxaProvider = createRuxaImage({ apiKey: provider.apiKey, baseURL: provider.baseUrl, }); const modelInstance = ruxaProvider.chat(selectedModel.model.modelId); - const result = await modelInstance.doGenerate({ messages, imageParams }); - imageMarkdown = result.text || ""; + executeGenerateBatch = async (batchImageCount) => { + const result = await modelInstance.doGenerate({ + messages, + imageParams: { + ...imageParams, + imageCount: batchImageCount, + }, + }); + return result.text || ""; + }; } else if (provider.type === "fal-image") { const falProvider = createFalImage({ apiKey: provider.apiKey, baseURL: provider.baseUrl, }); const modelInstance = falProvider.chat(selectedModel.model.modelId); - const result = await modelInstance.doGenerate({ messages, imageParams }); - imageMarkdown = result.text || ""; + executeGenerateBatch = async (batchImageCount) => { + const result = await modelInstance.doGenerate({ + messages, + imageParams: { + ...imageParams, + imageCount: batchImageCount, + }, + }); + return result.text || ""; + }; } else if (provider.type === "openai-image") { const openAIImageProvider = createOpenAIImage({ apiKey: provider.apiKey, baseURL: provider.baseUrl, }); const modelInstance = openAIImageProvider.chat(selectedModel.model.modelId); - const result = await modelInstance.doGenerate({ messages, imageParams }); - imageMarkdown = result.text || ""; + executeGenerateBatch = async (batchImageCount) => { + const result = await modelInstance.doGenerate({ + messages, + imageParams: { + ...imageParams, + imageCount: batchImageCount, + }, + }); + return result.text || ""; + }; } else if (provider.type === "midjourney") { const midjourneyProvider = createMidjourney({ apiKey: provider.apiKey, baseURL: provider.baseUrl, }); const modelInstance = midjourneyProvider.chat(selectedModel.model.modelId); - const result = await modelInstance.doGenerate({ messages, imageParams }); - imageMarkdown = result.text || ""; + executeGenerateBatch = async (batchImageCount) => { + const result = await modelInstance.doGenerate({ + messages, + imageParams: { + ...imageParams, + imageCount: batchImageCount, + }, + }); + return result.text || ""; + }; } else { throw new Error( `不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney 图片模型` ); } - const parsedImages = parseImagesFromMarkdown(imageMarkdown); - if (parsedImages.length === 0) { + const providerBatchLimit = getProviderBatchLimit(provider.type); + const targetImageCount = Math.max(1, imageCount); + const collectedImages: ParsedImage[] = []; + const maxBatches = Math.max(1, Math.ceil(targetImageCount / providerBatchLimit)); + + for (let batchIndex = 0; batchIndex < maxBatches; batchIndex += 1) { + const remaining = targetImageCount - collectedImages.length; + if (remaining <= 0) break; + const batchImageCount = Math.min(providerBatchLimit, remaining); + const imageMarkdown = await executeGenerateBatch(batchImageCount); + const parsedImages = parseImagesFromMarkdown(imageMarkdown); + if (parsedImages.length === 0) { + if (collectedImages.length > 0) { + break; + } + throw new Error("图片生成失败"); + } + collectedImages.push(...parsedImages.slice(0, batchImageCount)); + } + + if (collectedImages.length === 0) { throw new Error("图片生成失败"); } return { - images: parsedImages, + images: collectedImages.slice(0, targetImageCount), model: selectedModel.model.displayName, modelRef: selectedModel.model.id, prompt, diff --git a/lib/db/schema.ts b/lib/db/schema.ts index c4583eb85..739ceb5e1 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -596,8 +596,11 @@ export const canvasProject = pgTable("CanvasProject", { // 存储完整画布状态（节点、边、视口等） canvasData: jsonb("canvasData").$type<{ nodes: any[]; - edges: any[]; - viewport: { x: number; y: number; zoom: number }; + edges?: any[]; + selectedNodeIds?: string[]; + viewport?: { x: number; y: number; zoom: number }; + tool?: string; + suppressedMediaKeys?: string[]; }>(), createdAt: timestamp("createdAt").notNull().defaultNow(), updatedAt: timestamp("updatedAt").notNull().defaultNow(), diff --git a/lib/video-agent/agent-thread-state.ts b/lib/video-agent/agent-thread-state.ts index e475bb6eb..6721d6f78 100644 --- a/lib/video-agent/agent-thread-state.ts +++ b/lib/video-agent/agent-thread-state.ts @@ -1,5 +1,6 @@ import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; import type { AgentWorkflowStatus } from "@/lib/ai/runtime/agent-workflow-state"; +import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory"; import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory"; type StoryboardTaskDetail = { @@ -69,18 +70,25 @@ export function buildAgentThreadStateFromStoryboardTask( detail: StoryboardTaskDetail | null | undefined, options?: { storyboardPreferenceMemory?: StoryboardPreferenceMemory | null; + promptMemory?: PromptMemoryEntry | null; }, ): AgentThreadState | null { if (!detail?.task?.id) { - if (!options?.storyboardPreferenceMemory) return null; + if (!options?.storyboardPreferenceMemory && !options?.promptMemory) return null; return { workflowState: null, memoryHints: [ - "当前会话里存在 storyboard 偏好记忆，可在分镜/宫格任务里优先参考用户常用版式和确认习惯。", - ], + options?.storyboardPreferenceMemory + ? "当前会话里存在 storyboard 偏好记忆，可在分镜/宫格任务里优先参考用户常用版式和确认习惯。" + : "", + options?.promptMemory?.optimizedPrompt + ? "当前会话里存在上一轮可复用提示词；如果用户明确要求沿用上一个提示词继续做图，应优先把它作为本轮生图提示词。" + : "", + ].filter(Boolean), metadata: { workflowSource: "storyboard_preference_memory", storyboardPreferenceMemory: options.storyboardPreferenceMemory, + promptMemory: options?.promptMemory || undefined, }, }; } @@ -147,6 +155,7 @@ export function buildAgentThreadStateFromStoryboardTask( taskId: detail.task.id, shotCount: totalShots, storyboardPreferenceMemory: options?.storyboardPreferenceMemory || undefined, + promptMemory: options?.promptMemory || undefined, }, }; } diff --git a/store/canvas/index.ts b/store/canvas/index.ts index bc4decb1a..ea14cf7fb 100644 --- a/store/canvas/index.ts +++ b/store/canvas/index.ts @@ -73,6 +73,7 @@ interface CanvasStore { selectedNodeIds: string[]; viewport: Viewport; tool: ToolType; + suppressedMediaKeys: string[]; // 历史记录（用于撤销/重做） history: CanvasState[]; @@ -105,6 +106,7 @@ interface CanvasStore { updateNode: (id: string, updates: Partial) => void; deleteNode: (id: string) => void; deleteSelectedNodes: () => void; + addSuppressedMediaKeys: (keys: string[]) => void; // 选择操作 selectNode: (id: string, multi?: boolean) => void; @@ -179,6 +181,7 @@ export const useCanvasStore = create()( selectedNodeIds: [], viewport: defaultViewport, tool: 'select', + suppressedMediaKeys: [], history: [], historyIndex: -1, isPanelOpen: true, @@ -287,8 +290,36 @@ export const useCanvasStore = create()( } }), + addSuppressedMediaKeys: (keys) => set((state) => { + const normalized = Array.from( + new Set((keys || []).map((item) => String(item || "").trim()).filter(Boolean)) + ); + if (normalized.length === 0) return; + const existing = new Set(state.suppressedMediaKeys); + normalized.forEach((key) => existing.add(key)); + state.suppressedMediaKeys = Array.from(existing); + }), + deleteNode: (id) => { set((state) => { + const targetNode = state.nodes.find(n => n.id === id); + if (targetNode && (targetNode.type === 'image' || targetNode.type === 'video')) { + const data = ((targetNode.data as any) || {}) as Record; + const metadata = ((data.metadata as any) || {}) as Record; + const sourceMessageId = String(metadata.sourceMessageId || "").trim(); + const sourceImageKey = String(metadata.sourceImageKey || "").trim(); + const sourceMediaUrl = String(data.src || "").trim(); + const suppressionKeys = [ + sourceMessageId && sourceImageKey ? `image:${sourceMessageId}::${sourceImageKey}` : "", + targetNode.type === 'video' && sourceMediaUrl ? `video:${sourceMediaUrl}` : "", + targetNode.type === 'image' && sourceMediaUrl ? `image-url:${sourceMediaUrl}` : "", + ].filter(Boolean); + if (suppressionKeys.length > 0) { + const existing = new Set(state.suppressedMediaKeys); + suppressionKeys.forEach((key) => existing.add(key)); + state.suppressedMediaKeys = Array.from(existing); + } + } state.nodes = state.nodes.filter(n => n.id !== id); state.edges = state.edges.filter(e => e.source !== id && e.target !== id); state.selectedNodeIds = state.selectedNodeIds.filter(nid => nid !== id); @@ -299,6 +330,23 @@ export const useCanvasStore = create()( deleteSelectedNodes: () => { const { selectedNodeIds } = get(); set((state) => { + const selectedNodes = state.nodes.filter(n => selectedNodeIds.includes(n.id)); + const existing = new Set(state.suppressedMediaKeys); + selectedNodes.forEach((targetNode) => { + if (targetNode.type !== 'image' && targetNode.type !== 'video') return; + const data = ((targetNode.data as any) || {}) as Record; + const metadata = ((data.metadata as any) || {}) as Record; + const sourceMessageId = String(metadata.sourceMessageId || "").trim(); + const sourceImageKey = String(metadata.sourceImageKey || "").trim(); + const sourceMediaUrl = String(data.src || "").trim(); + const suppressionKeys = [ + sourceMessageId && sourceImageKey ? `image:${sourceMessageId}::${sourceImageKey}` : "", + targetNode.type === 'video' && sourceMediaUrl ? `video:${sourceMediaUrl}` : "", + targetNode.type === 'image' && sourceMediaUrl ? `image-url:${sourceMediaUrl}` : "", + ].filter(Boolean); + suppressionKeys.forEach((key) => existing.add(key)); + }); + state.suppressedMediaKeys = Array.from(existing); state.nodes = state.nodes.filter(n => !selectedNodeIds.includes(n.id)); state.edges = state.edges.filter( e => !selectedNodeIds.includes(e.source) && !selectedNodeIds.includes(e.target) @@ -451,6 +499,9 @@ export const useCanvasStore = create()( if (canvasState.selectedNodeIds) state.selectedNodeIds = canvasState.selectedNodeIds; if (canvasState.viewport) state.viewport = canvasState.viewport; if (canvasState.tool) state.tool = canvasState.tool; + state.suppressedMediaKeys = Array.isArray(canvasState.suppressedMediaKeys) + ? canvasState.suppressedMediaKeys.map((item) => String(item || "").trim()).filter(Boolean) + : []; }), clearCanvas: () => set((state) => { @@ -458,6 +509,7 @@ export const useCanvasStore = create()( state.edges = []; state.selectedNodeIds = []; state.viewport = defaultViewport; + state.suppressedMediaKeys = []; state.history = []; state.historyIndex = -1; }), @@ -470,6 +522,7 @@ export const useCanvasStore = create()( selectedNodeIds: state.selectedNodeIds, viewport: state.viewport, tool: state.tool, + suppressedMediaKeys: state.suppressedMediaKeys, }; }, diff --git a/types/canvas/index.ts b/types/canvas/index.ts index a1ee72ae8..d6e27bc7a 100644 --- a/types/canvas/index.ts +++ b/types/canvas/index.ts @@ -137,6 +137,7 @@ export interface CanvasState { selectedNodeIds: string[]; viewport: Viewport; tool: ToolType; + suppressedMediaKeys?: string[]; } // 画布项目