diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 3dd0b80fc..fcbfd0b3d 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -250,18 +250,21 @@ import { pruneTaskBoundaryContext, pruneTaskBoundaryHistoryMessages, } from "@/lib/ai/runtime/task-boundary-context-pruner"; +import { buildRecentDialogueMemory } from "@/lib/ai/runtime/recent-dialogue-memory"; import { resolveAgentModelSlotDescriptor, upsertAgentModelSlot, type AgentModelSlots, } from "@/lib/ai/runtime/agent-model-slots"; import { isPureGreetingTurnRequest } from "@/lib/ai/runtime/task-boundary-classifier"; +import { isCapabilityQuestionTurnRequest } from "@/lib/ai/runtime/task-boundary-text"; import { getProviderHealthSnapshot, recordProviderHealthFailure, recordProviderHealthSuccess, shouldAvoidProvider, } from "@/lib/ai/runtime/provider-health"; +import { resolveCanvasAgentRequestPolicy } from "@/lib/ai/runtime/canvas-agent-kernel"; import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter"; import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; @@ -541,6 +544,65 @@ function writeCanvasOperationShadowChunk(params: { ); } +const AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN = + /(?:^|\n)\s*(?:#{1,6}\s*)?(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)(?:\s|[:：]|$)|【\s*(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)\s*】|提示词|自动优化|模型|提交生成|回写|流程|步骤/; + +function buildAgentExecutionPrefaceText(params: { + friendlyMessage?: string | null; + fallback: string; +}) { + const fallback = String(params.fallback || "").replace(/\s+/g, " ").trim(); + const rawMessage = String(params.friendlyMessage || "") + .replace(//g, "") + .trim(); + + if (!rawMessage || AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN.test(rawMessage)) { + return fallback; + } + + const candidate = rawMessage + .split(/\r?\n/) + .map((line) => line.replace(/^[-*]\s*/, "").trim()) + .find(Boolean); + + if (!candidate || AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN.test(candidate)) { + return fallback; + } + + const firstSentence = candidate.match(/^(.{1,80}?[。！？!?~～])/u)?.[1] || candidate; + const compact = firstSentence.replace(/\s+/g, " ").trim().slice(0, 80); + return /[。！？!?~～]$/.test(compact) ? compact : `${compact}。`; +} + +async function writeAgentExecutionPrefaceMessage(params: { + writer: StreamWriter; + chatId: string; + label: string; + friendlyMessage?: string | null; + fallback: string; +}) { + const text = buildAgentExecutionPrefaceText({ + friendlyMessage: params.friendlyMessage, + fallback: params.fallback, + }); + if (!text) return null; + + const messageId = generateUUID(); + writeAssistantTextChunk(params.writer, text, params.label, messageId); + + try { + await saveAssistantTextOnlyMessage({ + chatId: params.chatId, + messageId, + text, + }); + } catch (error) { + chatApiLogger.warn("[Chat API] Failed to persist agent execution preface:", error); + } + + return messageId; +} + async function replayAssistantMessagesForDuplicateRequest({ chatId, userMessageId, @@ -785,6 +847,19 @@ const DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS = 12_000; const CHAT_CREATION_LOOKUP_RETRY_DELAYS_MS = [60, 180, 360] as const; const DETAIL_REPLACEMENT_PROMPT_PACK_REQUIRED_MESSAGE = "请先在「风格化提示词配置」里选择提示词组合，再执行详情场景替换；缺少提示词组时系统不会处理。"; +const CANVAS_AGENT_GREETING_MESSAGE = + "你好，我在。你可以直接告诉我想做什么：生图、修图、生成视频、分析图片都可以。"; +const CANVAS_AGENT_CAPABILITY_MESSAGE = [ + "我主要帮你完成画布里的视觉创作任务：", + "", + "- 生图：按文字、参考图或画布选中内容生成图片。", + "- 修图：换背景、去元素、局部重绘、扩图、增强、抠图、改尺寸。", + "- 视频：文生视频、图生视频、分镜/镜头视频需求整理与生成。", + "- 图片分析：识别主体、风格、构图、材质、光线，并整理可执行提示词。", + "- 画布协作：读取当前画布选择、槽位、最近结果，继续生成、修改或写回画布。", + "", + "你可以直接说：把这张图换成白底、参考左边图片生成 3 张主图、让这张图动起来。", +].join("\n"); const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -1025,6 +1100,10 @@ export async function POST(request: Request) { let confirmedPromptForConversationState: string | undefined; let smartAutoHistoryMessagesFromDb: DBMessage[] = []; let smartAutoMediaHistoryMessagesFromDb: DBMessage[] = []; + let recentDialogueMemory = buildRecentDialogueMemory({ + messages: [], + currentText: requestTextContent, + }); let preferredAgentModelId: string | undefined; let configuredAgentConversationModelId: string | undefined; let agentModelSlots: AgentModelSlots = {}; @@ -1036,8 +1115,19 @@ export async function POST(request: Request) { const shouldRespectDirectComposerMode = composerMode === "image" || composerMode === "video" || composerMode === "detail"; const shouldIsolateDirectComposerContext = shouldRespectDirectComposerMode; - let shouldRunAgentConversation = finalModelId === "auto"; + const requestPolicy = resolveCanvasAgentRequestPolicy({ + composerMode, + agentEntryMode: agentPresetId ? "preset" : "auto", + selectedChatModel: finalModelId, + hasCanvasControlIntentFastPath, + }); + finalModelId = requestPolicy.effectiveSelectedChatModel; + let shouldRunAgentConversation = + finalModelId === "auto" || requestPolicy.shouldForceToolOrchestration; let shouldKeepSelectedModel = false; + if (requestPolicy.reason === "agent_auto_composer") { + chatApiLogger.info("[Smart Auto Mode] 🧭 Agent composer forces tool orchestration, ignoring stale selected model"); + } if (shouldRespectDirectComposerMode && shouldRunAgentConversation) { shouldRunAgentConversation = false; chatApiLogger.info(`[Smart Auto Mode] ⛳ Direct composer mode (${composerMode}) disables initial auto-agent routing`); @@ -1158,9 +1248,20 @@ export async function POST(request: Request) { const hasCurrentMessageMediaInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); + const isGreetingTurnRequest = isPureGreetingTurnRequest(requestTextContent); + const isCapabilityQuestionRequest = + isCapabilityQuestionTurnRequest(requestTextContent); + const canvasFastPathMessage = isCapabilityQuestionRequest + ? CANVAS_AGENT_CAPABILITY_MESSAGE + : isGreetingTurnRequest + ? CANVAS_AGENT_GREETING_MESSAGE + : ""; if ( - shouldRunAgentConversation && - isPureGreetingTurnRequest(requestTextContent) && + (shouldRunAgentConversation || + isCanvasPageRequest || + contextPayload?.source === "canvas" || + Boolean(requestCanvasAgentContext)) && + canvasFastPathMessage && !hasCurrentMessageMediaInput ) { const session = await auth(); @@ -1169,7 +1270,6 @@ export async function POST(request: Request) { return new ChatSDKError("unauthorized:chat").toResponse(); } - const greetingMessage = "你好，我在。你可以直接告诉我想做什么：生图、修图、生成视频、分析图片都可以。"; const fastPathTitle = requestTextContent.trim().slice(0, 80) || "新对话"; await saveChat({ id, @@ -1190,7 +1290,12 @@ export async function POST(request: Request) { const greetingStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); - writeAssistantTextChunk(writer, greetingMessage, "greeting-fast-path", messageId); + writeAssistantTextChunk( + writer, + canvasFastPathMessage, + isCapabilityQuestionRequest ? "capability-fast-path" : "greeting-fast-path", + messageId + ); await saveMessages({ messages: [ @@ -1211,7 +1316,7 @@ export async function POST(request: Request) { id: messageId, chatId: id, role: "assistant", - parts: [{ type: "text", text: greetingMessage }], + parts: [{ type: "text", text: canvasFastPathMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, @@ -1418,6 +1523,17 @@ export async function POST(request: Request) { taskType: taskBoundaryDecision.taskType, taskMode: taskBoundaryDecision.taskMode, }); + recentDialogueMemory = buildRecentDialogueMemory({ + messages: loadedHistoryMessages, + currentText: smartAutoInputText, + maxEntries: taskBoundaryDecision.taskType === "chat" ? 6 : 4, + maxTotalChars: taskBoundaryDecision.taskType === "chat" ? 1000 : 700, + }); + chatApiLogger.info("[RecentDialogueMemory] Smart auto recall:", { + entries: recentDialogueMemory.entries.length, + isRecallRequest: recentDialogueMemory.isRecallRequest, + taskType: taskBoundaryDecision.taskType, + }); autoTextForFallback = smartAutoInputText; if (!requestWorkingState && !shouldIsolateDirectComposerContext) { @@ -1492,23 +1608,12 @@ export async function POST(request: Request) { resolutionConfidence: requestTargetResolution.confidence, }); - // 🆕 从历史消息中提取对话文本（用于上下文理解） - const historyMessages: { role: "user" | "assistant"; content: string }[] = []; - for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条 - const parts = msg.parts as any[]; - if (parts && parts.length > 0) { - // 提取文本内容 - const textParts = parts.filter((p: any) => p.type === 'text'); - const content = textParts.map((p: any) => p.text || '').join(' ').trim(); - if (content) { - historyMessages.push({ - role: msg.role as "user" | "assistant", - content: content.substring(0, 300), // 限制长度 - }); - } - } - } - chatApiLogger.info("[Smart Auto Mode] History context:", historyMessages.length, "messages"); + const historyMessages: { role: "user" | "assistant"; content: string }[] = + recentDialogueMemory.entries.map((entry) => ({ + role: entry.role, + content: entry.content, + })); + chatApiLogger.info("[Smart Auto Mode] History context:", historyMessages.length, "memory entries"); // 2. 使用 LangGraph 驱动的自动模式工作流 const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph"); @@ -1541,8 +1646,15 @@ export async function POST(request: Request) { workflowState: composedConversationState.threadState.workflowState, memoryHints: composedConversationState.threadState.memoryHints, contextSummary: composedConversationState.threadState.contextSummary, + recentDialogueMemory: recentDialogueMemory.contextSummary, }, }); + const agentGraphContext = [ + composedConversationState.composedConversationMemory.contextSummary, + recentDialogueMemory.contextSummary, + ] + .filter(Boolean) + .join("\n"); const autoGraphResult = await withOperationTimeout( runAutoAgentGraph({ text: smartAutoInputText, @@ -1570,10 +1682,7 @@ export async function POST(request: Request) { requestResolvedMediaInput.historyUserImageUrls.length, currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length, historyMessages, - context: - composedConversationState.composedConversationMemory.contextSummary || - effectiveAutoContextText || - undefined, + context: agentGraphContext || effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, threadState: composedConversationState.threadState, }), @@ -2676,6 +2785,21 @@ export async function POST(request: Request) { purpose: "media", hasCurrentImageInput: hasImageInputForConversationAction, }); + recentDialogueMemory = shouldIsolateDirectComposerContext + ? recentDialogueMemory + : buildRecentDialogueMemory({ + messages: loadedMessagesForBoundary, + currentText: requestTextContent, + maxEntries: taskBoundaryDecision.taskType === "chat" ? 6 : 4, + maxTotalChars: taskBoundaryDecision.taskType === "chat" ? 1000 : 700, + }); + if (recentDialogueMemory.entries.length > 0) { + chatApiLogger.info("[RecentDialogueMemory] Request recall:", { + entries: recentDialogueMemory.entries.length, + isRecallRequest: recentDialogueMemory.isRecallRequest, + taskType: taskBoundaryDecision.taskType, + }); + } if (!requestWorkingState && !shouldIsolateDirectComposerContext) { requestWorkingState = await getConversationWorkingStateByChatId({ @@ -7715,6 +7839,16 @@ export async function POST(request: Request) { ) { const directImageStream = createUIMessageStream({ execute: async ({ writer }) => { + await writeAgentExecutionPrefaceMessage({ + writer, + chatId: id, + label: "agent:generate_image:preface", + friendlyMessage: smartAutoFriendlyMessage, + fallback: + directAgentToolTargetImageUrls.length > 0 + ? "收到，我会参考你选的图片来生成新画面，稍等一下。" + : "好的，我按你的描述来做一版画面，稍等我处理一下。", + }); writeAgentProgressChunk( writer, buildDirectAgentImageContextProgress({ @@ -7762,6 +7896,32 @@ export async function POST(request: Request) { operationLabel: "agent-generate-image", resultType: "image", }); + const directImageStatusPayload = { + providerType: requestedModelProviderType, + modelName: finalModelId, + imageCount: + effectiveImageParams?.imageCount || + defaultImageQualityParams.imageCount || + 1, + }; + const stopDirectImageKeepalive = startStreamKeepalive( + writer, + "agent:generate_image", + DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS, + ); + const stopDirectImageStatus = + contextPayload?.source === "canvas" + ? setInterval(() => { + writeImageGenerationStatusChunk( + writer, + { + phase: "processing", + ...directImageStatusPayload, + }, + "agent:generate_image:status-processing-keepalive", + ); + }, DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS) + : null; try { const directImageResult = await executeDirectAgentImageGeneration({ requestTextContent, @@ -7903,6 +8063,11 @@ export async function POST(request: Request) { "", "agent:generate_image:end-error", ); + } finally { + stopDirectImageKeepalive(); + if (stopDirectImageStatus) { + clearInterval(stopDirectImageStatus); + } } }, }); @@ -7917,6 +8082,16 @@ export async function POST(request: Request) { ) { const directVideoStream = createUIMessageStream({ execute: async ({ writer }) => { + await writeAgentExecutionPrefaceMessage({ + writer, + chatId: id, + label: "agent:generate_video:preface", + friendlyMessage: smartAutoFriendlyMessage, + fallback: + directAgentToolTargetImageUrls.length > 0 + ? "收到，我会参考你选的画面来生成视频，稍等一下。" + : "好的，我按你的需求来做这段视频，稍等我处理一下。", + }); const assistantMessageId = generateUUID(); writeAgentProgressChunk( writer, @@ -8123,6 +8298,13 @@ export async function POST(request: Request) { const directToolLabel = getDirectAgentToolLabel(directAgentToolCapability); const directToolStream = createUIMessageStream({ execute: async ({ writer }) => { + await writeAgentExecutionPrefaceMessage({ + writer, + chatId: id, + label: `agent:${directAgentToolCapability}:preface`, + friendlyMessage: smartAutoFriendlyMessage, + fallback: `好的，我来帮你${directToolLabel}，稍等我处理一下。`, + }); writeAgentProgressChunk( writer, buildDirectAgentImageContextProgress({ @@ -8493,16 +8675,39 @@ export async function POST(request: Request) { resolvedRuntimeModelLookupId = actualModelId; } - const effectiveConversationSystemPrompt = shouldRunAgentConversation - ? `${await buildAgentRoleIdentityInstruction({ + const recentDialogueSystemContext = recentDialogueMemory.contextSummary + ? [ + recentDialogueMemory.contextSummary, + ...recentDialogueMemory.memoryHints, + ].join("\n") + : ""; + const baseConversationSystemPrompt = systemPrompt({ + selectedChatModel: finalModelId, + requestHints, + }); + const shouldUseProductIdentityInstruction = + shouldRunAgentConversation || + isCanvasPageRequest || + contextPayload?.source === "canvas" || + Boolean(requestCanvasAgentContext); + const agentIdentityInstruction = shouldUseProductIdentityInstruction + ? await buildAgentRoleIdentityInstruction({ allowVideoGeneration: allowAgentVideoGeneration, hasCanvasContext: Boolean( + isCanvasPageRequest || effectiveContextPayload?.source === "canvas" || effectiveContextPayload?.selectedNodeIds?.length || effectiveContextPayload?.canvasImageNodes?.length, ), - })}\n\n${systemPrompt({ selectedChatModel: finalModelId, requestHints })}` - : systemPrompt({ selectedChatModel: finalModelId, requestHints }); + }) + : ""; + const effectiveConversationSystemPrompt = [ + agentIdentityInstruction, + recentDialogueSystemContext, + baseConversationSystemPrompt, + ] + .filter(Boolean) + .join("\n\n"); const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor( resolvedRuntimeModelLookupId, diff --git a/app/api/canvas/clawd-chat/history/route.ts b/app/api/canvas/clawd-chat/history/route.ts new file mode 100644 index 000000000..822587749 --- /dev/null +++ b/app/api/canvas/clawd-chat/history/route.ts @@ -0,0 +1,42 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { getClawdChatsByUserId } from "@/lib/db/queries"; + +export const dynamic = "force-dynamic"; +export const revalidate = 0; + +export async function GET(request: Request) { + const session = await auth().catch(() => null); + + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const url = new URL(request.url); + const limit = Math.max( + 1, + Math.min(100, Math.round(Number(url.searchParams.get("limit") || 50))) + ); + + const chats = await getClawdChatsByUserId({ + userId: session.user.id, + limit, + }); + + return NextResponse.json( + { + chats: chats.map((chat) => ({ + id: chat.id, + title: chat.title, + createdAt: chat.createdAt?.toISOString?.() || chat.createdAt, + updatedAt: chat.updatedAt?.toISOString?.() || chat.updatedAt, + messageCount: Number(chat.messageCount || 0), + })), + }, + { + headers: { + "Cache-Control": "no-store, max-age=0", + }, + } + ); +} diff --git a/components/canvas/AiWashWorkspace.tsx b/components/canvas/AiWashWorkspace.tsx index 5e878afe9..10690b970 100644 --- a/components/canvas/AiWashWorkspace.tsx +++ b/components/canvas/AiWashWorkspace.tsx @@ -3,6 +3,7 @@ import { ChevronDown, Clock3, + Download, History as HistoryIcon, ImageIcon, Loader2, @@ -45,6 +46,7 @@ import { type AiWashControlLevel, type AiWashPreset, } from "@/lib/ai-wash/config"; +import { downloadCanvasImageAsPng } from "@/lib/client/canvas-image-download"; import { buildImageProxyUrl } from "@/lib/image-delivery"; import { cn } from "@/lib/utils"; import { imageGenerationService } from "@/services/canvas"; @@ -368,6 +370,11 @@ function getAiWashStatusLabel(status?: AiWashResultStatus) { } } +function buildAiWashDownloadName(result: AiWashResultCard) { + const createdAt = result.createdAt || Date.now(); + return `ai-wash-${result.title || result.presetId || "result"}-${createdAt}`; +} + function composePresetPrompt(params: { preset: AiWashPreset; selectedLevelIndexByControl: Record; @@ -456,6 +463,9 @@ export function AiWashWorkspace({ useState>({}); const [historySourceOverride, setHistorySourceOverride] = useState(""); const [isSubmitting, setIsSubmitting] = useState(false); + const [downloadingResultId, setDownloadingResultId] = useState( + null + ); const [isHistoryExpanded, setIsHistoryExpanded] = useState(false); const [lastErrorMessage, setLastErrorMessage] = useState(""); const [comparePositionById, setComparePositionById] = useState< @@ -977,6 +987,30 @@ export function AiWashWorkspace({ }); }, [compareSourceUrl, onApplyResult, selectedResult]); + const handleDownloadSelectedResult = useCallback(async () => { + if (!selectedResult?.imageUrl) { + toast.error("请先选择一张洗图结果"); + return; + } + + const downloadId = selectedResult.presetId || selectedResult.imageUrl; + setDownloadingResultId(downloadId); + try { + await downloadCanvasImageAsPng({ + url: selectedResult.imageUrl, + name: buildAiWashDownloadName(selectedResult), + candidates: [selectedResult.imageUrl], + }); + toast.success("图片已开始下载"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "下载失败"); + } finally { + setDownloadingResultId((current) => + current === downloadId ? null : current + ); + } + }, [selectedResult]); + const handleLoadHistoryBatch = useCallback((entry: AiWashHistoryBatch) => { const loadedPresetId = (entry.items.find((item) => item.isDefault) || entry.items[0])?.presetId || ""; @@ -1430,6 +1464,25 @@ export function AiWashWorkspace({ + {selectedResult?.imageUrl ? ( + + ) : null} + -
- - {isSessionMenuOpen && ( -
+ - ); - }) + + Claw + + {isClawdHistoryMenuOpen && ( +
+
+
+ Claw 对话历史 +
+ +
+
+ {isClawdHistoryLoading ? ( +
+ + 正在加载 Claw 历史 +
+ ) : (clawdHistoryData?.chats || []).length === 0 ? ( +
+ 暂无 Claw 历史 +
+ ) : ( + (clawdHistoryData?.chats || []).map((item) => { + const sessionId = normalizeCanvasChatSessionId(item.id); + const isActive = sessionId === activeChatId; + const timeLabel = formatClawdHistoryTime(item.updatedAt); + return ( + + ); + }) + )} +
-
- )} -
+ )} + + ) : ( +
+ + {isSessionMenuOpen && ( +
+
聊天窗口记录
+
+ {currentModeChatSessions.length === 0 ? ( +
暂无记录
+ ) : ( + currentModeChatSessions.map((session, index) => { + const isActive = session.id === activeChatId; + const timeLabel = new Date(session.updatedAt).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + }); + return ( + + ); + }) + )} +
+
+ )} +
+ )} + ) : null} ); } @@ -1021,6 +1055,9 @@ export function StyleProWorkspace({ useState("style"); const [selectedModelId, setSelectedModelId] = useState(""); const [uploading, setUploading] = useState(false); + const [downloadingImageId, setDownloadingImageId] = useState( + null + ); const [generationTaskState, setGenerationTaskState] = useState(null); const generating = @@ -1458,6 +1495,34 @@ export function StyleProWorkspace({ [config] ); + const handleDownloadResult = useCallback( + async (item?: StyleProHistoryItem | null) => { + const target = item || currentResult; + if (!target?.resultImageUrl) { + toast.error("暂无可下载的结果图"); + return; + } + + const downloadId = target.id || target.resultImageUrl; + setDownloadingImageId(downloadId); + try { + await downloadCanvasImageAsPng({ + url: target.resultImageUrl, + name: buildStyleProDownloadName(target), + candidates: [target.resultImageUrl], + }); + toast.success("图片已开始下载"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "下载失败"); + } finally { + setDownloadingImageId((current) => + current === downloadId ? null : current + ); + } + }, + [currentResult] + ); + const disabledReason = !sourceImageUrl ? "请先上传图片" : generationMode === "realistic" && config?.realisticModeAvailable === false @@ -1875,6 +1940,15 @@ export function StyleProWorkspace({ { + void handleDownloadResult(currentResult); + }} taskState={generationTaskState} /> diff --git a/lib/ai/context-memory/media-input.ts b/lib/ai/context-memory/media-input.ts index 6d06e1697..228c24391 100644 --- a/lib/ai/context-memory/media-input.ts +++ b/lib/ai/context-memory/media-input.ts @@ -308,6 +308,7 @@ export function resolveChatMediaInput( resolvedImageRefs, currentMessageImageUrls: currentMessageImageRefs.map((ref) => ref.url), selectedImageUrls: selectedImageRefs.map((ref) => ref.url), + canvasImageUrls: canvasImageRefs.map((ref) => ref.url), historyUserImageUrls: history.latestUser.map((ref) => ref.url), historyAssistantImageUrls: history.latestAssistant.map((ref) => ref.url), resolvedImageUrls: resolvedImageRefs.map((ref) => ref.url), diff --git a/lib/ai/context-memory/target-resolution.ts b/lib/ai/context-memory/target-resolution.ts index e0d8f8064..fb253758b 100644 --- a/lib/ai/context-memory/target-resolution.ts +++ b/lib/ai/context-memory/target-resolution.ts @@ -5,16 +5,17 @@ import type { ResolvedImageRef, ResolvedTargetAssets, } from "@/lib/ai/context-memory/types"; +import { + hasCanvasAgentActionableImageEditIntent, + hasCanvasAgentDeicticTarget, + resolveCanvasAgentTargetPreference, +} from "@/lib/ai/runtime/canvas-agent-kernel"; const ORIGINAL_IMAGE_REGEX = /(原图|上传图|参考图|最初那张|第一张图|素材图|源图|原始图片|原始那张|我发的那张|上传的那张|原来的图)/i; const LATEST_RESULT_REGEX = /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果|上一版|刚才那张结果|生成的图|生成图|结果图|出图结果|刚出的图|上一张生成图|上一轮结果图|现在这张生成图)/i; -const DEICTIC_TARGET_REGEX = - /(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i; const SELECTION_REGEX = /(选中的|当前这张|我选的|这几张|这些图|选中图)/i; -const ACTIONABLE_EDIT_REGEX = - /(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格|亮一点|暗一点|白底|透明底)/i; const ACTIONABLE_GENERATION_REGEX = /(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i; const IMPLICIT_HISTORY_TARGET_REGEX = @@ -255,6 +256,7 @@ export function resolveTargetAssets( const focusPolicy = params.contextPayload?.focusPolicy || "auto"; const workingStateGeneratedUrls = normalizeStringArray(workingState?.lastGeneratedImageUrls); const workingStateUploadedUrls = normalizeStringArray(workingState?.lastUploadedImageUrls); + const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls); const generatedRefs = workingStateGeneratedUrls.length > 0 ? toWorkingMemoryRefs(workingStateGeneratedUrls) : assistantRefs; const originalRefs = @@ -384,7 +386,39 @@ export function resolveTargetAssets( }); } - if (DEICTIC_TARGET_REGEX.test(text)) { + const canvasAgentTargetPreference = resolveCanvasAgentTargetPreference({ + text, + selectedImageCount: selectedRefs.length, + latestResultImageCount: generatedRefs.length, + workingMemoryTargetImageCount: workingStateResolvedUrls.length, + }); + if (canvasAgentTargetPreference.shouldPreferLatestResult) { + return buildResolvedTargetAssets({ + targetRefs: generatedRefs, + candidateRefs, + resolutionMode: + workingStateGeneratedUrls.length > 0 ? "working_memory" : "latest_result", + confidence: canvasAgentTargetPreference.confidence, + needsClarification: false, + reason: "用户使用指代表达并提出图片编辑意图，默认续接最近生成结果", + fromHistory: true, + }); + } + + if (canvasAgentTargetPreference.shouldPreferWorkingMemoryTarget) { + const refs = toWorkingMemoryRefs(workingStateResolvedUrls); + return buildResolvedTargetAssets({ + targetRefs: refs, + candidateRefs, + resolutionMode: "working_memory", + confidence: canvasAgentTargetPreference.confidence, + needsClarification: false, + reason: "用户使用指代表达并提出图片编辑意图，使用工作记忆中的上一轮目标图", + fromHistory: true, + }); + } + + if (hasCanvasAgentDeicticTarget(text)) { if (selectedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: selectedRefs, @@ -421,7 +455,7 @@ export function resolveTargetAssets( } if ( - ACTIONABLE_EDIT_REGEX.test(text) && + hasCanvasAgentActionableImageEditIntent(text) && currentRefs.length === 0 && generatedRefs.length > 0 && selectedLooksLikeOriginal && @@ -463,14 +497,28 @@ export function resolveTargetAssets( }); } - const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls); + if ( + canvasRefs.length === 1 && + (hasCanvasAgentActionableImageEditIntent(text) || focusPolicy === "auto") + ) { + return buildResolvedTargetAssets({ + targetRefs: [canvasRefs[0]], + candidateRefs, + resolutionMode: "text_reference", + confidence: 84, + needsClarification: false, + reason: "用户使用画布/图片指代表达，且当前画布仅有一张可处理图片", + fromHistory: true, + }); + } + const allowsImplicitHistoryTarget = IMPLICIT_HISTORY_TARGET_REGEX.test(text) && - (ACTIONABLE_EDIT_REGEX.test(text) || + (hasCanvasAgentActionableImageEditIntent(text) || ACTIONABLE_GENERATION_REGEX.test(text) || LATEST_RESULT_REGEX.test(text) || ORIGINAL_IMAGE_REGEX.test(text) || - DEICTIC_TARGET_REGEX.test(text) || + hasCanvasAgentDeicticTarget(text) || SELECTION_REGEX.test(text)); if (allowsImplicitHistoryTarget) { @@ -525,7 +573,19 @@ export function resolveTargetAssets( } } - if (ACTIONABLE_EDIT_REGEX.test(text)) { + if (hasCanvasAgentActionableImageEditIntent(text)) { + if (canvasRefs.length === 1) { + return buildResolvedTargetAssets({ + targetRefs: [canvasRefs[0]], + candidateRefs, + resolutionMode: "text_reference", + confidence: 82, + needsClarification: false, + reason: "用户表达了图片编辑意图，且当前画布仅有一张可处理图片", + fromHistory: true, + }); + } + return buildResolvedTargetAssets({ targetRefs: [], candidateRefs, diff --git a/lib/ai/context-memory/types.ts b/lib/ai/context-memory/types.ts index d183534fe..bc6cafdb4 100644 --- a/lib/ai/context-memory/types.ts +++ b/lib/ai/context-memory/types.ts @@ -57,6 +57,7 @@ export interface ResolvedChatMediaInput { resolvedImageRefs: ResolvedImageRef[]; currentMessageImageUrls: string[]; selectedImageUrls: string[]; + canvasImageUrls: string[]; historyUserImageUrls: string[]; historyAssistantImageUrls: string[]; resolvedImageUrls: string[]; diff --git a/lib/ai/runtime/canvas-agent-kernel.ts b/lib/ai/runtime/canvas-agent-kernel.ts new file mode 100644 index 000000000..6c5ae36f2 --- /dev/null +++ b/lib/ai/runtime/canvas-agent-kernel.ts @@ -0,0 +1,156 @@ +import type { DirectAgentToolCapability } from "@/lib/ai/runtime/direct-agent-execution"; + +export type CanvasAgentComposerMode = "agent" | "image" | "video" | "detail" | "clawd"; +export type CanvasAgentEntryMode = "auto" | "preset"; + +export interface CanvasAgentRequestPolicy { + effectiveSelectedChatModel: string; + shouldForceToolOrchestration: boolean; + reason: string | null; +} + +export interface CanvasAgentTargetPreference { + shouldPreferLatestResult: boolean; + shouldPreferWorkingMemoryTarget: boolean; + confidence: number; + reason: string | null; +} + +const DEICTIC_TARGET_REGEX = + /(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i; + +const ACTIONABLE_IMAGE_EDIT_REGEX = + /(抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|调色|调整颜色|亮度|对比度|饱和度|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调|处理)/i; + +export function normalizeCanvasAgentRequestText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +export function hasCanvasAgentDeicticTarget(text: string) { + return DEICTIC_TARGET_REGEX.test(String(text || "")); +} + +export function hasCanvasAgentActionableImageEditIntent(text: string) { + return ACTIONABLE_IMAGE_EDIT_REGEX.test(String(text || "")); +} + +export function inferCanvasAgentImageEditCapability( + text: string, +): DirectAgentToolCapability | null { + const normalizedText = normalizeCanvasAgentRequestText(text); + if (!normalizedText) return null; + if (/去水印|水印|去logo|移除logo|去除logo/.test(normalizedText)) { + return "remove_watermark"; + } + if (/抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景/.test(normalizedText)) { + return "remove_background"; + } + if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(normalizedText)) { + return "angle_variant"; + } + if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(normalizedText)) { + return "extend"; + } + if (/调整尺寸|裁剪|resize|crop|缩放|放大|缩小/.test(normalizedText)) { + return "resize"; + } + if (/局部|框选|选区|区域|涂抹|蒙版/.test(normalizedText)) { + return "local_edit"; + } + if (/高清|清晰|锐化|降噪|提升质量|修复|清晰化/.test(normalizedText)) { + return "enhance"; + } + if (hasCanvasAgentActionableImageEditIntent(text)) { + return "generic_edit"; + } + return null; +} + +export function resolveCanvasAgentRequestPolicy(params: { + composerMode?: string | null; + agentEntryMode?: string | null; + selectedChatModel?: string | null; + hasCanvasControlIntentFastPath?: boolean; +}): CanvasAgentRequestPolicy { + const composerMode = String(params.composerMode || "").trim(); + const agentEntryMode = String(params.agentEntryMode || "").trim(); + const selectedChatModel = String(params.selectedChatModel || "").trim() || "auto"; + const isAgentAutoComposer = composerMode === "agent" && agentEntryMode !== "preset"; + const shouldForceToolOrchestration = + isAgentAutoComposer || Boolean(params.hasCanvasControlIntentFastPath); + + if (isAgentAutoComposer) { + return { + effectiveSelectedChatModel: "auto", + shouldForceToolOrchestration: true, + reason: "agent_auto_composer", + }; + } + + if (params.hasCanvasControlIntentFastPath) { + return { + effectiveSelectedChatModel: selectedChatModel, + shouldForceToolOrchestration: true, + reason: "canvas_control_fast_path", + }; + } + + return { + effectiveSelectedChatModel: selectedChatModel, + shouldForceToolOrchestration, + reason: null, + }; +} + +export function resolveCanvasAgentTargetPreference(params: { + text: string; + selectedImageCount: number; + latestResultImageCount: number; + workingMemoryTargetImageCount: number; +}): CanvasAgentTargetPreference { + const hasDeicticTarget = hasCanvasAgentDeicticTarget(params.text); + const hasEditIntent = hasCanvasAgentActionableImageEditIntent(params.text); + const selectedImageCount = Math.max(0, Math.round(params.selectedImageCount || 0)); + const latestResultImageCount = Math.max( + 0, + Math.round(params.latestResultImageCount || 0), + ); + const workingMemoryTargetImageCount = Math.max( + 0, + Math.round(params.workingMemoryTargetImageCount || 0), + ); + + if (!hasDeicticTarget || !hasEditIntent || selectedImageCount > 0) { + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: false, + confidence: 0, + reason: null, + }; + } + + if (latestResultImageCount > 0) { + return { + shouldPreferLatestResult: true, + shouldPreferWorkingMemoryTarget: false, + confidence: 90, + reason: "deictic_edit_prefers_latest_result", + }; + } + + if (workingMemoryTargetImageCount > 0) { + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: true, + confidence: 86, + reason: "deictic_edit_uses_working_memory_target", + }; + } + + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: false, + confidence: 0, + reason: null, + }; +} diff --git a/lib/ai/runtime/direct-agent-execution.ts b/lib/ai/runtime/direct-agent-execution.ts index bd507cb55..79a6f6642 100644 --- a/lib/ai/runtime/direct-agent-execution.ts +++ b/lib/ai/runtime/direct-agent-execution.ts @@ -653,6 +653,7 @@ export function pickAgentSourceImageUrls(params: { params.requestTargetResolution?.targetImageUrls || [], params.requestResolvedMediaInput?.selectedImageUrls || [], params.requestResolvedMediaInput?.currentMessageImageUrls || [], + params.requestResolvedMediaInput?.canvasImageUrls || [], params.requestWorkingState?.lastResolvedTargetImageUrls || [], params.requestWorkingState?.lastGeneratedImageUrls || [], params.requestResolvedMediaInput?.historyAssistantImageUrls || [], diff --git a/lib/ai/runtime/direct-agent-response-builders.ts b/lib/ai/runtime/direct-agent-response-builders.ts index c1b349c86..0576f56d1 100644 --- a/lib/ai/runtime/direct-agent-response-builders.ts +++ b/lib/ai/runtime/direct-agent-response-builders.ts @@ -10,6 +10,56 @@ import { type DirectAgentToolCapability, } from "@/lib/ai/runtime/direct-agent-execution"; +function normalizeDirectAgentResultSentence(value: string) { + const normalized = String(value || "") + .replace(/结果如下[：:]?\s*$/u, "") + .replace(/[，,。:：]\s*$/u, "") + .trim(); + return normalized || "已完成处理"; +} + +function buildDirectAgentResultCard(params: { + result: string; + mediaMarkdown?: string | null; + followUp: string; +}) { + return [ + "### 执行结果", + `- ${normalizeDirectAgentResultSentence(params.result)}，结果如下。`, + params.mediaMarkdown, + "### 后续操作", + `- ${params.followUp}`, + ] + .filter(Boolean) + .join("\n\n"); +} + +function getDirectAgentFollowUpMessage(capability: DirectAgentToolCapability) { + switch (capability) { + case "remove_background": + return "如需微调边缘、替换背景或继续合成，可以直接告诉我。"; + case "remove_watermark": + return "如需继续修补局部痕迹或统一画面质感，可以直接告诉我。"; + case "enhance": + return "如需继续调整清晰度、色彩或细节强度，可以直接告诉我。"; + case "resize": + return "如需继续改尺寸、比例或输出规格，可以直接告诉我。"; + case "extend": + return "如需继续扩展画面、调整构图或补充边缘内容，可以直接告诉我。"; + case "angle_variant": + return "如需继续换角度、换机位或统一系列视角，可以直接告诉我。"; + case "local_edit": + case "box_cutout": + return "如需继续修改局部、替换元素或细化边缘，可以直接告诉我。"; + case "generate_video": + return "如需继续调整镜头、节奏、比例或时长，可以直接告诉我。"; + case "generate_image": + case "generic_edit": + default: + return "如需继续调整风格、构图、主体细节或尺寸，可以直接告诉我。"; + } +} + export function buildDirectAgentAnalysisAssistantText(params: { summary: string; answer?: string | null; @@ -32,12 +82,11 @@ export function buildDirectAgentImageGenerationAssistantText(params: { modelDisplayName?: string | null; }) { return prependImageModelBadge( - [ - getDirectAgentToolSuccessMessage("generate_image"), - params.imageMarkdown, - ] - .filter(Boolean) - .join("\n\n"), + buildDirectAgentResultCard({ + result: getDirectAgentToolSuccessMessage("generate_image"), + mediaMarkdown: params.imageMarkdown, + followUp: getDirectAgentFollowUpMessage("generate_image"), + }), params.modelDisplayName, ); } @@ -47,12 +96,11 @@ export function buildDirectAgentVideoGenerationAssistantText(params: { modelDisplayName?: string | null; }) { return prependImageModelBadge( - [ - getDirectAgentToolSuccessMessage("generate_video"), - buildAgentToolVideoMarkdown(params.videoUrl), - ] - .filter(Boolean) - .join("\n\n"), + buildDirectAgentResultCard({ + result: getDirectAgentToolSuccessMessage("generate_video"), + mediaMarkdown: buildAgentToolVideoMarkdown(params.videoUrl), + followUp: getDirectAgentFollowUpMessage("generate_video"), + }), params.modelDisplayName, ); } @@ -64,16 +112,15 @@ export function buildDirectAgentImageEditAssistantText(params: { imageSize?: string; }) { return prependImageModelBadge( - [ - getDirectAgentToolSuccessMessage(params.capability), - buildAgentToolImageMarkdown({ + buildDirectAgentResultCard({ + result: getDirectAgentToolSuccessMessage(params.capability), + mediaMarkdown: buildAgentToolImageMarkdown({ originalUrl: params.originalUrl, thumbnailUrl: params.thumbnailUrl, imageSize: params.imageSize, }), - ] - .filter(Boolean) - .join("\n\n"), + followUp: getDirectAgentFollowUpMessage(params.capability), + }), getDirectAgentToolLabel(params.capability), ); } @@ -87,12 +134,11 @@ export function buildDirectAgentBatchImageEditAssistantText(params: { }>; }) { return prependImageModelBadge( - [ - getDirectAgentToolSuccessMessage(params.capability), - buildAgentToolImagesMarkdown(params.results), - ] - .filter(Boolean) - .join("\n\n"), + buildDirectAgentResultCard({ + result: getDirectAgentToolSuccessMessage(params.capability), + mediaMarkdown: buildAgentToolImagesMarkdown(params.results), + followUp: getDirectAgentFollowUpMessage(params.capability), + }), getDirectAgentToolLabel(params.capability), ); } diff --git a/lib/ai/runtime/intent-arbiter.ts b/lib/ai/runtime/intent-arbiter.ts index cfd463d3b..4cb6141b0 100644 --- a/lib/ai/runtime/intent-arbiter.ts +++ b/lib/ai/runtime/intent-arbiter.ts @@ -5,6 +5,10 @@ import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options"; +import { + hasCanvasAgentActionableImageEditIntent, + inferCanvasAgentImageEditCapability, +} from "@/lib/ai/runtime/canvas-agent-kernel"; export type IntentArbiterMode = | "respond" @@ -90,10 +94,10 @@ function isExplicitImageGenerationRequest(text: string): boolean { const visualTarget = "(?:图|图片|海报|头像|插画|场景|主图|产品图|商品图|电商图|壁纸|封面|logo|图标)"; const createVisualTarget = new RegExp( - `(?:生成|创作|创建|制作|设计|做|出|画|绘制)${countToken}?张?.{0,32}${visualTarget}`, + `(?:生成|创作|创建|制作|设计|做|出|画(?!布)|绘制)${countToken}?张?.{0,32}${visualTarget}`, ); const drawSubject = new RegExp( - `(?:画|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`, + `(?:画(?!布)|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`, ); const continueVariants = new RegExp( `(?:再来|再出|再做|再给我|补|多来|继续生成|继续出)${countToken}?张`, @@ -116,23 +120,21 @@ function isExplicitImageGenerationRequest(text: string): boolean { } function isLikelyImageEditRequest(text: string): boolean { - const normalizedText = String(text || "").toLowerCase(); - return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( - normalizedText, - ); + return hasDirectAgentLocalEditIntent(text) || hasCanvasAgentActionableImageEditIntent(text); } function inferDefaultImageEditSubIntent(text: string): string { - const lowerText = String(text || "").toLowerCase(); - if (/去水印|水印|去logo|去\s*logo|移除logo|去除logo/.test(lowerText)) return "remove_watermark"; - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; - if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; - if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant"; + const inferredCapability = inferCanvasAgentImageEditCapability(text); + if ( + inferredCapability && + inferredCapability !== "generic_edit" && + inferredCapability !== "generate_image" && + inferredCapability !== "generate_video" && + inferredCapability !== "analyze" + ) { + return inferredCapability; + } if (hasDirectAgentLocalEditIntent(text)) return "local_edit"; - if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend"; - if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer"; - if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust"; - if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint"; return "enhance"; } @@ -195,6 +197,9 @@ function buildExecutionFromAnalysis(params: { targetSource: ResolvedTargetAssets["resolutionMode"]; }): IntentArbiterExecution { const mode = mapIntentToMode(params.analysis.intent, params.analysis.needsClarification); + const shouldKeepPlannedToolCalls = + params.execution?.mode === mode && + params.execution?.routeIntent === params.analysis.intent; return { mode, @@ -216,7 +221,7 @@ function buildExecutionFromAnalysis(params: { shouldStayConversational: mode === "respond" || mode === "analyze_image" || mode === "clarify", shouldPublishToCanvas: params.execution?.shouldPublishToCanvas || false, - toolCalls: params.execution?.toolCalls || [], + toolCalls: shouldKeepPlannedToolCalls ? params.execution?.toolCalls || [] : [], availableTools: params.execution?.availableTools || [], }; } @@ -260,6 +265,14 @@ function resolvePolicyIntent(params: { }; } + if (hasUsableImage && likelyEdit) { + return { + intent: "image_edit", + subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), + reason: "image_edit_policy_override", + }; + } + switch (params.actionResolution.kind) { case "image_edit_followup": return { @@ -290,13 +303,6 @@ function resolvePolicyIntent(params: { reason: "task_boundary_execution_video", }; case "execution_image": - if (hasUsableImage && likelyEdit && !explicitImageGen) { - return { - intent: "image_edit", - subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), - reason: "task_boundary_execution_image_edit", - }; - } return { intent: "image_gen", reason: "task_boundary_execution_image_generate", @@ -327,14 +333,6 @@ function resolvePolicyIntent(params: { }; } - if (hasUsableImage && likelyEdit && params.analysis.intent === "chat") { - return { - intent: "image_edit", - subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), - reason: "image_edit_policy_override", - }; - } - return { intent: params.analysis.intent, subIntent: diff --git a/lib/ai/runtime/recent-dialogue-memory.ts b/lib/ai/runtime/recent-dialogue-memory.ts new file mode 100644 index 000000000..b6b5640fd --- /dev/null +++ b/lib/ai/runtime/recent-dialogue-memory.ts @@ -0,0 +1,146 @@ +import type { DBMessage } from "@/lib/db/schema"; + +export interface RecentDialogueMemoryEntry { + role: "user" | "assistant"; + content: string; + messageId?: string; +} + +export interface RecentDialogueMemory { + entries: RecentDialogueMemoryEntry[]; + contextSummary: string; + memoryHints: string[]; + isRecallRequest: boolean; +} + +const USER_RECALL_PATTERN = + /(?:我|用户)?(?:刚才|刚刚|前面|之前|上面|上一(?:句|条|轮)|最近).{0,12}(?:说|发|问|讲|提|输入|发送).{0,12}(?:什么|啥|哪句|哪些|内容)|(?:我说了什么|我刚才说啥|我前面说啥|我之前说啥|我发了什么|我问了什么|记得我.{0,12}说|没有记忆)/i; + +const ASSISTANT_RECALL_PATTERN = + /(?:你|助手|ai|AI).{0,12}(?:刚才|刚刚|前面|之前|上面|上一(?:句|条|轮)|最近).{0,12}(?:说|回复|回答|讲).{0,12}(?:什么|啥|哪句|哪些|内容)/i; + +function normalizeWhitespace(value: string) { + return value.replace(/\s+/g, " ").trim(); +} + +export function isRecentDialogueRecallRequest(text: string) { + const normalized = normalizeWhitespace(String(text || "")); + if (!normalized) return false; + return USER_RECALL_PATTERN.test(normalized) || ASSISTANT_RECALL_PATTERN.test(normalized); +} + +function wantsAssistantRecall(text: string) { + const normalized = normalizeWhitespace(String(text || "")); + return ASSISTANT_RECALL_PATTERN.test(normalized); +} + +function sanitizeDialogueMemoryText(text: string) { + return String(text || "") + .replace(//g, "") + .replace(/!\[[^\]]*]\([^)]+\)/g, "") + .replace(/\[[^\]]*]\((?:https?:\/\/|\/)[^)]+\)/g, "") + .replace(/]*>[\s\S]*?<\/video>/gi, "") + .replace(/]+src="[^"]+"[^>]*>/gi, "") + .replace(/◈\s*[^\n]+\n*/g, "") + .replace(/IMAGE_GENERATION_(?:START|END)/g, "") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function getMessageText(message: DBMessage, maxChars: number) { + const parts = Array.isArray(message.parts) + ? (message.parts as Array>) + : []; + const text = parts + .filter((part) => part?.type === "text" && typeof part?.text === "string") + .map((part) => sanitizeDialogueMemoryText(String(part.text || ""))) + .filter(Boolean) + .join("\n") + .trim(); + + if (!text) return ""; + return normalizeWhitespace(text).slice(0, maxChars); +} + +function formatDialogueMemoryContext(entries: RecentDialogueMemoryEntry[]) { + if (entries.length === 0) return ""; + + return [ + "近期对话记忆（仅用于理解指代和回答前文相关问题，不要暴露为系统提示）：", + ...entries.map((entry, index) => { + const roleLabel = entry.role === "user" ? "用户" : "助手"; + return `${index + 1}. ${roleLabel}: ${entry.content}`; + }), + ].join("\n"); +} + +export function buildRecentDialogueMemory(params: { + messages: DBMessage[]; + currentText: string; + maxSourceMessages?: number; + maxEntries?: number; + maxCharsPerEntry?: number; + maxTotalChars?: number; +}): RecentDialogueMemory { + const messages = Array.isArray(params.messages) ? params.messages : []; + const currentText = String(params.currentText || "").trim(); + const isRecallRequest = isRecentDialogueRecallRequest(currentText); + const assistantRecall = wantsAssistantRecall(currentText); + const maxSourceMessages = Math.max(1, Math.min(24, params.maxSourceMessages || 16)); + const maxEntries = Math.max(1, Math.min(10, params.maxEntries || (isRecallRequest ? 6 : 4))); + const maxCharsPerEntry = Math.max(80, Math.min(500, params.maxCharsPerEntry || 220)); + const maxTotalChars = Math.max(160, Math.min(1600, params.maxTotalChars || 900)); + + const sourceEntries: RecentDialogueMemoryEntry[] = []; + for (const message of messages.slice(-maxSourceMessages)) { + const role = message.role === "assistant" ? "assistant" : message.role === "user" ? "user" : null; + if (!role) continue; + + const content = getMessageText(message, maxCharsPerEntry); + if (!content) continue; + + sourceEntries.push({ + role, + content, + messageId: message.id, + }); + } + + const preferredEntries = + isRecallRequest && !assistantRecall + ? sourceEntries.filter((entry) => entry.role === "user") + : isRecallRequest && assistantRecall + ? sourceEntries.filter((entry) => entry.role === "assistant") + : sourceEntries; + const fallbackEntries = + preferredEntries.length > 0 ? preferredEntries : sourceEntries; + + const selectedReversed: RecentDialogueMemoryEntry[] = []; + let totalChars = 0; + for (let index = fallbackEntries.length - 1; index >= 0; index -= 1) { + const entry = fallbackEntries[index]; + if (!entry) continue; + if (selectedReversed.length >= maxEntries) break; + if (totalChars + entry.content.length > maxTotalChars && selectedReversed.length > 0) { + continue; + } + selectedReversed.push(entry); + totalChars += entry.content.length; + } + + const entries = selectedReversed.reverse(); + const memoryHints = entries.length + ? [ + isRecallRequest + ? "用户正在询问前文或记忆，应优先根据近期对话记忆回答，不要声称没有上下文。" + : "近期对话记忆只用于理解指代和延续语境，不要逐字复述，除非用户明确询问前文。", + ] + : []; + + return { + entries, + contextSummary: formatDialogueMemoryContext(entries), + memoryHints, + isRecallRequest, + }; +} diff --git a/lib/ai/runtime/task-boundary-classifier.ts b/lib/ai/runtime/task-boundary-classifier.ts index a4dddf4a2..fe6991ed2 100644 --- a/lib/ai/runtime/task-boundary-classifier.ts +++ b/lib/ai/runtime/task-boundary-classifier.ts @@ -5,6 +5,7 @@ import type { PendingGridConfirmation, } from "@/lib/ai/conversation-action-resolver"; import { isExplicitPromptOptimizationRequest } from "@/lib/ai/smart-analyzer"; +import { isCapabilityQuestionTurnRequest } from "@/lib/ai/runtime/task-boundary-text"; import { detectStoryboardCommandIntent } from "@/lib/video-agent/storyboard-command-interpreter"; export type TaskBoundaryType = @@ -88,9 +89,7 @@ function isLikelyChatOnlyRequest(text: string): boolean { const normalized = normalizeCompactText(text); if (!normalized) return false; - return /你能做什么|你会什么|你可以做什么|你能帮我做什么|你能干什么|你可以帮我做什么|介绍一下你自己|介绍一下自己|说说你的能力|你有哪些功能|你有什么功能|怎么用|怎么玩|如何使用|帮助说明|帮助手册|功能介绍/.test( - normalized, - ); + return isCapabilityQuestionTurnRequest(normalized); } function isLikelyAnalysisOnlyRequest(text: string, hasImageContext: boolean): boolean { diff --git a/lib/ai/runtime/task-boundary-text.ts b/lib/ai/runtime/task-boundary-text.ts new file mode 100644 index 000000000..dad761f7f --- /dev/null +++ b/lib/ai/runtime/task-boundary-text.ts @@ -0,0 +1,12 @@ +export function normalizeCompactTaskBoundaryText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +export function isCapabilityQuestionTurnRequest(text: string): boolean { + const normalized = normalizeCompactTaskBoundaryText(text); + if (!normalized) return false; + + return /你.*(?:能|会|可以|可|嫩)?.*(?:为我做什么|帮我做什么|帮我什么|做什么|干什么)|你.*(?:有哪些|有什么).*(?:功能|能力)|(?:介绍|说说).*(?:功能|能力|你自己|自己)|怎么用|怎么玩|如何使用|帮助说明|帮助手册|功能介绍/.test( + normalized, + ); +} diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 3bde4b6ee..c22451481 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -33,7 +33,7 @@ export interface AnalysisResult { function inferDefaultImageEditSubIntent(text: string): AnalysisResult["subIntent"] { const lowerText = text.toLowerCase(); if (/去水印|水印|去logo|去\s*logo|移除logo|去除logo/.test(lowerText)) return "remove_watermark"; - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; + if (/抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景/.test(lowerText)) return "remove_background"; if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant"; if (hasDirectAgentLocalEditIntent(text)) return "local_edit"; @@ -112,10 +112,10 @@ export function isExplicitImageGenerationRequest(text: string): boolean { const visualTarget = "(?:图|图片|海报|头像|插画|场景|主图|产品图|商品图|电商图|壁纸|封面|logo|图标)"; const createVisualTarget = new RegExp( - `(?:生成|创作|创建|制作|设计|做|出|画|绘制)${countToken}?张?.{0,32}${visualTarget}`, + `(?:生成|创作|创建|制作|设计|做|出|画(?!布)|绘制)${countToken}?张?.{0,32}${visualTarget}`, ); const drawSubject = new RegExp( - `(?:画|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`, + `(?:画(?!布)|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`, ); const continueVariants = new RegExp( `(?:再来|再出|再做|再给我|补|多来|继续生成|继续出)${countToken}?张`, @@ -141,7 +141,7 @@ export function isExplicitImageGenerationRequest(text: string): boolean { function isLikelyImageEditRequest(text: string): boolean { const normalizedText = text.toLowerCase(); - return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( + return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( normalizedText, ); } @@ -2331,7 +2331,7 @@ export function fallbackKeywordAnalysis( } // 抠图/去背景 - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) { + if (/抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景/.test(lowerText)) { return { intent: "image_edit", subIntent: "remove_background", diff --git a/lib/ai/specialists/index.ts b/lib/ai/specialists/index.ts index f91793f61..75827ad67 100644 --- a/lib/ai/specialists/index.ts +++ b/lib/ai/specialists/index.ts @@ -250,7 +250,7 @@ function normalizeEditCapability( subIntent: "remove_watermark", }; } - if (subIntent === "remove_background" || /抠图|去背|去背景|透明底/.test(normalizedText)) { + if (subIntent === "remove_background" || /抠图|去背|去背景|透明底|抠出|扣出|抠出来|扣出来/.test(normalizedText)) { return { capability: "remove_background", toolName: "removeBackground", diff --git a/lib/canvas/local-upload-state.ts b/lib/canvas/local-upload-state.ts index 7845f4766..0f1cf4b6c 100644 --- a/lib/canvas/local-upload-state.ts +++ b/lib/canvas/local-upload-state.ts @@ -29,6 +29,13 @@ export function isCanvasImageReadyForAi(node: CanvasNode | null | undefined) { return false; } - const src = String(((node.data as Record)?.src || "")).trim(); + const data = (node.data || {}) as Record; + const src = String( + data.src || + data.previewImage || + data.imageUrl || + data.renderArtifacts?.screenshotUrl || + "" + ).trim(); return /^https?:\/\//i.test(src); } diff --git a/lib/db/queries.ts b/lib/db/queries.ts index d61dbebae..fd031aea9 100644 --- a/lib/db/queries.ts +++ b/lib/db/queries.ts @@ -186,6 +186,44 @@ export async function saveChat({ } } +export async function getClawdChatsByUserId({ + userId, + limit = 50, +}: { + userId: string; + limit?: number; +}) { + const safeLimit = Math.max(1, Math.min(100, Math.round(limit) || 50)); + const latestMessageAt = sql`max(${message.createdAt})`; + + try { + return await db + .select({ + id: chat.id, + title: chat.title, + createdAt: chat.createdAt, + updatedAt: latestMessageAt, + messageCount: sql`count(${message.id})::int`, + }) + .from(chat) + .innerJoin(message, eq(message.chatId, chat.id)) + .where( + and( + eq(chat.userId, userId), + sql`coalesce(${message.videoMetadata}->>'composerMode', '') = 'clawd'` + ) + ) + .groupBy(chat.id, chat.title, chat.createdAt) + .orderBy(desc(latestMessageAt)) + .limit(safeLimit); + } catch (_error) { + throw new ChatSDKError( + "bad_request:database", + "Failed to get Claw chats by user id" + ); + } +} + export async function deleteChatById({ id }: { id: string }) { try { await db.delete(vote).where(eq(vote.chatId, id)); diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index d51b0caec..ee23944a0 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -4,12 +4,15 @@ import { extractAssistantImageUrls, extractVideoUrls, normalizeMediaUrl, + sortCanvasMessagesByCreatedAt, stripMediaMarkup, } from "@/components/canvas/chat-panel/canvas-chat-media"; import { filterCanvasMediaPayloadForUnhandledItems, + buildAgentProgressNodeSteps, getCanvasMediaPayloadHandledItemIds, getCanvasOperationHandledMediaItemIds, + getAgentProgressStepLabel, isCanvasAgentViewOperationClientEnabled, normalizeCanvasAgentClientBoolean, shouldRetryCanvasViewOperationAfterRefresh, @@ -44,7 +47,18 @@ import { } from "@/lib/canvas-toolbar-config"; import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification"; import { hasCanvasAgentRoutingContext } from "@/lib/ai/canvas-agent-routing"; +import { + inferCanvasAgentImageEditCapability, + resolveCanvasAgentRequestPolicy, + resolveCanvasAgentTargetPreference, +} from "@/lib/ai/runtime/canvas-agent-kernel"; +import { isExplicitImageGenerationRequest } from "@/lib/ai/smart-analyzer"; import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent"; +import { + buildRecentDialogueMemory, + isRecentDialogueRecallRequest, +} from "@/lib/ai/runtime/recent-dialogue-memory"; +import { isCapabilityQuestionTurnRequest } from "@/lib/ai/runtime/task-boundary-text"; import { isAgentModeScopedChatId, isScopedEmptyHistoryRequest, @@ -127,9 +141,14 @@ import { getDirectAgentToolSuccessMessage, hasExplicitDirectAgentBatchImageEditIntent, isDirectAgentBatchImageEditCapability, + resolveDirectAgentToolContext, resolveDirectAgentToolExecution, shouldRunDirectAgentBatchImageEdit, } from "@/lib/ai/runtime/direct-agent-execution"; +import { + buildDirectAgentImageEditAssistantText, + buildDirectAgentImageGenerationAssistantText, +} from "@/lib/ai/runtime/direct-agent-response-builders"; import { inferAngleEditOptionsFromText } from "@/lib/ai/runtime/angle-edit-options"; import { hasDirectAgentLocalEditIntent, @@ -209,8 +228,69 @@ import { import type { PickedObjectAttachment } from "@/components/canvas/chat-panel/canvas-chat-session"; import type { CanvasAgentContext } from "@/types/canvas/agent-protocol"; import type { ChatMessage } from "@/lib/types"; +import type { DBMessage } from "@/lib/db/schema"; + +function dbTextMessage( + id: string, + role: "user" | "assistant", + text: string, +): DBMessage { + return { + id, + role, + parts: [{ type: "text", text }], + } as unknown as DBMessage; +} test.describe("canvas chat pure helpers", () => { + test("keeps optimistic messages without timestamps after persisted history", () => { + const messages = [ + { + id: "optimistic-latest-user", + role: "user", + parts: [{ type: "text", text: "你能为我做什么？列出表给我" }], + }, + { + id: "persisted-first-user", + role: "user", + createdAt: "2026-07-07T01:00:00.000Z", + parts: [{ type: "text", text: "你好" }], + }, + { + id: "persisted-first-assistant", + role: "assistant", + createdAt: "2026-07-07T01:00:01.000Z", + parts: [{ type: "text", text: "你好，我在。" }], + }, + ] as ChatMessage[]; + + expect(sortCanvasMessagesByCreatedAt(messages).map((message) => message.id)).toEqual([ + "persisted-first-user", + "persisted-first-assistant", + "optimistic-latest-user", + ]); + }); + + test("preserves relative order among messages without timestamps", () => { + const messages = [ + { + id: "pending-user", + role: "user", + parts: [{ type: "text", text: "生成一张图" }], + }, + { + id: "pending-assistant", + role: "assistant", + parts: [{ type: "text", text: "分析中" }], + }, + ] as ChatMessage[]; + + expect(sortCanvasMessagesByCreatedAt(messages).map((message) => message.id)).toEqual([ + "pending-user", + "pending-assistant", + ]); + }); + test("formats uploaded spreadsheet rows as safe markdown tables for OpenClaw", () => { const markdown = rowsToOpenClawMarkdownTable([ ["名称", "备注"], @@ -249,6 +329,52 @@ test.describe("canvas chat pure helpers", () => { expect(upstream).toContain("| SKU | 销量 |"); }); + test("recognizes canvas product capability questions with typo tolerance", () => { + expect(isCapabilityQuestionTurnRequest("你能为我做什么？")).toBe(true); + expect(isCapabilityQuestionTurnRequest("你嫩为我做什么？")).toBe(true); + expect(isCapabilityQuestionTurnRequest("你有哪些功能")).toBe(true); + expect(isCapabilityQuestionTurnRequest("帮我把这张图换成白底")).toBe(false); + }); + + test("builds bounded recent dialogue memory for user recall requests", () => { + const memory = buildRecentDialogueMemory({ + messages: [ + dbTextMessage("m1", "user", "你好"), + dbTextMessage("m2", "assistant", "你好，我在。"), + dbTextMessage("m3", "user", "你是一个小可爱"), + dbTextMessage("m4", "assistant", "嘿嘿，谢谢你的夸奖。"), + ], + currentText: "我前面说了什么？", + }); + + expect(isRecentDialogueRecallRequest("我前面说了什么？")).toBe(true); + expect(memory.isRecallRequest).toBe(true); + expect(memory.entries.map((entry) => entry.role)).toEqual(["user", "user"]); + expect(memory.entries.at(-1)?.content).toBe("你是一个小可爱"); + expect(memory.contextSummary).toContain("用户: 你是一个小可爱"); + expect(memory.memoryHints.join(" ")).toContain("不要声称没有上下文"); + }); + + test("sanitizes media markup from recent dialogue memory", () => { + const memory = buildRecentDialogueMemory({ + messages: [ + dbTextMessage("m1", "user", "生成一只小猫"), + dbTextMessage( + "m2", + "assistant", + "\n![原图](https://example.com/cat.png)\n已完成本次生成" + ), + ], + currentText: "你刚才回复了什么？", + }); + + expect(memory.entries).toHaveLength(1); + expect(memory.entries[0].role).toBe("assistant"); + expect(memory.entries[0].content).toBe("已完成本次生成"); + expect(memory.contextSummary).not.toContain("https://example.com"); + expect(memory.contextSummary).not.toContain("IMAGE_GENERATION_START"); + }); + test("infers explicit agent image counts from user text", () => { expect(inferAgentImageCountFromText("再出三版")).toBe(3); expect(inferAgentImageCountFromText("基于这张图再来 3 张")).toBe(3); @@ -1130,6 +1256,273 @@ test.describe("canvas chat pure helpers", () => { ); }); + test("formats direct agent image results as structured markdown cards", () => { + const generatedText = buildDirectAgentImageGenerationAssistantText({ + imageMarkdown: "![生成结果](https://cdn.example.com/generated.png)", + modelDisplayName: "Test Image Model", + }); + + expect(generatedText).toContain("### 执行结果"); + expect(generatedText).toContain("- 已完成图片生成，结果如下。"); + expect(generatedText).toContain("![生成结果](https://cdn.example.com/generated.png)"); + expect(generatedText).toContain("### 后续操作"); + + const editText = buildDirectAgentImageEditAssistantText({ + capability: "remove_background", + originalUrl: "https://cdn.example.com/cutout.png", + }); + + expect(editText).toContain("### 执行结果"); + expect(editText).toContain("- 已完成抠图处理，结果如下。"); + expect(editText).toContain("https://cdn.example.com/cutout.png"); + expect(editText).toContain("替换背景"); + }); + + test("builds capability-aware agent progress node labels", () => { + const selectingSteps = buildAgentProgressNodeSteps({ + stage: "planning", + currentStep: "select_tool", + capability: "remove_background", + }); + + expect(selectingSteps.map((step) => step.label)).toEqual([ + "理解用户需求", + "分析使用工具", + "确认抠图工具", + "抠图中", + "整理结果", + ]); + expect(selectingSteps.map((step) => step.state)).toEqual([ + "completed", + "completed", + "current", + "upcoming", + "upcoming", + ]); + + const executingSteps = buildAgentProgressNodeSteps({ + stage: "executing", + currentStep: "execute", + capability: "remove_background", + }); + + expect(executingSteps[2]?.state).toBe("completed"); + expect(executingSteps[3]).toMatchObject({ + label: "抠图中", + state: "current", + }); + expect(getAgentProgressStepLabel("execute", { capability: "generate_video" })).toBe( + "视频生成中" + ); + }); + + test("resolves deictic canvas edit requests to latest results and direct tools", () => { + const latestResultUrl = "https://cdn.example.com/latest-cat-result.png"; + const requestPolicy = resolveCanvasAgentRequestPolicy({ + composerMode: "agent", + selectedChatModel: "conversation-model-id", + }); + + expect(requestPolicy.effectiveSelectedChatModel).toBe("auto"); + expect(requestPolicy.shouldForceToolOrchestration).toBe(true); + + const targetPreference = resolveCanvasAgentTargetPreference({ + text: "将这只猫从图中扣图扣出来", + selectedImageCount: 0, + latestResultImageCount: 1, + workingMemoryTargetImageCount: 1, + }); + + expect(targetPreference.shouldPreferLatestResult).toBe(true); + + const mediaInput = { + imageRefs: [], + currentMessageImageRefs: [], + selectedImageRefs: [], + canvasImageRefs: [], + historyUserImageRefs: [], + historyAssistantImageRefs: [ + { + url: latestResultUrl, + normalizedUrl: latestResultUrl, + source: "history_assistant_result", + role: "history_result", + }, + ], + resolvedImageRefs: [], + currentMessageImageUrls: [], + selectedImageUrls: [], + historyUserImageUrls: [], + historyAssistantImageUrls: [latestResultUrl], + resolvedImageUrls: [latestResultUrl], + selectedAssetIds: [], + candidateAssetIds: [], + uploadedAssetIds: [], + hasImage: true, + source: "history", + fromHistory: true, + } as any; + const workingState = { + lastGeneratedImageUrls: [latestResultUrl], + lastResolvedTargetImageUrls: [latestResultUrl], + }; + const targetResolution = { + targetRefs: [], + candidateRefs: [], + targetImageUrls: [latestResultUrl], + targetAssetIds: [], + candidateImageUrls: [], + candidateAssetIds: [], + resolutionMode: "working_memory", + confidence: targetPreference.confidence, + needsClarification: false, + reason: targetPreference.reason, + fromHistory: true, + } as any; + + const directContext = resolveDirectAgentToolContext({ + execution: { + mode: "edit_image", + routeIntent: "image_edit", + routeSubIntent: "remove_background", + targetSource: targetResolution.resolutionMode, + preferredCapability: "remove_background", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: ["removeBackground"], + } as any, + requestResolvedMediaInput: mediaInput, + requestTargetResolution: targetResolution, + requestWorkingState: workingState, + }); + + expect(directContext?.execution.toolName).toBe("removeBackground"); + expect(directContext?.targetImageUrl).toBe(latestResultUrl); + }); + + test("falls back to canvas images for direct agent edit tools", () => { + const canvasImageUrl = "https://cdn.example.com/canvas-cat.png"; + const mediaInput = { + imageRefs: [], + currentMessageImageRefs: [], + selectedImageRefs: [], + canvasImageRefs: [ + { + url: canvasImageUrl, + normalizedUrl: canvasImageUrl, + nodeId: "cat-node", + source: "canvas_context", + role: "reference", + x: 120, + y: 80, + width: 640, + height: 640, + isSelected: false, + }, + ], + historyUserImageRefs: [], + historyAssistantImageRefs: [], + resolvedImageRefs: [], + currentMessageImageUrls: [], + selectedImageUrls: [], + canvasImageUrls: [canvasImageUrl], + historyUserImageUrls: [], + historyAssistantImageUrls: [], + resolvedImageUrls: [], + selectedAssetIds: [], + candidateAssetIds: [], + uploadedAssetIds: [], + hasImage: false, + source: "none", + fromHistory: false, + } as any; + + const directContext = resolveDirectAgentToolContext({ + execution: { + mode: "edit_image", + routeIntent: "image_edit", + routeSubIntent: "remove_background", + targetSource: "text_reference", + preferredCapability: "remove_background", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: ["removeBackground"], + } as any, + requestResolvedMediaInput: mediaInput, + }); + + expect(directContext?.execution.toolName).toBe("removeBackground"); + expect(directContext?.targetImageUrl).toBe(canvasImageUrl); + }); + + test("routes canvas subject cutout wording to remove background instead of image generation", () => { + const canvasImageUrl = "https://cdn.example.com/canvas-dog.png"; + const userText = "帮我将画布中的狗的图扣出来"; + + expect(isExplicitImageGenerationRequest(userText)).toBe(false); + expect(inferCanvasAgentImageEditCapability(userText)).toBe("remove_background"); + + const directContext = resolveDirectAgentToolContext({ + execution: { + mode: "edit_image", + routeIntent: "image_edit", + routeSubIntent: "remove_background", + targetSource: "text_reference", + preferredCapability: "remove_background", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: ["resolveTargetImage", "removeBackground"], + } as any, + requestResolvedMediaInput: { + imageRefs: [], + currentMessageImageRefs: [], + selectedImageRefs: [], + canvasImageRefs: [ + { + url: canvasImageUrl, + normalizedUrl: canvasImageUrl, + nodeId: "dog-node", + source: "canvas_context", + role: "reference", + }, + ], + historyUserImageRefs: [], + historyAssistantImageRefs: [], + resolvedImageRefs: [], + currentMessageImageUrls: [], + selectedImageUrls: [], + canvasImageUrls: [canvasImageUrl], + historyUserImageUrls: [], + historyAssistantImageUrls: [], + resolvedImageUrls: [], + selectedAssetIds: [], + candidateAssetIds: [], + uploadedAssetIds: [], + hasImage: true, + source: "canvas_context", + fromHistory: false, + } as any, + }); + + expect(directContext?.execution.toolName).toBe("removeBackground"); + expect(directContext?.targetImageUrl).toBe(canvasImageUrl); + }); + test("only runs direct image edit batches for explicit safe batch requests", () => { expect( shouldRunDirectAgentBatchImageEdit({