diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 5853e8ccc..c0b99dd5e 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -49,10 +49,19 @@ import { getAiProviderById, checkUserImageQuota, consumeUserPoints, + createMessageTargetResolution, + getConversationWorkingStateByChatId, + upsertConversationWorkingState, } from "@/lib/db/queries"; import { db } from "@/lib/db"; import type { DBMessage } from "@/lib/db/schema"; import { generatedImage, message as messageTable } from "@/lib/db/schema"; +import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input"; +import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution"; +import type { + ResolvedChatMediaInput, + ResolvedTargetAssets, +} from "@/lib/ai/context-memory/types"; import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; import { uploadFile } from "@/lib/storage/service"; @@ -1626,6 +1635,7 @@ async function saveGeneratedImages( const matches = [...imageMarkdown.matchAll(imageRegex)]; let savedImageCount = 0; + const savedOriginalUrls: string[] = []; for (const match of matches) { const altText = match[1] || ""; @@ -1644,12 +1654,30 @@ async function saveGeneratedImages( modelId, }); savedImageCount += 1; + if (!savedOriginalUrls.includes(originalUrl)) { + savedOriginalUrls.push(originalUrl); + } console.log(`[Gallery] Saved image: ${originalUrl.substring(0, 80)}...`); } catch (error) { console.error("[Gallery] Failed to save image record:", error); } } + if (savedOriginalUrls.length > 0) { + try { + await upsertConversationWorkingState({ + userId, + chatId, + currentFocusImageUrls: savedOriginalUrls, + lastGeneratedImageUrls: savedOriginalUrls, + lastResolvedTargetImageUrls: savedOriginalUrls, + lastMessageId: messageId, + }); + } catch (memoryError) { + console.warn("[Gallery] Failed to update working state with generated images:", memoryError); + } + } + if (savedImageCount <= 0) return; try { @@ -1882,6 +1910,7 @@ export async function POST(request: Request) { selectedVisibilityType, hiddenPrompt, autoContext, + contextPayload, deepThinkingEnabled, }: { id: string; @@ -1890,6 +1919,7 @@ export async function POST(request: Request) { selectedVisibilityType: VisibilityType; hiddenPrompt?: string; autoContext?: string; + contextPayload?: PostRequestBody["contextPayload"]; deepThinkingEnabled?: boolean; } = requestBody; const requestStartedAt = Date.now(); @@ -1900,6 +1930,12 @@ export async function POST(request: Request) { return ""; } })(); + const requestTextContent = message.parts + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join(" ") + .trim(); + const autoContextText = typeof autoContext === "string" ? autoContext.trim() : ""; const now = Date.now(); for (const [key, startedAt] of inFlightChatRequests.entries()) { @@ -1941,6 +1977,11 @@ export async function POST(request: Request) { let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张） let smartAutoNeedsClarification = false; // 是否需要先追问用户 let smartAutoClarificationQuestion: string | undefined; + let requestResolvedMediaInput: ResolvedChatMediaInput | null = null; + let requestTargetResolution: ResolvedTargetAssets | null = null; + let requestWorkingState: + | Awaited> + | null = null; let preferredAgentModelId: string | undefined; const deepThinkingOn = deepThinkingEnabled !== false; let shouldRunAgentConversation = finalModelId === "auto"; @@ -2032,126 +2073,62 @@ export async function POST(request: Request) { // 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb） const smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id }); console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB"); - - // 1. 提取用户消息内容 - const lastUserMessage = message; - const textContent = lastUserMessage.parts - .filter((p): p is { type: "text"; text: string } => p.type === "text") - .map((p) => p.text) - .join(" "); - autoTextForFallback = textContent; - const autoContextText = typeof autoContext === "string" ? autoContext.trim() : ""; - const hasSelectedImageHint = /用户已选中\s*\d+\s*张图片/.test(autoContextText); - const hasExplicitTargetHint = /第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(textContent); - const hasActionableEditIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test( - textContent - ); - const currentMessageImageInputs = collectImageInputsFromMessageParts( - lastUserMessage.parts as any[], - requestOrigin, - ).slice(0, 4); - let imageUrls = currentMessageImageInputs; - let hasImage = imageUrls.length > 0; - autoCurrentImageUrlsForFallback = imageUrls.slice(); - - // 🆕 历史图片上下文感知：如果用户没有发送新图片，自动使用历史中最近的图片（支持多张） - // 逻辑：只要用户在对话中没有附上新图片，就默认继续处理之前的图片 - if (imageUrls.length === 0 && smartAutoHistoryMessagesFromDb.length > 0) { - console.log("[Smart Auto Mode] 🔍 No image in current message, auto-searching history..."); - - const explicitFollowLatestResult = /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果)/i.test( - textContent - ); - const explicitUseOriginalUpload = /(原图|上传图|参考图|最初那张|第一张图|素材图)/i.test( - textContent - ); - - const latestAssistantImages: string[] = []; - const latestUserImages: string[] = []; - const pushUniqueUrl = (bucket: string[], maybeUrl?: string) => { - if (!maybeUrl || typeof maybeUrl !== "string") return; - const normalized = maybeUrl.trim(); - if (!/^https?:\/\//i.test(normalized)) return; - if (!bucket.includes(normalized)) { - bucket.push(normalized); - } - }; - - // 从历史消息中倒序查找最近的“AI生成图”和“用户上传图” - for (let i = smartAutoHistoryMessagesFromDb.length - 1; i >= 0; i--) { - const msg = smartAutoHistoryMessagesFromDb[i]; - const parts = msg.parts as any[]; - - if (!parts) continue; - - if (msg.role === "assistant" && latestAssistantImages.length === 0) { - for (const part of parts) { - if (part.type === "text" && part.text) { - const parsedImageUrls = extractOriginalImageUrlsFromText(part.text); - for (const parsedUrl of parsedImageUrls) { - pushUniqueUrl(latestAssistantImages, parsedUrl); - } - } - } - } - - if (msg.role === "user" && latestUserImages.length === 0) { - for (const part of parts) { - if ((part.type === "file" || part.type === "image") && part.url) { - pushUniqueUrl(latestUserImages, part.url); - } - if (part.type === "image_url" && part.image_url?.url) { - pushUniqueUrl(latestUserImages, part.image_url.url); - } - if (part.type === "text" && part.text) { - const parsedImageUrls = extractOriginalImageUrlsFromText(part.text); - for (const parsedUrl of parsedImageUrls) { - pushUniqueUrl(latestUserImages, parsedUrl); - } - } - } - } - - if (latestAssistantImages.length > 0 && latestUserImages.length > 0) { - break; - } - } - - if (explicitUseOriginalUpload && latestUserImages.length > 0) { - smartAutoHistoryImages = latestUserImages.slice(0, 4); - console.log( - "[Smart Auto Mode] 🖼️ Using latest user-uploaded image(s) by explicit instruction:", - smartAutoHistoryImages.length - ); - } else if (latestAssistantImages.length > 0) { - smartAutoHistoryImages = latestAssistantImages.slice(0, 4); - console.log( - explicitFollowLatestResult - ? "[Smart Auto Mode] 🖼️ Using latest generated image(s) by explicit instruction:" - : "[Smart Auto Mode] 🖼️ Using latest generated image(s) for continuous edit:", - smartAutoHistoryImages.length - ); - } else if (latestUserImages.length > 0) { - smartAutoHistoryImages = latestUserImages.slice(0, 4); - console.log( - "[Smart Auto Mode] 🖼️ Fallback to latest user-uploaded image(s):", - smartAutoHistoryImages.length - ); - } - - if (smartAutoHistoryImages.length > 0) { - hasImage = true; - imageUrls = smartAutoHistoryImages; - autoFromHistoryForFallback = true; - autoCurrentImageUrlsForFallback = imageUrls.slice(); - console.log("[Smart Auto Mode] ✅ Auto-using", smartAutoHistoryImages.length, "history image(s) for context continuity"); - } else { - console.log("[Smart Auto Mode] ℹ️ No image found in history, proceeding as pure text"); - } + autoTextForFallback = requestTextContent; + if (!requestWorkingState) { + requestWorkingState = await getConversationWorkingStateByChatId({ + chatId: id, + }); + } + if (!requestResolvedMediaInput) { + requestResolvedMediaInput = resolveChatMediaInput({ + messageParts: message.parts as any[], + historyMessages: smartAutoHistoryMessagesFromDb, + contextPayload: contextPayload || undefined, + requestOrigin, + maxImages: 4, + }); + } + if (!requestTargetResolution) { + requestTargetResolution = resolveTargetAssets({ + text: requestTextContent, + mediaInput: requestResolvedMediaInput, + contextPayload: contextPayload || undefined, + workingState: requestWorkingState || undefined, + }); } - console.log("[Smart Auto Mode] Input:", { textContent: textContent.substring(0, 100), hasImage, imageCount: imageUrls.length, fromHistory: smartAutoHistoryImages.length > 0 }); + const hasSelectedImageHint = + Boolean(contextPayload?.selectedNodeIds?.length) || + /用户已选中\s*\d+\s*张图片/.test(autoContextText); + const hasExplicitTargetHint = + requestTargetResolution.resolutionMode === "text_reference" || + /第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(requestTextContent); + const hasActionableEditIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test( + requestTextContent + ); + + const currentMessageImageInputs = requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4); + let imageUrls = + requestTargetResolution.targetImageUrls.length > 0 + ? requestTargetResolution.targetImageUrls.slice(0, 4) + : requestResolvedMediaInput.resolvedImageUrls.slice(0, 4); + let hasImage = imageUrls.length > 0; + autoCurrentImageUrlsForFallback = + currentMessageImageInputs.length > 0 ? currentMessageImageInputs.slice() : imageUrls.slice(); + smartAutoHistoryImages = + currentMessageImageInputs.length === 0 ? imageUrls.slice(0, 4) : []; + autoFromHistoryForFallback = + currentMessageImageInputs.length === 0 && imageUrls.length > 0; + + console.log("[Smart Auto Mode] Input:", { + textContent: requestTextContent.substring(0, 100), + hasImage, + imageCount: imageUrls.length, + fromHistory: smartAutoHistoryImages.length > 0, + resolutionMode: requestTargetResolution.resolutionMode, + resolutionConfidence: requestTargetResolution.confidence, + }); // 🆕 从历史消息中提取对话文本（用于上下文理解） const historyMessages: { role: "user" | "assistant"; content: string }[] = []; @@ -2186,7 +2163,7 @@ export async function POST(request: Request) { } const autoGraphResult = await withOperationTimeout( runAutoAgentGraph({ - text: textContent, + text: requestTextContent, hasImage, imageUrls, imageFromHistory: smartAutoHistoryImages.length > 0, @@ -2218,8 +2195,8 @@ export async function POST(request: Request) { ) { smartAutoNeedsClarification = false; smartAutoClarificationQuestion = undefined; - if (!smartAutoOptimizedPrompt && textContent.trim()) { - smartAutoOptimizedPrompt = textContent.trim(); + if (!smartAutoOptimizedPrompt && requestTextContent.trim()) { + smartAutoOptimizedPrompt = requestTextContent.trim(); } if ( !smartAutoFriendlyMessage || @@ -2280,7 +2257,7 @@ export async function POST(request: Request) { if (autoGraphResult.optimizedPrompt) { smartAutoOptimizedPrompt = autoGraphResult.optimizedPrompt; console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", { - original: textContent.substring(0, 50), + original: requestTextContent.substring(0, 50), optimized: autoGraphResult.optimizedPrompt.substring(0, 100), }); } @@ -2311,7 +2288,7 @@ export async function POST(request: Request) { smartAutoVideoPlan = undefined; smartAutoRecommendedVideoParams = undefined; if (smartAutoPromptFromVideoPlan) { - smartAutoOptimizedPrompt = textContent.trim() || undefined; + smartAutoOptimizedPrompt = requestTextContent.trim() || undefined; smartAutoPromptFromVideoPlan = false; } smartAutoFriendlyMessage = @@ -2821,6 +2798,36 @@ export async function POST(request: Request) { messagesFromDb = await getMessagesByChatId({ id }); } + if (!requestWorkingState) { + requestWorkingState = await getConversationWorkingStateByChatId({ + chatId: id, + }); + } + if (!requestResolvedMediaInput) { + requestResolvedMediaInput = resolveChatMediaInput({ + messageParts: message.parts as any[], + historyMessages: messagesFromDb, + contextPayload: contextPayload || undefined, + requestOrigin, + maxImages: 4, + }); + } + if (!requestTargetResolution) { + requestTargetResolution = resolveTargetAssets({ + text: requestTextContent, + mediaInput: requestResolvedMediaInput, + contextPayload: contextPayload || undefined, + workingState: requestWorkingState || undefined, + }); + } + if ( + requestResolvedMediaInput.currentMessageImageUrls.length === 0 && + smartAutoHistoryImages.length === 0 && + requestTargetResolution.targetImageUrls.length > 0 + ) { + smartAutoHistoryImages = requestTargetResolution.targetImageUrls.slice(0, 4); + } + const uiMessages = [...convertToUIMessages(messagesFromDb), message]; const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : ""; const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : ""; @@ -2874,6 +2881,76 @@ export async function POST(request: Request) { return createSseResponse(duplicateStream); } + if (requestResolvedMediaInput && requestTargetResolution) { + try { + await createMessageTargetResolution({ + userId: session.user.id, + chatId: id, + messageId: message.id, + projectId: contextPayload?.projectId || null, + rawText: requestTextContent, + uploadedAssetIds: requestResolvedMediaInput.uploadedAssetIds, + uploadedImageUrls: requestResolvedMediaInput.currentMessageImageUrls, + selectedAssetIds: requestResolvedMediaInput.selectedAssetIds, + selectedImageUrls: requestResolvedMediaInput.selectedImageUrls, + candidateAssetIds: requestResolvedMediaInput.candidateAssetIds, + candidateImageUrls: requestResolvedMediaInput.imageRefs.map((ref) => ref.url), + resolvedAssetIds: requestTargetResolution.targetAssetIds, + resolvedImageUrls: requestTargetResolution.targetImageUrls, + resolutionMode: requestTargetResolution.resolutionMode, + confidence: requestTargetResolution.confidence, + needsClarification: requestTargetResolution.needsClarification, + reasoning: { + reason: requestTargetResolution.reason, + fromHistory: requestTargetResolution.fromHistory, + focusPolicy: contextPayload?.focusPolicy || null, + contextSource: contextPayload?.source || null, + }, + }); + + await upsertConversationWorkingState({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + currentIntent: + smartAutoDetectedIntent || + (requestResolvedMediaInput.hasImage + ? "image_edit" + : requestBody.videoParams + ? "video_gen" + : null), + currentFocusAssetIds: requestTargetResolution.targetAssetIds, + currentFocusImageUrls: requestTargetResolution.targetImageUrls, + selectedAssetIds: requestResolvedMediaInput.selectedAssetIds, + selectedImageUrls: requestResolvedMediaInput.selectedImageUrls, + lastUploadedAssetIds: requestResolvedMediaInput.uploadedAssetIds, + lastUploadedImageUrls: requestResolvedMediaInput.currentMessageImageUrls, + lastReferencedAssetIds: requestTargetResolution.candidateAssetIds, + lastReferencedImageUrls: requestTargetResolution.candidateImageUrls, + lastResolvedTargetAssetIds: requestTargetResolution.targetAssetIds, + lastResolvedTargetImageUrls: requestTargetResolution.targetImageUrls, + activeCanvasNodeIds: contextPayload?.selectedNodeIds || [], + lastCanvasSnapshot: + Array.isArray(contextPayload?.canvasImageNodes) && contextPayload?.canvasImageNodes.length > 0 + ? { + source: contextPayload?.source || "canvas", + selectedNodeIds: contextPayload?.selectedNodeIds || [], + canvasImageNodes: contextPayload?.canvasImageNodes, + } + : null, + lastTurnSummary: { + text: requestTextContent.slice(0, 500), + resolutionMode: requestTargetResolution.resolutionMode, + confidence: requestTargetResolution.confidence, + }, + lastMessageId: message.id, + incrementTurnCount: true, + }); + } catch (memoryError) { + console.warn("[Chat API] Failed to persist context-memory state:", memoryError); + } + } + const streamId = generateUUID(); await createStreamId({ streamId, chatId: id }); diff --git a/app/(chat)/api/chat/schema.ts b/app/(chat)/api/chat/schema.ts index deb22f6dd..f340e4363 100644 --- a/app/(chat)/api/chat/schema.ts +++ b/app/(chat)/api/chat/schema.ts @@ -43,6 +43,33 @@ const imageParamsSchema = z.object({ n: z.number().min(1).max(10).optional(), // 兼容 n 参数 }).optional(); +const contextPayloadSchema = z + .object({ + source: z.enum(["chat", "canvas", "realtime-canvas"]).optional(), + projectId: z.string().uuid().optional(), + selectedNodeIds: z.array(z.string().min(1).max(128)).max(20).optional(), + selectedAssetIds: z.array(z.string().uuid()).max(20).optional(), + focusPolicy: z.enum(["auto", "selected", "latest_result", "original"]).optional(), + canvasImageNodes: z + .array( + z.object({ + nodeId: z.string().min(1).max(128), + assetId: z.string().uuid().optional(), + url: z.string().url(), + title: z.string().max(256).optional(), + prompt: z.string().max(2000).optional(), + x: z.number(), + y: z.number(), + width: z.number().optional(), + height: z.number().optional(), + isSelected: z.boolean().optional(), + }), + ) + .max(20) + .optional(), + }) + .optional(); + export const postRequestBodySchema = z.object({ id: z.string().uuid(), message: z.object({ @@ -58,6 +85,7 @@ export const postRequestBodySchema = z.object({ deepThinkingEnabled: z.boolean().optional(), videoParams: videoParamsSchema, // 视频参数 imageParams: imageParamsSchema, // 🆕 图片生成参数 + contextPayload: contextPayloadSchema, }); export type PostRequestBody = z.infer; diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index b4139db31..1ade4abf9 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -3177,10 +3177,12 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF id: rawId, modelId: currentModelIdRef.current, imageParams: imageParamsRef.current, + contextPayload: buildContextPayload(), lastMessage: request.messages.at(-1), }); const hiddenPrompt = hiddenPromptRef.current || undefined; const autoContext = autoContextRef.current || undefined; + const contextPayload = buildContextPayload(); hiddenPromptRef.current = null; autoContextRef.current = null; return { @@ -3197,6 +3199,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF imageParams: isVideoModelRef.current ? undefined : imageParamsRef.current || undefined, hiddenPrompt, autoContext, + contextPayload, ...request.body, }, }; @@ -3951,6 +3954,67 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF return lines.join("\n"); }, [nodes, selectedImageNodes]); + const buildContextPayload = useCallback(() => { + const imageNodes = nodes.filter( + (node) => + node.type === "image" && + typeof (node.data as any)?.src === "string" && + String((node.data as any).src).trim().length > 0, + ); + + if (imageNodes.length === 0) return undefined; + + const MAX_CONTEXT_IMAGES = 12; + const selectedIds = new Set(selectedImageNodes.map((node) => node.id)); + const prioritizedNodes = + selectedImageNodes.length > 0 + ? [ + ...selectedImageNodes, + ...imageNodes.filter((node) => !selectedIds.has(node.id)), + ] + : imageNodes; + const contextNodes = prioritizedNodes.slice(0, MAX_CONTEXT_IMAGES); + const selectedAssetIds = Array.from( + new Set( + selectedImageNodes + .map((node) => { + const assetId = (node.data as any)?.assetId; + return typeof assetId === "string" && assetId.trim() ? assetId.trim() : null; + }) + .filter((value): value is string => Boolean(value)), + ), + ); + + return { + source: "canvas" as const, + projectId: rawChatId || undefined, + selectedNodeIds: selectedImageNodes.map((node) => node.id), + selectedAssetIds, + focusPolicy: selectedImageNodes.length > 0 ? ("selected" as const) : ("auto" as const), + canvasImageNodes: contextNodes.map((node) => ({ + nodeId: node.id, + assetId: + typeof (node.data as any)?.assetId === "string" && (node.data as any).assetId.trim() + ? String((node.data as any).assetId).trim() + : undefined, + url: String((node.data as any)?.src || ""), + title: + typeof (node.data as any)?.title === "string" + ? String((node.data as any).title).slice(0, 256) + : undefined, + prompt: + typeof (node.data as any)?.prompt === "string" + ? String((node.data as any).prompt).slice(0, 2000) + : undefined, + x: Number(node.position.x || 0), + y: Number(node.position.y || 0), + width: Number(node.size?.width || 0) || undefined, + height: Number(node.size?.height || 0) || undefined, + isSelected: selectedIds.has(node.id), + })), + }; + }, [nodes, rawChatId, selectedImageNodes]); + const updateAttachmentAtVideoSlot = useCallback( (slotIndex: number, attachment: PickedObjectAttachment | null) => { const slotCount = Math.max(videoFrameSlots.length, 2); @@ -6012,6 +6076,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF sendMessage, clearSelection, buildAutoContext, + buildContextPayload, isVideoModel, videoParams, videoFrameSlots.length, diff --git a/components/canvas/nodes/ImageNode.tsx b/components/canvas/nodes/ImageNode.tsx index 1dbe283f3..f82adf218 100644 --- a/components/canvas/nodes/ImageNode.tsx +++ b/components/canvas/nodes/ImageNode.tsx @@ -2498,6 +2498,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) { {resizeAnnotatorOpen && data.src && ( (null); + const objectUrlRef = useRef(null); const toCanvasLoadUrl = useCallback((url: string) => { return buildImageProxyUrl(url); }, []); + const revokeObjectUrl = useCallback(() => { + if (objectUrlRef.current) { + URL.revokeObjectURL(objectUrlRef.current); + objectUrlRef.current = null; + } + }, []); + + const createImageElement = useCallback((src: string, useCrossOrigin = true) => { + return new Promise((resolve, reject) => { + const img = new window.Image(); + if (useCrossOrigin) { + img.crossOrigin = "anonymous"; + } + img.onload = () => resolve(img); + img.onerror = () => reject(new Error(`图片加载失败: ${src}`)); + img.src = src; + }); + }, []); + + const loadImageFromSource = useCallback(async (sourceUrl: string) => { + const normalizedSource = String(sourceUrl || "").trim(); + if (!normalizedSource) { + throw new Error("图片地址为空"); + } + + const directUrl = normalizedSource; + const proxiedUrl = toCanvasLoadUrl(normalizedSource); + const candidateUrls = Array.from( + new Set([proxiedUrl, directUrl].filter((item): item is string => Boolean(item))) + ); + + for (const candidateUrl of candidateUrls) { + try { + const shouldUseCrossOrigin = !candidateUrl.startsWith("data:") && !candidateUrl.startsWith("blob:"); + const imageElement = await createImageElement(candidateUrl, shouldUseCrossOrigin); + revokeObjectUrl(); + return imageElement; + } catch { + continue; + } + } + + for (const candidateUrl of candidateUrls) { + try { + const response = await fetch(candidateUrl, { cache: "force-cache" }); + if (!response.ok) { + continue; + } + + const blob = await response.blob(); + const objectUrl = URL.createObjectURL(blob); + const imageElement = await createImageElement(objectUrl, false); + revokeObjectUrl(); + objectUrlRef.current = objectUrl; + return imageElement; + } catch { + continue; + } + } + + throw new Error("图片加载失败"); + }, [createImageElement, revokeObjectUrl, toCanvasLoadUrl]); + + useEffect(() => { + return () => { + revokeObjectUrl(); + }; + }, [revokeObjectUrl]); + // Handle open change const handleOpenChange = (newOpen: boolean) => { onOpenChange?.(newOpen); @@ -160,18 +230,10 @@ export function ImageAnnotator({ setProcessType(null); setActions([]); setCurrentAction(null); - - const img = new window.Image(); - img.crossOrigin = "anonymous"; - img.onload = () => { - setImage(img); - setIsLoading(false); - }; - img.onerror = () => { - setIsLoading(false); - toast.error("图片加载失败"); - }; - img.src = toCanvasLoadUrl(imageUrl); + + const img = await loadImageFromSource(imageUrl); + setImage(img); + setIsLoading(false); toast.success("图片上传成功"); } catch (error) { console.error("上传图片失败:", error); @@ -179,7 +241,7 @@ export function ImageAnnotator({ setIsLoading(false); } } - }, []); + }, [loadImageFromSource]); const handleDragOver = useCallback((e: React.DragEvent) => { e.preventDefault(); @@ -215,18 +277,10 @@ export function ImageAnnotator({ setProcessType(null); setActions([]); setCurrentAction(null); - - const img = new window.Image(); - img.crossOrigin = "anonymous"; - img.onload = () => { - setImage(img); - setIsLoading(false); - }; - img.onerror = () => { - setIsLoading(false); - toast.error("图片加载失败"); - }; - img.src = toCanvasLoadUrl(imageUrl); + + const img = await loadImageFromSource(imageUrl); + setImage(img); + setIsLoading(false); toast.success("图片上传成功"); } catch (error) { console.error("上传图片失败:", error); @@ -240,7 +294,7 @@ export function ImageAnnotator({ document.addEventListener('paste', handlePaste); return () => document.removeEventListener('paste', handlePaste); - }, [mounted, originalImageUrl]); + }, [loadImageFromSource, mounted, originalImageUrl]); // 尺寸调整相关状态 const [isResizeMode, setIsResizeMode] = useState(initialResizeMode); @@ -362,19 +416,10 @@ export function ImageAnnotator({ setProcessType(null); setActions([]); setCurrentAction(null); - - // 触发图片加载逻辑 - const img = new window.Image(); - img.crossOrigin = "anonymous"; - img.onload = () => { - setImage(img); - setIsLoading(false); - }; - img.onerror = () => { - setIsLoading(false); - toast.error("图片加载失败"); - }; - img.src = toCanvasLoadUrl(imageUrl); + + const img = await loadImageFromSource(imageUrl); + setImage(img); + setIsLoading(false); toast.success("图片上传成功"); } catch (error) { console.error("上传图片失败:", error); @@ -390,29 +435,41 @@ export function ImageAnnotator({ useEffect(() => { if (!imageUrl) return; // 如果没有初始图片，不执行加载逻辑 - setIsLoading(true); - setIsResizeMode(initialResizeMode); - setOriginalImageUrl(imageUrl); - setProcessedImageUrl(null); - setProcessedImageBase64(null); - setProcessType(null); - const img = new window.Image(); - img.crossOrigin = "anonymous"; - img.onload = () => { - setImage(img); - setIsLoading(false); - if (img) { + let cancelled = false; + + const initializeImage = async () => { + try { + setIsLoading(true); + setIsResizeMode(initialResizeMode); + setOriginalImageUrl(imageUrl); + setProcessedImageUrl(null); + setProcessedImageBase64(null); + setProcessType(null); + + const img = await loadImageFromSource(imageUrl); + if (cancelled) return; + + setImage(img); + setIsLoading(false); setIsResizeMode(initialResizeMode); setResizeCanvasWidth(img.width); setResizeCanvasHeight(img.height); setSelectedAspectRatio(null); + } catch (error) { + if (cancelled) return; + console.error("初始化图片加载失败:", error); + setImage(null); + setIsLoading(false); + toast.error("当前图片加载失败，请重试"); } }; - img.onerror = () => { - setIsLoading(false); + + initializeImage(); + + return () => { + cancelled = true; }; - img.src = toCanvasLoadUrl(imageUrl); - }, [imageUrl, initialResizeMode, toCanvasLoadUrl]); + }, [imageUrl, initialResizeMode, loadImageFromSource]); // 初始化 canvas - 适配弹窗大小，固定编辑器大小，图片缩小显示 useEffect(() => { diff --git a/lib/ai/context-memory/media-input.ts b/lib/ai/context-memory/media-input.ts new file mode 100644 index 000000000..db1d9e3cd --- /dev/null +++ b/lib/ai/context-memory/media-input.ts @@ -0,0 +1,309 @@ +import "server-only"; + +import { extractImageUrlsFromText, inferOriginalImageUrl } from "@/lib/image-url"; +import type { DBMessage } from "@/lib/db/schema"; +import type { + CanvasImageNodeContext, + ResolveChatMediaInputParams, + ResolvedChatMediaInput, + ResolvedImageRef, +} from "@/lib/ai/context-memory/types"; + +function normalizeMediaUrl(url: string): string { + const raw = String(url || "").trim(); + if (!raw) return ""; + + if (/^data:image\/[a-z0-9+.-]+;base64,/i.test(raw)) { + return raw.replace(/\s+/g, ""); + } + + try { + const parsed = new URL(raw); + return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, "").toLowerCase(); + } catch { + return raw + .split("#")[0] + .split("?")[0] + .replace(/\/+$/, "") + .toLowerCase(); + } +} + +function normalizeImageInputRef(rawValue: unknown, requestOrigin?: string): string | null { + if (typeof rawValue !== "string") return null; + const trimmed = rawValue.trim(); + if (!trimmed) return null; + + if (/^data:image\/[a-z0-9+.-]+;base64,/i.test(trimmed)) { + return trimmed.replace(/\s+/g, ""); + } + + if (/^https?:\/\//i.test(trimmed)) { + return inferOriginalImageUrl(trimmed); + } + + if (requestOrigin && trimmed.startsWith("/")) { + try { + return new URL(trimmed, requestOrigin).toString(); + } catch { + return null; + } + } + + return null; +} + +function collectImageInputsFromMessageParts( + parts: Array> | undefined, + requestOrigin?: string, +): string[] { + if (!Array.isArray(parts) || parts.length === 0) return []; + + const result: string[] = []; + const seen = new Set(); + const pushInput = (rawValue: unknown) => { + const normalized = normalizeImageInputRef(rawValue, requestOrigin); + if (!normalized) return; + const key = normalizeMediaUrl(normalized); + if (!key || seen.has(key)) return; + seen.add(key); + result.push(normalized); + }; + + for (const part of parts) { + if (!part) continue; + + if (part.type === "file") { + pushInput(part.url); + pushInput(part.data); + continue; + } + + if (part.type === "image") { + pushInput(part.url); + pushInput(part.image); + pushInput(part.data); + continue; + } + + if (part.type === "image_url") { + pushInput(part.image_url?.url); + pushInput(part.url); + continue; + } + + if (part.type === "text" && typeof part.text === "string") { + for (const parsedUrl of extractImageUrlsFromText(part.text)) { + pushInput(parsedUrl); + } + } + } + + return result; +} + +function dedupeRefs(refs: ResolvedImageRef[], maxImages: number): ResolvedImageRef[] { + const seen = new Set(); + const result: ResolvedImageRef[] = []; + + for (const ref of refs) { + const key = ref.normalizedUrl || normalizeMediaUrl(ref.url); + if (!key || seen.has(key)) continue; + seen.add(key); + result.push({ ...ref, normalizedUrl: key }); + if (result.length >= maxImages) break; + } + + return result; +} + +function toCanvasImageRefs(nodes: CanvasImageNodeContext[] | undefined, maxImages: number) { + if (!Array.isArray(nodes) || nodes.length === 0) return [] as ResolvedImageRef[]; + + return dedupeRefs( + nodes + .filter((node) => typeof node?.url === "string" && node.url.trim()) + .map((node) => ({ + url: inferOriginalImageUrl(node.url), + normalizedUrl: normalizeMediaUrl(inferOriginalImageUrl(node.url)), + assetId: typeof node.assetId === "string" ? node.assetId.trim() || undefined : undefined, + nodeId: node.nodeId, + title: node.title, + prompt: node.prompt, + x: node.x, + y: node.y, + width: node.width, + height: node.height, + isSelected: Boolean(node.isSelected), + source: node.isSelected ? "canvas_selection" : "canvas_context", + role: node.isSelected ? "selected" : "reference", + })), + maxImages, + ); +} + +function extractHistoryImageRefs(params: { + historyMessages: DBMessage[]; + requestOrigin?: string; + maxImages: number; +}) { + const latestAssistant: ResolvedImageRef[] = []; + const latestUser: ResolvedImageRef[] = []; + const seenAssistant = new Set(); + const seenUser = new Set(); + + const pushRef = ( + bucket: ResolvedImageRef[], + seen: Set, + url: string, + options: Pick, + ) => { + const normalizedUrl = normalizeMediaUrl(url); + if (!normalizedUrl || seen.has(normalizedUrl) || bucket.length >= params.maxImages) return; + seen.add(normalizedUrl); + bucket.push({ + url, + normalizedUrl, + source: options.source, + role: options.role, + messageId: options.messageId, + }); + }; + + for (let index = params.historyMessages.length - 1; index >= 0; index -= 1) { + const historyMessage = params.historyMessages[index]; + const parts = historyMessage.parts as Array> | undefined; + if (!Array.isArray(parts) || parts.length === 0) continue; + + const urls = collectImageInputsFromMessageParts(parts, params.requestOrigin); + if (!urls.length) continue; + + if (historyMessage.role === "assistant" && latestAssistant.length < params.maxImages) { + for (const url of urls) { + pushRef(latestAssistant, seenAssistant, url, { + source: "history_assistant_result", + role: "history_result", + messageId: historyMessage.id, + }); + } + } + + if (historyMessage.role === "user" && latestUser.length < params.maxImages) { + for (const url of urls) { + pushRef(latestUser, seenUser, url, { + source: "history_user_upload", + role: "reference", + messageId: historyMessage.id, + }); + } + } + + if (latestAssistant.length >= params.maxImages && latestUser.length >= params.maxImages) { + break; + } + } + + return { + latestAssistant, + latestUser, + }; +} + +export function resolveChatMediaInput( + params: ResolveChatMediaInputParams, +): ResolvedChatMediaInput { + const maxImages = Math.max(1, Math.min(12, Math.round(params.maxImages || 4))); + const currentMessageImageRefs = dedupeRefs( + collectImageInputsFromMessageParts(params.messageParts, params.requestOrigin).map((url) => ({ + url, + normalizedUrl: normalizeMediaUrl(url), + source: "current_message", + role: "upload", + })), + maxImages, + ); + + const canvasImageRefs = toCanvasImageRefs(params.contextPayload?.canvasImageNodes, maxImages); + const selectedNodeIds = new Set(params.contextPayload?.selectedNodeIds || []); + const selectedImageRefs = dedupeRefs( + canvasImageRefs.filter((ref) => ref.isSelected || (ref.nodeId && selectedNodeIds.has(ref.nodeId))), + maxImages, + ); + const history = extractHistoryImageRefs({ + historyMessages: params.historyMessages, + requestOrigin: params.requestOrigin, + maxImages, + }); + + const resolvedImageRefs = + currentMessageImageRefs.length > 0 + ? currentMessageImageRefs + : selectedImageRefs.length > 0 + ? selectedImageRefs + : history.latestAssistant.length > 0 + ? history.latestAssistant + : history.latestUser; + + const imageRefs = dedupeRefs( + [ + ...currentMessageImageRefs, + ...selectedImageRefs, + ...canvasImageRefs, + ...history.latestAssistant, + ...history.latestUser, + ], + maxImages * 4, + ); + + const selectedAssetIds = Array.from( + new Set( + selectedImageRefs + .map((ref) => ref.assetId) + .filter((value): value is string => typeof value === "string" && value.trim().length > 0), + ), + ); + const uploadedAssetIds = Array.from( + new Set( + currentMessageImageRefs + .map((ref) => ref.assetId) + .filter((value): value is string => typeof value === "string" && value.trim().length > 0), + ), + ); + const candidateAssetIds = Array.from( + new Set( + imageRefs + .map((ref) => ref.assetId) + .filter((value): value is string => typeof value === "string" && value.trim().length > 0), + ), + ); + + const source = + currentMessageImageRefs.length > 0 + ? "current_message" + : selectedImageRefs.length > 0 + ? "canvas_selection" + : resolvedImageRefs.length > 0 + ? "history" + : "none"; + + return { + imageRefs, + currentMessageImageRefs, + selectedImageRefs, + canvasImageRefs, + historyUserImageRefs: history.latestUser, + historyAssistantImageRefs: history.latestAssistant, + resolvedImageRefs, + currentMessageImageUrls: currentMessageImageRefs.map((ref) => ref.url), + selectedImageUrls: selectedImageRefs.map((ref) => ref.url), + historyUserImageUrls: history.latestUser.map((ref) => ref.url), + historyAssistantImageUrls: history.latestAssistant.map((ref) => ref.url), + resolvedImageUrls: resolvedImageRefs.map((ref) => ref.url), + selectedAssetIds, + candidateAssetIds, + uploadedAssetIds, + hasImage: resolvedImageRefs.length > 0, + source, + fromHistory: source === "history" || source === "canvas_selection", + }; +} diff --git a/lib/ai/context-memory/target-resolution.ts b/lib/ai/context-memory/target-resolution.ts new file mode 100644 index 000000000..f204e720d --- /dev/null +++ b/lib/ai/context-memory/target-resolution.ts @@ -0,0 +1,378 @@ +import "server-only"; + +import type { + ResolveTargetAssetsParams, + ResolvedImageRef, + ResolvedTargetAssets, +} from "@/lib/ai/context-memory/types"; + +const ORIGINAL_IMAGE_REGEX = + /(原图|上传图|参考图|最初那张|第一张图|素材图|源图|原始图片|原始那张)/i; +const LATEST_RESULT_REGEX = + /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果|上一版|刚才那张结果)/i; +const SELECTION_REGEX = /(选中的|当前这张|我选的|这几张|这些图|选中图)/i; +const ACTIONABLE_EDIT_REGEX = + /(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格|亮一点|暗一点|白底|透明底)/i; + +function normalizeStringArray(values?: string[] | null): string[] { + if (!Array.isArray(values)) return []; + return Array.from( + new Set( + values + .filter((value): value is string => typeof value === "string") + .map((value) => value.trim()) + .filter(Boolean), + ), + ); +} + +function chineseNumberToInt(value: string): number | null { + const normalized = value.trim(); + if (!normalized) return null; + if (/^\d+$/.test(normalized)) return Number.parseInt(normalized, 10); + + const map: Record = { + 一: 1, + 二: 2, + 两: 2, + 三: 3, + 四: 4, + 五: 5, + 六: 6, + 七: 7, + 八: 8, + 九: 9, + 十: 10, + }; + + if (normalized === "十") return 10; + if (normalized.length === 1) return map[normalized] ?? null; + if (normalized.startsWith("十")) { + return 10 + (map[normalized.slice(1)] ?? 0); + } + if (normalized.endsWith("十")) { + return (map[normalized.slice(0, 1)] ?? 0) * 10; + } + if (normalized.includes("十")) { + const [head, tail] = normalized.split("十"); + return (map[head] ?? 0) * 10 + (map[tail] ?? 0); + } + return null; +} + +function sortRefsForCanvasOrdering(refs: ResolvedImageRef[]) { + return [...refs].sort((left, right) => { + const leftY = Number(left.y ?? 0); + const rightY = Number(right.y ?? 0); + if (Math.abs(leftY - rightY) > 12) { + return leftY - rightY; + } + return Number(left.x ?? 0) - Number(right.x ?? 0); + }); +} + +function getPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): string { + if (refs.length <= 1) return "中间"; + const xs = refs.map((item) => Number(item.x ?? 0)); + const ys = refs.map((item) => Number(item.y ?? 0)); + const minX = Math.min(...xs); + const maxX = Math.max(...xs); + const minY = Math.min(...ys); + const maxY = Math.max(...ys); + const spanX = Math.max(1, maxX - minX); + const spanY = Math.max(1, maxY - minY); + const relX = (Number(ref.x ?? 0) - minX) / spanX; + const relY = (Number(ref.y ?? 0) - minY) / spanY; + + const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中"; + const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中"; + + if (horizontal === "中" && vertical === "中") return "中间"; + return `${vertical}${horizontal}`; +} + +function pickByOrdinal(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] { + const matched = text.match(/第\s*([一二三四五六七八九十两\d]+)\s*(?:张|个|幅|图)/i); + const numeric = matched ? chineseNumberToInt(matched[1]) : null; + if (!numeric || numeric <= 0) return []; + const ordered = sortRefsForCanvasOrdering(refs); + return ordered[numeric - 1] ? [ordered[numeric - 1]] : []; +} + +function pickBySpatialHint(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] { + if (!refs.length) return []; + + const normalizedText = text.replace(/\s+/g, ""); + const hasLeft = normalizedText.includes("左"); + const hasRight = normalizedText.includes("右"); + const hasTop = normalizedText.includes("上"); + const hasBottom = normalizedText.includes("下"); + const hasCenter = normalizedText.includes("中间") || normalizedText.includes("中"); + + if (!(hasLeft || hasRight || hasTop || hasBottom || hasCenter)) return []; + + const withLabels = refs.map((ref) => ({ + ref, + label: getPositionLabel(ref, refs), + })); + + return withLabels + .filter(({ label }) => { + if (hasLeft && !label.includes("左")) return false; + if (hasRight && !label.includes("右")) return false; + if (hasTop && !label.includes("上")) return false; + if (hasBottom && !label.includes("下")) return false; + if (hasCenter && !(label.includes("中") || label === "中间")) return false; + return true; + }) + .map(({ ref }) => ref); +} + +function buildResolvedTargetAssets(params: { + targetRefs: ResolvedImageRef[]; + candidateRefs: ResolvedImageRef[]; + resolutionMode: ResolvedTargetAssets["resolutionMode"]; + confidence: number; + needsClarification: boolean; + reason: string; + fromHistory: boolean; +}): ResolvedTargetAssets { + const targetRefs = params.targetRefs; + const candidateRefs = params.candidateRefs; + return { + targetRefs, + candidateRefs, + targetImageUrls: targetRefs.map((ref) => ref.url), + targetAssetIds: normalizeStringArray(targetRefs.map((ref) => ref.assetId || "")), + candidateImageUrls: candidateRefs.map((ref) => ref.url), + candidateAssetIds: normalizeStringArray(candidateRefs.map((ref) => ref.assetId || "")), + resolutionMode: params.resolutionMode, + confidence: Math.max(0, Math.min(100, Math.round(params.confidence))), + needsClarification: params.needsClarification, + reason: params.reason, + fromHistory: params.fromHistory, + }; +} + +export function resolveTargetAssets( + params: ResolveTargetAssetsParams, +): ResolvedTargetAssets { + const text = String(params.text || "").trim(); + const mediaInput = params.mediaInput; + const canvasRefs = mediaInput.canvasImageRefs; + const selectedRefs = mediaInput.selectedImageRefs; + const currentRefs = mediaInput.currentMessageImageRefs; + const assistantRefs = mediaInput.historyAssistantImageRefs; + const userRefs = mediaInput.historyUserImageRefs; + const candidateRefs = mediaInput.imageRefs; + const workingState = params.workingState; + const focusPolicy = params.contextPayload?.focusPolicy || "auto"; + + if (focusPolicy === "original" && userRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: userRefs, + candidateRefs, + resolutionMode: "text_reference", + confidence: 96, + needsClarification: false, + reason: "前端上下文指定优先使用原图", + fromHistory: true, + }); + } + + if (focusPolicy === "latest_result" && assistantRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: assistantRefs, + candidateRefs, + resolutionMode: "latest_result", + confidence: 96, + needsClarification: false, + reason: "前端上下文指定优先使用上一轮结果", + fromHistory: true, + }); + } + + if (focusPolicy === "selected" && selectedRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: selectedRefs, + candidateRefs, + resolutionMode: "explicit_selection", + confidence: 97, + needsClarification: false, + reason: "前端已明确选中目标图片", + fromHistory: true, + }); + } + + if (ORIGINAL_IMAGE_REGEX.test(text) && userRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: userRefs, + candidateRefs, + resolutionMode: "text_reference", + confidence: 94, + needsClarification: false, + reason: "用户文本明确指向原图/上传图", + fromHistory: true, + }); + } + + if (LATEST_RESULT_REGEX.test(text) && assistantRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: assistantRefs, + candidateRefs, + resolutionMode: "latest_result", + confidence: 94, + needsClarification: false, + reason: "用户文本明确指向上一轮结果", + fromHistory: true, + }); + } + + const ordinalRefs = pickByOrdinal(text, canvasRefs); + if (ordinalRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: ordinalRefs, + candidateRefs, + resolutionMode: "text_reference", + confidence: 88, + needsClarification: false, + reason: "根据用户文本中的序号定位到画布图片", + fromHistory: true, + }); + } + + const spatialRefs = pickBySpatialHint(text, canvasRefs); + if (spatialRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: spatialRefs, + candidateRefs, + resolutionMode: "text_reference", + confidence: spatialRefs.length === 1 ? 90 : 76, + needsClarification: false, + reason: "根据用户文本中的方位指向定位到画布图片", + fromHistory: true, + }); + } + + if (currentRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: currentRefs, + candidateRefs, + resolutionMode: "current_upload", + confidence: 93, + needsClarification: false, + reason: "本轮消息包含图片输入，默认优先处理当前上传图片", + fromHistory: false, + }); + } + + if ((SELECTION_REGEX.test(text) || focusPolicy === "selected") && selectedRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: selectedRefs, + candidateRefs, + resolutionMode: "explicit_selection", + confidence: 92, + needsClarification: false, + reason: "用户文本或上下文明确要求使用当前选中的图片", + fromHistory: true, + }); + } + + if (selectedRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: selectedRefs, + candidateRefs, + resolutionMode: "explicit_selection", + confidence: 86, + needsClarification: false, + reason: "当前存在画布选中图片，默认优先继续处理选中目标", + fromHistory: true, + }); + } + + const workingStateResolvedUrls = normalizeStringArray( + workingState?.lastResolvedTargetImageUrls, + ); + if (workingStateResolvedUrls.length > 0) { + const refs = workingStateResolvedUrls.map((url) => ({ + url, + normalizedUrl: url, + source: "working_memory" as const, + role: "working_memory" as const, + })); + return buildResolvedTargetAssets({ + targetRefs: refs, + candidateRefs, + resolutionMode: "working_memory", + confidence: 82, + needsClarification: false, + reason: "使用会话工作记忆中的上一轮已解析目标图", + fromHistory: true, + }); + } + + const workingStateGeneratedUrls = normalizeStringArray(workingState?.lastGeneratedImageUrls); + if (workingStateGeneratedUrls.length > 0) { + const refs = workingStateGeneratedUrls.map((url) => ({ + url, + normalizedUrl: url, + source: "working_memory" as const, + role: "working_memory" as const, + })); + return buildResolvedTargetAssets({ + targetRefs: refs, + candidateRefs, + resolutionMode: "working_memory", + confidence: 78, + needsClarification: false, + reason: "使用会话工作记忆中的最近生成结果", + fromHistory: true, + }); + } + + if (assistantRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: assistantRefs, + candidateRefs, + resolutionMode: "latest_result", + confidence: 74, + needsClarification: false, + reason: "未显式指定时，默认续接最近一轮生成结果", + fromHistory: true, + }); + } + + if (userRefs.length > 0) { + return buildResolvedTargetAssets({ + targetRefs: userRefs, + candidateRefs, + resolutionMode: "working_memory", + confidence: 68, + needsClarification: false, + reason: "回退使用最近一次用户上传图片", + fromHistory: true, + }); + } + + if (ACTIONABLE_EDIT_REGEX.test(text)) { + return buildResolvedTargetAssets({ + targetRefs: [], + candidateRefs, + resolutionMode: "clarification", + confidence: 0, + needsClarification: true, + reason: "用户表达了编辑意图，但当前没有可用图片目标", + fromHistory: false, + }); + } + + return buildResolvedTargetAssets({ + targetRefs: [], + candidateRefs, + resolutionMode: "none", + confidence: 0, + needsClarification: false, + reason: "当前请求没有可定位的图片目标", + fromHistory: false, + }); +} diff --git a/lib/ai/context-memory/types.ts b/lib/ai/context-memory/types.ts new file mode 100644 index 000000000..900d2320f --- /dev/null +++ b/lib/ai/context-memory/types.ts @@ -0,0 +1,119 @@ +import type { DBMessage } from "@/lib/db/schema"; + +export interface CanvasImageNodeContext { + nodeId: string; + assetId?: string; + url: string; + title?: string; + prompt?: string; + x: number; + y: number; + width?: number; + height?: number; + isSelected?: boolean; +} + +export interface ContextPayload { + source?: "chat" | "canvas" | "realtime-canvas"; + projectId?: string; + selectedNodeIds?: string[]; + selectedAssetIds?: string[]; + focusPolicy?: "auto" | "selected" | "latest_result" | "original"; + canvasImageNodes?: CanvasImageNodeContext[]; +} + +export interface ResolvedImageRef { + url: string; + normalizedUrl: string; + assetId?: string; + nodeId?: string; + title?: string; + prompt?: string; + x?: number; + y?: number; + width?: number; + height?: number; + isSelected?: boolean; + source: + | "current_message" + | "canvas_selection" + | "canvas_context" + | "history_user_upload" + | "history_assistant_result" + | "working_memory"; + role: "upload" | "selected" | "reference" | "history_result" | "working_memory"; + messageId?: string; +} + +export interface ResolvedChatMediaInput { + imageRefs: ResolvedImageRef[]; + currentMessageImageRefs: ResolvedImageRef[]; + selectedImageRefs: ResolvedImageRef[]; + canvasImageRefs: ResolvedImageRef[]; + historyUserImageRefs: ResolvedImageRef[]; + historyAssistantImageRefs: ResolvedImageRef[]; + resolvedImageRefs: ResolvedImageRef[]; + currentMessageImageUrls: string[]; + selectedImageUrls: string[]; + historyUserImageUrls: string[]; + historyAssistantImageUrls: string[]; + resolvedImageUrls: string[]; + selectedAssetIds: string[]; + candidateAssetIds: string[]; + uploadedAssetIds: string[]; + hasImage: boolean; + source: "current_message" | "canvas_selection" | "history" | "none"; + fromHistory: boolean; +} + +export interface WorkingStateImageMemory { + currentFocusAssetIds?: string[] | null; + currentFocusImageUrls?: string[] | null; + selectedAssetIds?: string[] | null; + selectedImageUrls?: string[] | null; + lastUploadedAssetIds?: string[] | null; + lastUploadedImageUrls?: string[] | null; + lastGeneratedAssetIds?: string[] | null; + lastGeneratedImageUrls?: string[] | null; + lastReferencedAssetIds?: string[] | null; + lastReferencedImageUrls?: string[] | null; + lastResolvedTargetAssetIds?: string[] | null; + lastResolvedTargetImageUrls?: string[] | null; + activeCanvasNodeIds?: string[] | null; +} + +export interface ResolvedTargetAssets { + targetRefs: ResolvedImageRef[]; + candidateRefs: ResolvedImageRef[]; + targetImageUrls: string[]; + targetAssetIds: string[]; + candidateImageUrls: string[]; + candidateAssetIds: string[]; + resolutionMode: + | "explicit_selection" + | "current_upload" + | "text_reference" + | "working_memory" + | "latest_result" + | "clarification" + | "none"; + confidence: number; + needsClarification: boolean; + reason: string; + fromHistory: boolean; +} + +export interface ResolveChatMediaInputParams { + messageParts: Array> | undefined; + historyMessages: DBMessage[]; + contextPayload?: ContextPayload; + requestOrigin?: string; + maxImages?: number; +} + +export interface ResolveTargetAssetsParams { + text: string; + mediaInput: ResolvedChatMediaInput; + contextPayload?: ContextPayload; + workingState?: WorkingStateImageMemory | null; +} diff --git a/lib/db/migrations/0050_add_context_memory_tables.sql b/lib/db/migrations/0050_add_context_memory_tables.sql new file mode 100644 index 000000000..2488e7d92 --- /dev/null +++ b/lib/db/migrations/0050_add_context_memory_tables.sql @@ -0,0 +1,101 @@ +CREATE TABLE IF NOT EXISTS "ConversationWorkingState" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "userId" uuid NOT NULL, + "chatId" uuid NOT NULL, + "projectId" uuid, + "currentIntent" varchar(64), + "currentSubIntent" varchar(64), + "currentFocusAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "currentFocusImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "selectedAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "selectedImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastUploadedAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastUploadedImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastGeneratedAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastGeneratedImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastReferencedAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastReferencedImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastResolvedTargetAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastResolvedTargetImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "activeCanvasNodeIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "lastCanvasSnapshot" jsonb, + "lastTurnSummary" jsonb, + "lastMessageId" uuid, + "turnCount" integer DEFAULT 0 NOT NULL, + "createdAt" timestamp DEFAULT now() NOT NULL, + "updatedAt" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "ConversationWorkingState_chatId_unique" UNIQUE("chatId") +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "MessageTargetResolution" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "userId" uuid NOT NULL, + "chatId" uuid NOT NULL, + "messageId" uuid, + "projectId" uuid, + "rawText" text NOT NULL, + "uploadedAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "uploadedImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "selectedAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "selectedImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "candidateAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "candidateImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "resolvedAssetIds" jsonb DEFAULT '[]'::jsonb NOT NULL, + "resolvedImageUrls" jsonb DEFAULT '[]'::jsonb NOT NULL, + "resolutionMode" varchar(64) NOT NULL, + "confidence" integer DEFAULT 0 NOT NULL, + "needsClarification" boolean DEFAULT false NOT NULL, + "reasoning" jsonb, + "createdAt" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "ConversationWorkingState" ADD CONSTRAINT "ConversationWorkingState_userId_User_id_fk" FOREIGN KEY ("userId") REFERENCES "public"."User"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "ConversationWorkingState" ADD CONSTRAINT "ConversationWorkingState_chatId_Chat_id_fk" FOREIGN KEY ("chatId") REFERENCES "public"."Chat"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "ConversationWorkingState" ADD CONSTRAINT "ConversationWorkingState_projectId_CanvasProject_id_fk" FOREIGN KEY ("projectId") REFERENCES "public"."CanvasProject"("id") ON DELETE set null ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "MessageTargetResolution" ADD CONSTRAINT "MessageTargetResolution_userId_User_id_fk" FOREIGN KEY ("userId") REFERENCES "public"."User"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "MessageTargetResolution" ADD CONSTRAINT "MessageTargetResolution_chatId_Chat_id_fk" FOREIGN KEY ("chatId") REFERENCES "public"."Chat"("id") ON DELETE cascade ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "MessageTargetResolution" ADD CONSTRAINT "MessageTargetResolution_messageId_Message_v2_id_fk" FOREIGN KEY ("messageId") REFERENCES "public"."Message_v2"("id") ON DELETE set null ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "MessageTargetResolution" ADD CONSTRAINT "MessageTargetResolution_projectId_CanvasProject_id_fk" FOREIGN KEY ("projectId") REFERENCES "public"."CanvasProject"("id") ON DELETE set null ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "ConversationWorkingState_user_updated_idx" + ON "ConversationWorkingState"("userId", "updatedAt"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "MessageTargetResolution_chat_created_idx" + ON "MessageTargetResolution"("chatId", "createdAt"); +--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "MessageTargetResolution_message_idx" + ON "MessageTargetResolution"("messageId"); diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json index cd7ece9c6..9fb54e0ec 100644 --- a/lib/db/migrations/meta/_journal.json +++ b/lib/db/migrations/meta/_journal.json @@ -253,6 +253,13 @@ "when": 1772766180000, "tag": "0049_add_model3d_assets", "breakpoints": true + }, + { + "idx": 36, + "version": "7", + "when": 1772788800000, + "tag": "0050_add_context_memory_tables", + "breakpoints": true } ] } diff --git a/lib/db/queries.ts b/lib/db/queries.ts index 3a14a277c..dd18d4736 100644 --- a/lib/db/queries.ts +++ b/lib/db/queries.ts @@ -52,6 +52,10 @@ import { pointsLedger, tutorialVideos, messageDeprecated, + conversationWorkingState, + type ConversationWorkingState, + messageTargetResolution, + type MessageTargetResolution, } from "./schema"; import { generateHashedPassword } from "./utils"; import { @@ -666,6 +670,245 @@ export async function updateChatLastContextById({ } } +function normalizeStringArray(value?: string[] | null): string[] { + if (!Array.isArray(value)) return []; + const deduped = new Set(); + for (const item of value) { + if (typeof item !== "string") continue; + const trimmed = item.trim(); + if (!trimmed) continue; + deduped.add(trimmed); + } + return Array.from(deduped); +} + +export async function getConversationWorkingStateByChatId({ + chatId, +}: { + chatId: string; +}): Promise { + try { + const rows = await db + .select() + .from(conversationWorkingState) + .where(eq(conversationWorkingState.chatId, chatId)) + .limit(1); + return rows[0] ?? null; + } catch (_error) { + throw new ChatSDKError( + "bad_request:database", + "Failed to get conversation working state" + ); + } +} + +export async function upsertConversationWorkingState(params: { + userId: string; + chatId: string; + projectId?: string | null; + currentIntent?: string | null; + currentSubIntent?: string | null; + currentFocusAssetIds?: string[] | null; + currentFocusImageUrls?: string[] | null; + selectedAssetIds?: string[] | null; + selectedImageUrls?: string[] | null; + lastUploadedAssetIds?: string[] | null; + lastUploadedImageUrls?: string[] | null; + lastGeneratedAssetIds?: string[] | null; + lastGeneratedImageUrls?: string[] | null; + lastReferencedAssetIds?: string[] | null; + lastReferencedImageUrls?: string[] | null; + lastResolvedTargetAssetIds?: string[] | null; + lastResolvedTargetImageUrls?: string[] | null; + activeCanvasNodeIds?: string[] | null; + lastCanvasSnapshot?: any; + lastTurnSummary?: any; + lastMessageId?: string | null; + incrementTurnCount?: boolean; +}): Promise { + try { + const existing = await getConversationWorkingStateByChatId({ + chatId: params.chatId, + }); + const now = new Date(); + + const values = { + userId: params.userId, + chatId: params.chatId, + projectId: + params.projectId !== undefined ? params.projectId : existing?.projectId ?? null, + currentIntent: + params.currentIntent !== undefined + ? params.currentIntent + : existing?.currentIntent ?? null, + currentSubIntent: + params.currentSubIntent !== undefined + ? params.currentSubIntent + : existing?.currentSubIntent ?? null, + currentFocusAssetIds: + params.currentFocusAssetIds !== undefined + ? normalizeStringArray(params.currentFocusAssetIds) + : normalizeStringArray(existing?.currentFocusAssetIds), + currentFocusImageUrls: + params.currentFocusImageUrls !== undefined + ? normalizeStringArray(params.currentFocusImageUrls) + : normalizeStringArray(existing?.currentFocusImageUrls), + selectedAssetIds: + params.selectedAssetIds !== undefined + ? normalizeStringArray(params.selectedAssetIds) + : normalizeStringArray(existing?.selectedAssetIds), + selectedImageUrls: + params.selectedImageUrls !== undefined + ? normalizeStringArray(params.selectedImageUrls) + : normalizeStringArray(existing?.selectedImageUrls), + lastUploadedAssetIds: + params.lastUploadedAssetIds !== undefined + ? normalizeStringArray(params.lastUploadedAssetIds) + : normalizeStringArray(existing?.lastUploadedAssetIds), + lastUploadedImageUrls: + params.lastUploadedImageUrls !== undefined + ? normalizeStringArray(params.lastUploadedImageUrls) + : normalizeStringArray(existing?.lastUploadedImageUrls), + lastGeneratedAssetIds: + params.lastGeneratedAssetIds !== undefined + ? normalizeStringArray(params.lastGeneratedAssetIds) + : normalizeStringArray(existing?.lastGeneratedAssetIds), + lastGeneratedImageUrls: + params.lastGeneratedImageUrls !== undefined + ? normalizeStringArray(params.lastGeneratedImageUrls) + : normalizeStringArray(existing?.lastGeneratedImageUrls), + lastReferencedAssetIds: + params.lastReferencedAssetIds !== undefined + ? normalizeStringArray(params.lastReferencedAssetIds) + : normalizeStringArray(existing?.lastReferencedAssetIds), + lastReferencedImageUrls: + params.lastReferencedImageUrls !== undefined + ? normalizeStringArray(params.lastReferencedImageUrls) + : normalizeStringArray(existing?.lastReferencedImageUrls), + lastResolvedTargetAssetIds: + params.lastResolvedTargetAssetIds !== undefined + ? normalizeStringArray(params.lastResolvedTargetAssetIds) + : normalizeStringArray(existing?.lastResolvedTargetAssetIds), + lastResolvedTargetImageUrls: + params.lastResolvedTargetImageUrls !== undefined + ? normalizeStringArray(params.lastResolvedTargetImageUrls) + : normalizeStringArray(existing?.lastResolvedTargetImageUrls), + activeCanvasNodeIds: + params.activeCanvasNodeIds !== undefined + ? normalizeStringArray(params.activeCanvasNodeIds) + : normalizeStringArray(existing?.activeCanvasNodeIds), + lastCanvasSnapshot: + params.lastCanvasSnapshot !== undefined + ? params.lastCanvasSnapshot + : existing?.lastCanvasSnapshot ?? null, + lastTurnSummary: + params.lastTurnSummary !== undefined + ? params.lastTurnSummary + : existing?.lastTurnSummary ?? null, + lastMessageId: + params.lastMessageId !== undefined + ? params.lastMessageId + : existing?.lastMessageId ?? null, + turnCount: + (existing?.turnCount ?? 0) + (params.incrementTurnCount ? 1 : 0), + createdAt: existing?.createdAt ?? now, + updatedAt: now, + }; + + const [result] = await db + .insert(conversationWorkingState) + .values(values) + .onConflictDoUpdate({ + target: conversationWorkingState.chatId, + set: { + userId: values.userId, + projectId: values.projectId, + currentIntent: values.currentIntent, + currentSubIntent: values.currentSubIntent, + currentFocusAssetIds: values.currentFocusAssetIds, + currentFocusImageUrls: values.currentFocusImageUrls, + selectedAssetIds: values.selectedAssetIds, + selectedImageUrls: values.selectedImageUrls, + lastUploadedAssetIds: values.lastUploadedAssetIds, + lastUploadedImageUrls: values.lastUploadedImageUrls, + lastGeneratedAssetIds: values.lastGeneratedAssetIds, + lastGeneratedImageUrls: values.lastGeneratedImageUrls, + lastReferencedAssetIds: values.lastReferencedAssetIds, + lastReferencedImageUrls: values.lastReferencedImageUrls, + lastResolvedTargetAssetIds: values.lastResolvedTargetAssetIds, + lastResolvedTargetImageUrls: values.lastResolvedTargetImageUrls, + activeCanvasNodeIds: values.activeCanvasNodeIds, + lastCanvasSnapshot: values.lastCanvasSnapshot, + lastTurnSummary: values.lastTurnSummary, + lastMessageId: values.lastMessageId, + turnCount: values.turnCount, + updatedAt: values.updatedAt, + }, + }) + .returning(); + + return result ?? null; + } catch (_error) { + throw new ChatSDKError( + "bad_request:database", + "Failed to upsert conversation working state" + ); + } +} + +export async function createMessageTargetResolution(params: { + userId: string; + chatId: string; + messageId?: string | null; + projectId?: string | null; + rawText: string; + uploadedAssetIds?: string[] | null; + uploadedImageUrls?: string[] | null; + selectedAssetIds?: string[] | null; + selectedImageUrls?: string[] | null; + candidateAssetIds?: string[] | null; + candidateImageUrls?: string[] | null; + resolvedAssetIds?: string[] | null; + resolvedImageUrls?: string[] | null; + resolutionMode: string; + confidence?: number | null; + needsClarification?: boolean; + reasoning?: any; +}): Promise { + try { + const [result] = await db + .insert(messageTargetResolution) + .values({ + userId: params.userId, + chatId: params.chatId, + messageId: params.messageId ?? null, + projectId: params.projectId ?? null, + rawText: params.rawText, + uploadedAssetIds: normalizeStringArray(params.uploadedAssetIds), + uploadedImageUrls: normalizeStringArray(params.uploadedImageUrls), + selectedAssetIds: normalizeStringArray(params.selectedAssetIds), + selectedImageUrls: normalizeStringArray(params.selectedImageUrls), + candidateAssetIds: normalizeStringArray(params.candidateAssetIds), + candidateImageUrls: normalizeStringArray(params.candidateImageUrls), + resolvedAssetIds: normalizeStringArray(params.resolvedAssetIds), + resolvedImageUrls: normalizeStringArray(params.resolvedImageUrls), + resolutionMode: params.resolutionMode, + confidence: Math.max(0, Math.min(100, Math.round(params.confidence ?? 0))), + needsClarification: Boolean(params.needsClarification), + reasoning: params.reasoning ?? null, + createdAt: new Date(), + }) + .returning(); + + return result ?? null; + } catch (_error) { + throw new ChatSDKError( + "bad_request:database", + "Failed to create message target resolution" + ); + } +} + export async function getMessageCountByUserId({ id, differenceInHours, diff --git a/lib/db/schema.ts b/lib/db/schema.ts index fd7d921e7..40d710048 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -665,3 +665,80 @@ export const videoStoryboardShotTask = pgTable("VideoStoryboardShotTask", { }); export type VideoStoryboardShotTask = InferSelectModel; + +// 会话工作记忆表（用于连续对话中的图片/资产目标定位） +export const conversationWorkingState = pgTable("ConversationWorkingState", { + id: uuid("id").primaryKey().notNull().defaultRandom(), + userId: uuid("userId") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + chatId: uuid("chatId") + .notNull() + .references(() => chat.id, { onDelete: "cascade" }) + .unique(), + projectId: uuid("projectId").references(() => canvasProject.id, { + onDelete: "set null", + }), + currentIntent: varchar("currentIntent", { length: 64 }), + currentSubIntent: varchar("currentSubIntent", { length: 64 }), + currentFocusAssetIds: jsonb("currentFocusAssetIds").$type().notNull().default([]), + currentFocusImageUrls: jsonb("currentFocusImageUrls").$type().notNull().default([]), + selectedAssetIds: jsonb("selectedAssetIds").$type().notNull().default([]), + selectedImageUrls: jsonb("selectedImageUrls").$type().notNull().default([]), + lastUploadedAssetIds: jsonb("lastUploadedAssetIds").$type().notNull().default([]), + lastUploadedImageUrls: jsonb("lastUploadedImageUrls").$type().notNull().default([]), + lastGeneratedAssetIds: jsonb("lastGeneratedAssetIds").$type().notNull().default([]), + lastGeneratedImageUrls: jsonb("lastGeneratedImageUrls").$type().notNull().default([]), + lastReferencedAssetIds: jsonb("lastReferencedAssetIds").$type().notNull().default([]), + lastReferencedImageUrls: jsonb("lastReferencedImageUrls").$type().notNull().default([]), + lastResolvedTargetAssetIds: jsonb("lastResolvedTargetAssetIds") + .$type() + .notNull() + .default([]), + lastResolvedTargetImageUrls: jsonb("lastResolvedTargetImageUrls") + .$type() + .notNull() + .default([]), + activeCanvasNodeIds: jsonb("activeCanvasNodeIds").$type().notNull().default([]), + lastCanvasSnapshot: jsonb("lastCanvasSnapshot"), + lastTurnSummary: jsonb("lastTurnSummary"), + lastMessageId: uuid("lastMessageId"), + turnCount: integer("turnCount").notNull().default(0), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow(), +}); + +export type ConversationWorkingState = InferSelectModel; + +// 每轮消息的目标资产/图片定位审计表 +export const messageTargetResolution = pgTable("MessageTargetResolution", { + id: uuid("id").primaryKey().notNull().defaultRandom(), + userId: uuid("userId") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + chatId: uuid("chatId") + .notNull() + .references(() => chat.id, { onDelete: "cascade" }), + messageId: uuid("messageId").references(() => message.id, { + onDelete: "set null", + }), + projectId: uuid("projectId").references(() => canvasProject.id, { + onDelete: "set null", + }), + rawText: text("rawText").notNull(), + uploadedAssetIds: jsonb("uploadedAssetIds").$type().notNull().default([]), + uploadedImageUrls: jsonb("uploadedImageUrls").$type().notNull().default([]), + selectedAssetIds: jsonb("selectedAssetIds").$type().notNull().default([]), + selectedImageUrls: jsonb("selectedImageUrls").$type().notNull().default([]), + candidateAssetIds: jsonb("candidateAssetIds").$type().notNull().default([]), + candidateImageUrls: jsonb("candidateImageUrls").$type().notNull().default([]), + resolvedAssetIds: jsonb("resolvedAssetIds").$type().notNull().default([]), + resolvedImageUrls: jsonb("resolvedImageUrls").$type().notNull().default([]), + resolutionMode: varchar("resolutionMode", { length: 64 }).notNull(), + confidence: integer("confidence").notNull().default(0), + needsClarification: boolean("needsClarification").notNull().default(false), + reasoning: jsonb("reasoning"), + createdAt: timestamp("createdAt").notNull().defaultNow(), +}); + +export type MessageTargetResolution = InferSelectModel; diff --git a/types/canvas/index.ts b/types/canvas/index.ts index 21855b38c..ed915bb5d 100644 --- a/types/canvas/index.ts +++ b/types/canvas/index.ts @@ -41,6 +41,7 @@ export interface ImageTextEditItem { // 图片节点数据 export interface ImageNodeData { src: string; + assetId?: string; title?: string; note?: string; prompt?: string; @@ -59,6 +60,7 @@ export interface ImageNodeData { // 视频节点数据 export interface VideoNodeData { src: string; + assetId?: string; thumbnail?: string; prompt?: string; model?: string; @@ -91,6 +93,7 @@ export interface SceneFusionNodeData { // 3D 模型节点数据 export interface Model3DNodeData { src: string; + assetId?: string; previewImage?: string; prompt?: string; modelVersion?: string;