From cc80bd5dac8baaa550d606be95abea6cfaa9ba45 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 24 Jun 2026 18:58:38 +0800 Subject: [PATCH] Add multi-image reference context resolution --- components/canvas/CanvasChatPanel.tsx | 17 +- lib/canvas/multi-image-reference-context.ts | 222 ++++++++++++++++++++ 2 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 lib/canvas/multi-image-reference-context.ts diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index d8623820f..081a25c47 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -124,6 +124,7 @@ import { shouldBypassCanvasImagePointsPrecheckForAgentControl, shouldPreserveCanvasAgentSlotOnlyContext, } from "@/lib/canvas/agent-client-intent"; +import { resolveMultiImageReferenceContext } from "@/lib/canvas/multi-image-reference-context"; import { orderItemsBySelectionIds } from "@/lib/canvas/selection-order"; import { getCanvasSkillIconComponent } from "@/components/canvas/canvas-skill-icons"; import { ImageParamsBar, type ImageGenerationParams } from "@/components/image-params-bar"; @@ -6442,7 +6443,9 @@ export function CanvasChatPanel({ isDetailGenerationMode, agentEnabled: effectiveAgentEnabled, }); - const selectedImageNodesForContext = Array.from( + const MAX_CONTEXT_IMAGES = 12; + const MAX_AGENT_CANDIDATE_NODES = 30; + let selectedImageNodesForContext = Array.from( new Map( [ ...selectedImageNodes.filter((node) => @@ -6453,6 +6456,16 @@ export function CanvasChatPanel({ ).values() ); + if (shouldIsolateDirectModeContext) { + selectedImageNodesForContext = resolveMultiImageReferenceContext({ + text: input, + selectedItems: selectedImageNodesForContext, + candidateItems: imageNodes, + getItemId: (node) => node.id, + maxItems: 4, + }); + } + if ( shouldIsolateDirectModeContext && selectedImageNodesForContext.length === 0 && @@ -6463,8 +6476,6 @@ export function CanvasChatPanel({ return undefined; } - const MAX_CONTEXT_IMAGES = 12; - const MAX_AGENT_CANDIDATE_NODES = 30; const selectedCanvasNodes = orderItemsBySelectionIds( nodes.filter((node) => selectedNodeIdSet.has(node.id)), effectiveSelectedNodeIds, diff --git a/lib/canvas/multi-image-reference-context.ts b/lib/canvas/multi-image-reference-context.ts new file mode 100644 index 000000000..04c86e111 --- /dev/null +++ b/lib/canvas/multi-image-reference-context.ts @@ -0,0 +1,222 @@ +const ORDINAL_REFERENCE_PATTERNS = [ + /(?:第一|第\s*1|图\s*1|图片\s*1|照片\s*1|first\s+(?:image|photo|reference)|1st\s+(?:image|photo|reference))[\s\S]{0,120}(?:第二|第\s*2|图\s*2|图片\s*2|照片\s*2|second\s+(?:image|photo|reference)|2nd\s+(?:image|photo|reference))/i, + /(?:第二|第\s*2|图\s*2|图片\s*2|照片\s*2|second\s+(?:image|photo|reference)|2nd\s+(?:image|photo|reference))[\s\S]{0,120}(?:第一|第\s*1|图\s*1|图片\s*1|照片\s*1|first\s+(?:image|photo|reference)|1st\s+(?:image|photo|reference))/i, + /(?:第三|第\s*3|图\s*3|图片\s*3|照片\s*3|third\s+(?:image|photo|reference)|3rd\s+(?:image|photo|reference))/i, + /(?:第四|第\s*4|图\s*4|图片\s*4|照片\s*4|fourth\s+(?:image|photo|reference)|4th\s+(?:image|photo|reference))/i, +]; + +const MULTI_IMAGE_REFERENCE_PATTERNS = [ + ...ORDINAL_REFERENCE_PATTERNS, + /(?:左图|右图|上图|下图|左侧图|右侧图|左边的图|右边的图|左侧图片|右侧图片)/i, + /(?:这|这些|这几|上述|以上|下面|上传|给的|参考|选中|画布|原图|素材)[\s\S]{0,18}(?:两|俩|二|三|四|五|六|七|八|2|3|4|5|6|7|8|多|几)[\s\S]{0,12}(?:张|个)?(?:图|图片|照片|参考图|素材)/i, + /(?:两|俩|二|三|四|五|六|七|八|2|3|4|5|6|7|8|多|几)[\s\S]{0,8}(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图)/i, + /(?:两|俩|二|三|四|五|六|七|八|2|3|4|5|6|7|8|多|几)[\s\S]{0,8}(?:张|个)?(?:图|图片|照片)[\s\S]{0,28}(?:合成|融合|拼接|对比|混合|组合|参考|编辑|处理|改)/i, + /(?:合成|融合|拼接|对比|混合|组合|参考|编辑|处理)[\s\S]{0,28}(?:两|俩|二|三|四|五|六|七|八|2|3|4|5|6|7|8|多|几|第一|第二|第三|第四|图\s*1|图\s*2|图\s*3|图\s*4)[\s\S]{0,12}(?:图|图片|照片|参考|素材)?/i, + /(?:multi|multiple|two|several|many)[\s\S]{0,20}(?:reference|input|source)[\s\S]{0,16}(?:image|photo)s?/i, + /(?:reference|use|combine|merge|blend|edit)[\s\S]{0,24}(?:first|1st)[\s\S]{0,80}(?:second|2nd)[\s\S]{0,32}(?:image|photo|reference)?/i, +]; + +const REQUESTED_COUNT_PATTERNS: Array<{ count: number; pattern: RegExp }> = [ + { + count: 8, + pattern: /(?:第\s*(?:八|8)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*8\b|(?:八|8)\s*(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图))/i, + }, + { + count: 7, + pattern: /(?:第\s*(?:七|7)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*7\b|(?:七|7)\s*(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图))/i, + }, + { + count: 6, + pattern: /(?:第\s*(?:六|6)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*6\b|(?:六|6)\s*(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图))/i, + }, + { + count: 5, + pattern: /(?:第\s*(?:五|5)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*5\b|(?:五|5)\s*(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图))/i, + }, + { + count: 4, + pattern: /(?:第\s*(?:四|4)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*4\b|(?:四|4)\s*(?:张|个)?(?:图|图片|照片|参考图|素材图|输入图|源图|上传图|原图))/i, + }, + { + count: 3, + pattern: /(?:第\s*(?:三|3)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*3\b|(?:三|3)\s*(?:张|个)?(?:图|图片|照片|参考图|素材图|输入图|源图|上传图|原图))/i, + }, + { + count: 2, + pattern: /(?:第\s*(?:二|两|2)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*2\b|(?:两|俩|二|2)\s*(?:张|个)?(?:图|图片|照片|参考图|素材图|输入图|源图|上传图|原图))/i, + }, +]; + +function normalizeText(text: string) { + return String(text || "").replace(/\s+/g, " ").trim(); +} + +function normalizeItemId(value: string | null | undefined) { + return String(value || "").trim(); +} + +function clampMaxItems(value: number | undefined) { + return Math.max(1, Math.min(8, Math.round(value || 4))); +} + +function dedupeItemsById( + items: T[], + getItemId: (item: T) => string | null | undefined, +) { + const seen = new Set(); + const result: T[] = []; + + for (const item of items) { + const id = normalizeItemId(getItemId(item)); + if (!id || seen.has(id)) continue; + seen.add(id); + result.push(item); + } + + return result; +} + +function getItemIdSet( + items: T[], + getItemId: (item: T) => string | null | undefined, +) { + return new Set( + items + .map((item) => normalizeItemId(getItemId(item))) + .filter(Boolean), + ); +} + +function appendMissingItems(params: { + result: T[]; + candidates: T[]; + targetCount: number; + getItemId: (item: T) => string | null | undefined; +}) { + const seen = getItemIdSet(params.result, params.getItemId); + for (const item of params.candidates) { + if (params.result.length >= params.targetCount) break; + const id = normalizeItemId(params.getItemId(item)); + if (!id || seen.has(id)) continue; + seen.add(id); + params.result.push(item); + } +} + +function getCandidateWindow(params: { + candidates: T[]; + selectedIds: Set; + targetCount: number; + getItemId: (item: T) => string | null | undefined; +}) { + if (params.candidates.length === 0) return []; + if (params.selectedIds.size === 0) { + return params.candidates.slice(-params.targetCount); + } + + let selectedIndex = -1; + params.candidates.forEach((item, index) => { + const id = normalizeItemId(params.getItemId(item)); + if (id && params.selectedIds.has(id)) { + selectedIndex = index; + } + }); + + if (selectedIndex < 0) return params.candidates.slice(-params.targetCount); + + const start = Math.max(0, selectedIndex - params.targetCount + 1); + const windowItems = params.candidates.slice(start, selectedIndex + 1); + if (windowItems.length >= params.targetCount) return windowItems; + + return [ + ...windowItems, + ...params.candidates.slice( + selectedIndex + 1, + selectedIndex + 1 + (params.targetCount - windowItems.length), + ), + ]; +} + +export function hasMultiImageReferenceIntent(text: string) { + const normalized = normalizeText(text); + if (!normalized) return false; + return MULTI_IMAGE_REFERENCE_PATTERNS.some((pattern) => pattern.test(normalized)); +} + +export function getRequestedImageReferenceCount(text: string) { + const normalized = normalizeText(text); + if (!normalized) return 0; + if (!hasMultiImageReferenceIntent(normalized)) return 0; + if (/(?:多图|多张|多份|multiple|several|many)/i.test(normalized)) return 4; + + for (const item of REQUESTED_COUNT_PATTERNS) { + if (item.pattern.test(normalized)) return item.count; + } + + return 2; +} + +export function resolveMultiImageReferenceContext(params: { + text: string; + selectedItems: T[]; + candidateItems: T[]; + getItemId: (item: T) => string | null | undefined; + maxItems?: number; +}) { + const maxItems = clampMaxItems(params.maxItems); + const selectedItems = dedupeItemsById(params.selectedItems || [], params.getItemId).slice( + 0, + maxItems, + ); + if (!hasMultiImageReferenceIntent(params.text)) return selectedItems; + + const candidateItems = dedupeItemsById(params.candidateItems || [], params.getItemId); + const requestedCount = Math.min( + maxItems, + Math.max(2, selectedItems.length, getRequestedImageReferenceCount(params.text)), + ); + + if (selectedItems.length >= requestedCount) { + return selectedItems.slice(0, maxItems); + } + + const selectedIds = getItemIdSet(selectedItems, params.getItemId); + const candidateWindow = getCandidateWindow({ + candidates: candidateItems, + selectedIds, + targetCount: requestedCount, + getItemId: params.getItemId, + }); + + if (selectedItems.length >= 2) { + const result = selectedItems.slice(0, requestedCount); + appendMissingItems({ + result, + candidates: candidateWindow, + targetCount: requestedCount, + getItemId: params.getItemId, + }); + appendMissingItems({ + result, + candidates: candidateItems, + targetCount: requestedCount, + getItemId: params.getItemId, + }); + return result.slice(0, maxItems); + } + + const result = candidateWindow.slice(0, requestedCount); + appendMissingItems({ + result, + candidates: selectedItems, + targetCount: requestedCount, + getItemId: params.getItemId, + }); + appendMissingItems({ + result, + candidates: candidateItems, + targetCount: requestedCount, + getItemId: params.getItemId, + }); + return result.slice(0, maxItems); +}