Add multi-image reference context resolution
This commit is contained in:
134
parent
cd90c8074c
commit
cc80bd5dac
@ -124,6 +124,7 @@ import {
|
||||
shouldBypassCanvasImagePointsPrecheckForAgentControl,
|
||||
shouldPreserveCanvasAgentSlotOnlyContext,
|
||||
} from "@/lib/canvas/agent-client-intent";
|
||||
import { resolveMultiImageReferenceContext } from "@/lib/canvas/multi-image-reference-context";
|
||||
import { orderItemsBySelectionIds } from "@/lib/canvas/selection-order";
|
||||
import { getCanvasSkillIconComponent } from "@/components/canvas/canvas-skill-icons";
|
||||
import { ImageParamsBar, type ImageGenerationParams } from "@/components/image-params-bar";
|
||||
@ -6442,7 +6443,9 @@ export function CanvasChatPanel({
|
||||
isDetailGenerationMode,
|
||||
agentEnabled: effectiveAgentEnabled,
|
||||
});
|
||||
const selectedImageNodesForContext = Array.from(
|
||||
const MAX_CONTEXT_IMAGES = 12;
|
||||
const MAX_AGENT_CANDIDATE_NODES = 30;
|
||||
let selectedImageNodesForContext = Array.from(
|
||||
new Map(
|
||||
[
|
||||
...selectedImageNodes.filter((node) =>
|
||||
@ -6453,6 +6456,16 @@ export function CanvasChatPanel({
|
||||
).values()
|
||||
);
|
||||
|
||||
if (shouldIsolateDirectModeContext) {
|
||||
selectedImageNodesForContext = resolveMultiImageReferenceContext({
|
||||
text: input,
|
||||
selectedItems: selectedImageNodesForContext,
|
||||
candidateItems: imageNodes,
|
||||
getItemId: (node) => node.id,
|
||||
maxItems: 4,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
shouldIsolateDirectModeContext &&
|
||||
selectedImageNodesForContext.length === 0 &&
|
||||
@ -6463,8 +6476,6 @@ export function CanvasChatPanel({
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const MAX_CONTEXT_IMAGES = 12;
|
||||
const MAX_AGENT_CANDIDATE_NODES = 30;
|
||||
const selectedCanvasNodes = orderItemsBySelectionIds(
|
||||
nodes.filter((node) => selectedNodeIdSet.has(node.id)),
|
||||
effectiveSelectedNodeIds,
|
||||
|
||||
222
lib/canvas/multi-image-reference-context.ts
Normal file
222
lib/canvas/multi-image-reference-context.ts
Normal file
@ -0,0 +1,222 @@
|
||||
const ORDINAL_REFERENCE_PATTERNS = [
|
||||
/(?:第一|第\s*1|图\s*1|图片\s*1|照片\s*1|first\s+(?:image|photo|reference)|1st\s+(?:image|photo|reference))[\s\S]{0,120}(?:第二|第\s*2|图\s*2|图片\s*2|照片\s*2|second\s+(?:image|photo|reference)|2nd\s+(?:image|photo|reference))/i,
|
||||
/(?:第二|第\s*2|图\s*2|图片\s*2|照片\s*2|second\s+(?:image|photo|reference)|2nd\s+(?:image|photo|reference))[\s\S]{0,120}(?:第一|第\s*1|图\s*1|图片\s*1|照片\s*1|first\s+(?:image|photo|reference)|1st\s+(?:image|photo|reference))/i,
|
||||
/(?:第三|第\s*3|图\s*3|图片\s*3|照片\s*3|third\s+(?:image|photo|reference)|3rd\s+(?:image|photo|reference))/i,
|
||||
/(?:第四|第\s*4|图\s*4|图片\s*4|照片\s*4|fourth\s+(?:image|photo|reference)|4th\s+(?:image|photo|reference))/i,
|
||||
];
|
||||
|
||||
const MULTI_IMAGE_REFERENCE_PATTERNS = [
|
||||
...ORDINAL_REFERENCE_PATTERNS,
|
||||
/(?:左图|右图|上图|下图|左侧图|右侧图|左边的图|右边的图|左侧图片|右侧图片)/i,
|
||||
/(?:这|这些|这几|上述|以上|下面|上传|给的|参考|选中|画布|原图|素材)[\s\S]{0,18}(?:两|俩|二|三|四|五|六|七|八|2|3|4|5|6|7|8|多|几)[\s\S]{0,12}(?:张|个)?(?:图|图片|照片|参考图|素材)/i,
|
||||
/(?:两|俩|二|三|四|五|六|七|八|2|3|4|5|6|7|8|多|几)[\s\S]{0,8}(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图)/i,
|
||||
/(?:两|俩|二|三|四|五|六|七|八|2|3|4|5|6|7|8|多|几)[\s\S]{0,8}(?:张|个)?(?:图|图片|照片)[\s\S]{0,28}(?:合成|融合|拼接|对比|混合|组合|参考|编辑|处理|改)/i,
|
||||
/(?:合成|融合|拼接|对比|混合|组合|参考|编辑|处理)[\s\S]{0,28}(?:两|俩|二|三|四|五|六|七|八|2|3|4|5|6|7|8|多|几|第一|第二|第三|第四|图\s*1|图\s*2|图\s*3|图\s*4)[\s\S]{0,12}(?:图|图片|照片|参考|素材)?/i,
|
||||
/(?:multi|multiple|two|several|many)[\s\S]{0,20}(?:reference|input|source)[\s\S]{0,16}(?:image|photo)s?/i,
|
||||
/(?:reference|use|combine|merge|blend|edit)[\s\S]{0,24}(?:first|1st)[\s\S]{0,80}(?:second|2nd)[\s\S]{0,32}(?:image|photo|reference)?/i,
|
||||
];
|
||||
|
||||
const REQUESTED_COUNT_PATTERNS: Array<{ count: number; pattern: RegExp }> = [
|
||||
{
|
||||
count: 8,
|
||||
pattern: /(?:第\s*(?:八|8)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*8\b|(?:八|8)\s*(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图))/i,
|
||||
},
|
||||
{
|
||||
count: 7,
|
||||
pattern: /(?:第\s*(?:七|7)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*7\b|(?:七|7)\s*(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图))/i,
|
||||
},
|
||||
{
|
||||
count: 6,
|
||||
pattern: /(?:第\s*(?:六|6)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*6\b|(?:六|6)\s*(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图))/i,
|
||||
},
|
||||
{
|
||||
count: 5,
|
||||
pattern: /(?:第\s*(?:五|5)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*5\b|(?:五|5)\s*(?:张|个)?(?:参考图|素材图|输入图|源图|上传图|原图))/i,
|
||||
},
|
||||
{
|
||||
count: 4,
|
||||
pattern: /(?:第\s*(?:四|4)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*4\b|(?:四|4)\s*(?:张|个)?(?:图|图片|照片|参考图|素材图|输入图|源图|上传图|原图))/i,
|
||||
},
|
||||
{
|
||||
count: 3,
|
||||
pattern: /(?:第\s*(?:三|3)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*3\b|(?:三|3)\s*(?:张|个)?(?:图|图片|照片|参考图|素材图|输入图|源图|上传图|原图))/i,
|
||||
},
|
||||
{
|
||||
count: 2,
|
||||
pattern: /(?:第\s*(?:二|两|2)\s*(?:张|个)?(?:图|图片|照片)|(?:图|图片|照片|image)\s*2\b|(?:两|俩|二|2)\s*(?:张|个)?(?:图|图片|照片|参考图|素材图|输入图|源图|上传图|原图))/i,
|
||||
},
|
||||
];
|
||||
|
||||
function normalizeText(text: string) {
|
||||
return String(text || "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function normalizeItemId(value: string | null | undefined) {
|
||||
return String(value || "").trim();
|
||||
}
|
||||
|
||||
function clampMaxItems(value: number | undefined) {
|
||||
return Math.max(1, Math.min(8, Math.round(value || 4)));
|
||||
}
|
||||
|
||||
function dedupeItemsById<T>(
|
||||
items: T[],
|
||||
getItemId: (item: T) => string | null | undefined,
|
||||
) {
|
||||
const seen = new Set<string>();
|
||||
const result: T[] = [];
|
||||
|
||||
for (const item of items) {
|
||||
const id = normalizeItemId(getItemId(item));
|
||||
if (!id || seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
result.push(item);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function getItemIdSet<T>(
|
||||
items: T[],
|
||||
getItemId: (item: T) => string | null | undefined,
|
||||
) {
|
||||
return new Set(
|
||||
items
|
||||
.map((item) => normalizeItemId(getItemId(item)))
|
||||
.filter(Boolean),
|
||||
);
|
||||
}
|
||||
|
||||
function appendMissingItems<T>(params: {
|
||||
result: T[];
|
||||
candidates: T[];
|
||||
targetCount: number;
|
||||
getItemId: (item: T) => string | null | undefined;
|
||||
}) {
|
||||
const seen = getItemIdSet(params.result, params.getItemId);
|
||||
for (const item of params.candidates) {
|
||||
if (params.result.length >= params.targetCount) break;
|
||||
const id = normalizeItemId(params.getItemId(item));
|
||||
if (!id || seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
params.result.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
function getCandidateWindow<T>(params: {
|
||||
candidates: T[];
|
||||
selectedIds: Set<string>;
|
||||
targetCount: number;
|
||||
getItemId: (item: T) => string | null | undefined;
|
||||
}) {
|
||||
if (params.candidates.length === 0) return [];
|
||||
if (params.selectedIds.size === 0) {
|
||||
return params.candidates.slice(-params.targetCount);
|
||||
}
|
||||
|
||||
let selectedIndex = -1;
|
||||
params.candidates.forEach((item, index) => {
|
||||
const id = normalizeItemId(params.getItemId(item));
|
||||
if (id && params.selectedIds.has(id)) {
|
||||
selectedIndex = index;
|
||||
}
|
||||
});
|
||||
|
||||
if (selectedIndex < 0) return params.candidates.slice(-params.targetCount);
|
||||
|
||||
const start = Math.max(0, selectedIndex - params.targetCount + 1);
|
||||
const windowItems = params.candidates.slice(start, selectedIndex + 1);
|
||||
if (windowItems.length >= params.targetCount) return windowItems;
|
||||
|
||||
return [
|
||||
...windowItems,
|
||||
...params.candidates.slice(
|
||||
selectedIndex + 1,
|
||||
selectedIndex + 1 + (params.targetCount - windowItems.length),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
export function hasMultiImageReferenceIntent(text: string) {
|
||||
const normalized = normalizeText(text);
|
||||
if (!normalized) return false;
|
||||
return MULTI_IMAGE_REFERENCE_PATTERNS.some((pattern) => pattern.test(normalized));
|
||||
}
|
||||
|
||||
export function getRequestedImageReferenceCount(text: string) {
|
||||
const normalized = normalizeText(text);
|
||||
if (!normalized) return 0;
|
||||
if (!hasMultiImageReferenceIntent(normalized)) return 0;
|
||||
if (/(?:多图|多张|多份|multiple|several|many)/i.test(normalized)) return 4;
|
||||
|
||||
for (const item of REQUESTED_COUNT_PATTERNS) {
|
||||
if (item.pattern.test(normalized)) return item.count;
|
||||
}
|
||||
|
||||
return 2;
|
||||
}
|
||||
|
||||
export function resolveMultiImageReferenceContext<T>(params: {
|
||||
text: string;
|
||||
selectedItems: T[];
|
||||
candidateItems: T[];
|
||||
getItemId: (item: T) => string | null | undefined;
|
||||
maxItems?: number;
|
||||
}) {
|
||||
const maxItems = clampMaxItems(params.maxItems);
|
||||
const selectedItems = dedupeItemsById(params.selectedItems || [], params.getItemId).slice(
|
||||
0,
|
||||
maxItems,
|
||||
);
|
||||
if (!hasMultiImageReferenceIntent(params.text)) return selectedItems;
|
||||
|
||||
const candidateItems = dedupeItemsById(params.candidateItems || [], params.getItemId);
|
||||
const requestedCount = Math.min(
|
||||
maxItems,
|
||||
Math.max(2, selectedItems.length, getRequestedImageReferenceCount(params.text)),
|
||||
);
|
||||
|
||||
if (selectedItems.length >= requestedCount) {
|
||||
return selectedItems.slice(0, maxItems);
|
||||
}
|
||||
|
||||
const selectedIds = getItemIdSet(selectedItems, params.getItemId);
|
||||
const candidateWindow = getCandidateWindow({
|
||||
candidates: candidateItems,
|
||||
selectedIds,
|
||||
targetCount: requestedCount,
|
||||
getItemId: params.getItemId,
|
||||
});
|
||||
|
||||
if (selectedItems.length >= 2) {
|
||||
const result = selectedItems.slice(0, requestedCount);
|
||||
appendMissingItems({
|
||||
result,
|
||||
candidates: candidateWindow,
|
||||
targetCount: requestedCount,
|
||||
getItemId: params.getItemId,
|
||||
});
|
||||
appendMissingItems({
|
||||
result,
|
||||
candidates: candidateItems,
|
||||
targetCount: requestedCount,
|
||||
getItemId: params.getItemId,
|
||||
});
|
||||
return result.slice(0, maxItems);
|
||||
}
|
||||
|
||||
const result = candidateWindow.slice(0, requestedCount);
|
||||
appendMissingItems({
|
||||
result,
|
||||
candidates: selectedItems,
|
||||
targetCount: requestedCount,
|
||||
getItemId: params.getItemId,
|
||||
});
|
||||
appendMissingItems({
|
||||
result,
|
||||
candidates: candidateItems,
|
||||
targetCount: requestedCount,
|
||||
getItemId: params.getItemId,
|
||||
});
|
||||
return result.slice(0, maxItems);
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user