Refine multi-image target resolution
This commit is contained in:
parent
cc80bd5dac
commit
d10d638119
@ -6484,6 +6484,10 @@ export function CanvasChatPanel({
|
|||||||
const selectedContextNodes = selectedImageNodesForContext.slice(0, 20);
|
const selectedContextNodes = selectedImageNodesForContext.slice(0, 20);
|
||||||
generationAnchorNodesRef.current = selectedContextNodes.slice(0, 4);
|
generationAnchorNodesRef.current = selectedContextNodes.slice(0, 4);
|
||||||
const selectedIds = new Set(selectedContextNodes.map((node) => node.id));
|
const selectedIds = new Set(selectedContextNodes.map((node) => node.id));
|
||||||
|
const selectedPayloadNodeIds =
|
||||||
|
selectedContextNodes.length > 0
|
||||||
|
? selectedContextNodes.map((node) => node.id).slice(0, 20)
|
||||||
|
: selectedCanvasNodes.map((node) => node.id).slice(0, 20);
|
||||||
const prioritizedNodes =
|
const prioritizedNodes =
|
||||||
shouldIsolateDirectModeContext
|
shouldIsolateDirectModeContext
|
||||||
? selectedContextNodes
|
? selectedContextNodes
|
||||||
@ -6715,7 +6719,7 @@ export function CanvasChatPanel({
|
|||||||
return {
|
return {
|
||||||
source: "canvas" as const,
|
source: "canvas" as const,
|
||||||
projectId,
|
projectId,
|
||||||
selectedNodeIds: selectedCanvasNodes.map((node) => node.id).slice(0, 20),
|
selectedNodeIds: selectedPayloadNodeIds,
|
||||||
selectedAssetIds,
|
selectedAssetIds,
|
||||||
focusPolicy: agentFocusPolicy,
|
focusPolicy: agentFocusPolicy,
|
||||||
canvasImageNodes: contextNodes.map((node) => ({
|
canvasImageNodes: contextNodes.map((node) => ({
|
||||||
|
|||||||
@ -97,25 +97,76 @@ function getPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): stri
|
|||||||
return `${vertical}${horizontal}`;
|
return `${vertical}${horizontal}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function collectOrdinalIndexes(text: string): number[] {
|
||||||
|
const indexes = new Set<number>();
|
||||||
|
const normalizedText = String(text || "");
|
||||||
|
const chineseOrdinalPattern =
|
||||||
|
/第\s*([一二三四五六七八九十两\d]+)\s*(?:张|个|幅)?\s*(?:图|图片|照片)?/gi;
|
||||||
|
const numberedImagePattern = /(?:图|图片|照片|image|photo|reference)\s*([1-9]\d*)\b/gi;
|
||||||
|
const englishOrdinals: Array<{ index: number; pattern: RegExp }> = [
|
||||||
|
{ index: 0, pattern: /\b(?:first|1st)\s+(?:image|photo|reference)\b/i },
|
||||||
|
{ index: 1, pattern: /\b(?:second|2nd)\s+(?:image|photo|reference)\b/i },
|
||||||
|
{ index: 2, pattern: /\b(?:third|3rd)\s+(?:image|photo|reference)\b/i },
|
||||||
|
{ index: 3, pattern: /\b(?:fourth|4th)\s+(?:image|photo|reference)\b/i },
|
||||||
|
{ index: 4, pattern: /\b(?:fifth|5th)\s+(?:image|photo|reference)\b/i },
|
||||||
|
{ index: 5, pattern: /\b(?:sixth|6th)\s+(?:image|photo|reference)\b/i },
|
||||||
|
{ index: 6, pattern: /\b(?:seventh|7th)\s+(?:image|photo|reference)\b/i },
|
||||||
|
{ index: 7, pattern: /\b(?:eighth|8th)\s+(?:image|photo|reference)\b/i },
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const match of normalizedText.matchAll(chineseOrdinalPattern)) {
|
||||||
|
const numeric = chineseNumberToInt(match[1] || "");
|
||||||
|
if (numeric && numeric > 0) indexes.add(numeric - 1);
|
||||||
|
}
|
||||||
|
for (const match of normalizedText.matchAll(numberedImagePattern)) {
|
||||||
|
const numeric = Number.parseInt(match[1] || "", 10);
|
||||||
|
if (Number.isFinite(numeric) && numeric > 0) indexes.add(numeric - 1);
|
||||||
|
}
|
||||||
|
for (const item of englishOrdinals) {
|
||||||
|
if (item.pattern.test(normalizedText)) indexes.add(item.index);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array.from(indexes).sort((left, right) => left - right);
|
||||||
|
}
|
||||||
|
|
||||||
function pickByOrdinal(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] {
|
function pickByOrdinal(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] {
|
||||||
const matched = text.match(/第\s*([一二三四五六七八九十两\d]+)\s*(?:张|个|幅|图)/i);
|
const ordinalIndexes = collectOrdinalIndexes(text);
|
||||||
const numeric = matched ? chineseNumberToInt(matched[1]) : null;
|
if (ordinalIndexes.length === 0) return [];
|
||||||
if (!numeric || numeric <= 0) return [];
|
|
||||||
const ordered = sortRefsForCanvasOrdering(refs);
|
const ordered = sortRefsForCanvasOrdering(refs);
|
||||||
return ordered[numeric - 1] ? [ordered[numeric - 1]] : [];
|
return ordinalIndexes
|
||||||
|
.map((index) => ordered[index])
|
||||||
|
.filter((ref): ref is ResolvedImageRef => Boolean(ref));
|
||||||
}
|
}
|
||||||
|
|
||||||
function pickBySpatialHint(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] {
|
function pickBySpatialHint(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] {
|
||||||
if (!refs.length) return [];
|
if (!refs.length) return [];
|
||||||
|
|
||||||
const normalizedText = text.replace(/\s+/g, "");
|
const normalizedText = text.replace(/\s+/g, "");
|
||||||
const hasLeft = /左边|左侧|左面|左上|左下|靠左/.test(normalizedText);
|
const hasLeftTop = /左上/.test(normalizedText);
|
||||||
const hasRight = /右边|右侧|右面|右上|右下|靠右/.test(normalizedText);
|
const hasLeftBottom = /左下/.test(normalizedText);
|
||||||
const hasTop = /上面|上方|上边|上侧|顶部|靠上|最上/.test(normalizedText);
|
const hasRightTop = /右上/.test(normalizedText);
|
||||||
const hasBottom = /下面|下方|下边|下侧|底部|靠下|最下/.test(normalizedText);
|
const hasRightBottom = /右下/.test(normalizedText);
|
||||||
|
const hasLeft = /左图|左边|左侧|左面|靠左/.test(normalizedText);
|
||||||
|
const hasRight = /右图|右边|右侧|右面|靠右/.test(normalizedText);
|
||||||
|
const hasTop = /上图|上面|上方|上边|上侧|顶部|靠上|最上/.test(normalizedText);
|
||||||
|
const hasBottom = /下图|下面|下方|下边|下侧|底部|靠下|最下/.test(normalizedText);
|
||||||
const hasCenter = /中间|居中|正中|中央/.test(normalizedText);
|
const hasCenter = /中间|居中|正中|中央/.test(normalizedText);
|
||||||
|
|
||||||
if (!(hasLeft || hasRight || hasTop || hasBottom || hasCenter)) return [];
|
if (
|
||||||
|
!(
|
||||||
|
hasLeftTop ||
|
||||||
|
hasLeftBottom ||
|
||||||
|
hasRightTop ||
|
||||||
|
hasRightBottom ||
|
||||||
|
hasLeft ||
|
||||||
|
hasRight ||
|
||||||
|
hasTop ||
|
||||||
|
hasBottom ||
|
||||||
|
hasCenter
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
const withLabels = refs.map((ref) => ({
|
const withLabels = refs.map((ref) => ({
|
||||||
ref,
|
ref,
|
||||||
@ -124,12 +175,18 @@ function pickBySpatialHint(text: string, refs: ResolvedImageRef[]): ResolvedImag
|
|||||||
|
|
||||||
return withLabels
|
return withLabels
|
||||||
.filter(({ label }) => {
|
.filter(({ label }) => {
|
||||||
if (hasLeft && !label.includes("左")) return false;
|
const matchesCompound =
|
||||||
if (hasRight && !label.includes("右")) return false;
|
(hasLeftTop && label.includes("左") && label.includes("上")) ||
|
||||||
if (hasTop && !label.includes("上")) return false;
|
(hasLeftBottom && label.includes("左") && label.includes("下")) ||
|
||||||
if (hasBottom && !label.includes("下")) return false;
|
(hasRightTop && label.includes("右") && label.includes("上")) ||
|
||||||
if (hasCenter && !(label.includes("中") || label === "中间")) return false;
|
(hasRightBottom && label.includes("右") && label.includes("下"));
|
||||||
return true;
|
const matchesAxis =
|
||||||
|
(hasLeft && label.includes("左")) ||
|
||||||
|
(hasRight && label.includes("右")) ||
|
||||||
|
(hasTop && label.includes("上")) ||
|
||||||
|
(hasBottom && label.includes("下")) ||
|
||||||
|
(hasCenter && (label.includes("中") || label === "中间"));
|
||||||
|
return matchesCompound || matchesAxis;
|
||||||
})
|
})
|
||||||
.map(({ ref }) => ref);
|
.map(({ ref }) => ref);
|
||||||
}
|
}
|
||||||
@ -204,6 +261,22 @@ export function resolveTargetAssets(
|
|||||||
);
|
);
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
const ordinalRefs = pickByOrdinal(text, canvasRefs);
|
||||||
|
if (ordinalRefs.length > 0) {
|
||||||
|
return buildResolvedTargetAssets({
|
||||||
|
targetRefs: ordinalRefs,
|
||||||
|
candidateRefs,
|
||||||
|
resolutionMode: "text_reference",
|
||||||
|
confidence: ordinalRefs.length > 1 ? 91 : 88,
|
||||||
|
needsClarification: false,
|
||||||
|
reason:
|
||||||
|
ordinalRefs.length > 1
|
||||||
|
? "根据用户文本中的多个序号定位到多张画布图片"
|
||||||
|
: "根据用户文本中的序号定位到画布图片",
|
||||||
|
fromHistory: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
if (ORIGINAL_IMAGE_REGEX.test(text) && originalRefs.length > 0) {
|
if (ORIGINAL_IMAGE_REGEX.test(text) && originalRefs.length > 0) {
|
||||||
return buildResolvedTargetAssets({
|
return buildResolvedTargetAssets({
|
||||||
targetRefs: originalRefs,
|
targetRefs: originalRefs,
|
||||||
@ -252,19 +325,6 @@ export function resolveTargetAssets(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const ordinalRefs = pickByOrdinal(text, canvasRefs);
|
|
||||||
if (ordinalRefs.length > 0) {
|
|
||||||
return buildResolvedTargetAssets({
|
|
||||||
targetRefs: ordinalRefs,
|
|
||||||
candidateRefs,
|
|
||||||
resolutionMode: "text_reference",
|
|
||||||
confidence: 88,
|
|
||||||
needsClarification: false,
|
|
||||||
reason: "根据用户文本中的序号定位到画布图片",
|
|
||||||
fromHistory: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const spatialRefs = pickBySpatialHint(text, canvasRefs);
|
const spatialRefs = pickBySpatialHint(text, canvasRefs);
|
||||||
if (spatialRefs.length > 0) {
|
if (spatialRefs.length > 0) {
|
||||||
return buildResolvedTargetAssets({
|
return buildResolvedTargetAssets({
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user