539 lines
18 KiB
TypeScript
539 lines
18 KiB
TypeScript
import "server-only";
|
|
|
|
import type {
|
|
ResolveTargetAssetsParams,
|
|
ResolvedImageRef,
|
|
ResolvedTargetAssets,
|
|
} from "@/lib/ai/context-memory/types";
|
|
import {
|
|
hasCanvasAgentActionableImageEditIntent,
|
|
hasCanvasAgentDeicticTarget,
|
|
resolveCanvasAgentTargetPreference,
|
|
} from "@/lib/ai/runtime/canvas-agent-kernel";
|
|
import { collectOrdinalImageIndexes } from "@/lib/ai/context-memory/ordinal-image-references";
|
|
|
|
const ORIGINAL_IMAGE_REGEX =
|
|
/(原图|上传图|参考图|最初那张|第一张图|素材图|源图|原始图片|原始那张|我发的那张|上传的那张|原来的图)/i;
|
|
const LATEST_RESULT_REGEX =
|
|
/(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果|上一版|刚才那张结果|生成的图|生成图|结果图|出图结果|刚出的图|上一张生成图|上一轮结果图|现在这张生成图)/i;
|
|
const SELECTION_REGEX = /(选中的|当前这张|我选的|这几张|这些图|选中图)/i;
|
|
const ACTIONABLE_GENERATION_REGEX =
|
|
/(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i;
|
|
const IMPLICIT_HISTORY_TARGET_REGEX =
|
|
/(继续|接着|沿用|按刚才|照刚才|上次|上一轮|上一版|刚才|刚生成|最新生成|结果图|生成图|这张|这个|当前|原图|上传图|选中)/i;
|
|
|
|
function normalizeStringArray(values?: string[] | null): string[] {
|
|
if (!Array.isArray(values)) return [];
|
|
return Array.from(
|
|
new Set(
|
|
values
|
|
.filter((value): value is string => typeof value === "string")
|
|
.map((value) => value.trim())
|
|
.filter(Boolean),
|
|
),
|
|
);
|
|
}
|
|
|
|
function sortRefsForCanvasOrdering(refs: ResolvedImageRef[]) {
|
|
return [...refs].sort((left, right) => {
|
|
const leftY = Number(left.y ?? 0);
|
|
const rightY = Number(right.y ?? 0);
|
|
if (Math.abs(leftY - rightY) > 12) {
|
|
return leftY - rightY;
|
|
}
|
|
return Number(left.x ?? 0) - Number(right.x ?? 0);
|
|
});
|
|
}
|
|
|
|
function getPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): string {
|
|
if (refs.length <= 1) return "中间";
|
|
const xs = refs.map((item) => Number(item.x ?? 0));
|
|
const ys = refs.map((item) => Number(item.y ?? 0));
|
|
const minX = Math.min(...xs);
|
|
const maxX = Math.max(...xs);
|
|
const minY = Math.min(...ys);
|
|
const maxY = Math.max(...ys);
|
|
const spanX = Math.max(1, maxX - minX);
|
|
const spanY = Math.max(1, maxY - minY);
|
|
const relX = (Number(ref.x ?? 0) - minX) / spanX;
|
|
const relY = (Number(ref.y ?? 0) - minY) / spanY;
|
|
|
|
const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中";
|
|
const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中";
|
|
|
|
if (horizontal === "中" && vertical === "中") return "中间";
|
|
return `${vertical}${horizontal}`;
|
|
}
|
|
|
|
function pickByOrdinal(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] {
|
|
const ordinalIndexes = collectOrdinalImageIndexes(text);
|
|
if (ordinalIndexes.length === 0) return [];
|
|
const ordered = sortRefsForCanvasOrdering(refs);
|
|
return ordinalIndexes
|
|
.map((index) => ordered[index])
|
|
.filter((ref): ref is ResolvedImageRef => Boolean(ref));
|
|
}
|
|
|
|
function pickBySpatialHint(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] {
|
|
if (!refs.length) return [];
|
|
|
|
const normalizedText = text.replace(/\s+/g, "");
|
|
const hasLeftTop = /左上/.test(normalizedText);
|
|
const hasLeftBottom = /左下/.test(normalizedText);
|
|
const hasRightTop = /右上/.test(normalizedText);
|
|
const hasRightBottom = /右下/.test(normalizedText);
|
|
const hasLeft = /左图|左边|左侧|左面|靠左/.test(normalizedText);
|
|
const hasRight = /右图|右边|右侧|右面|靠右/.test(normalizedText);
|
|
const hasTop = /上图|上面|上方|上边|上侧|顶部|靠上|最上/.test(normalizedText);
|
|
const hasBottom = /下图|下面|下方|下边|下侧|底部|靠下|最下/.test(normalizedText);
|
|
const hasCenter = /中间|居中|正中|中央/.test(normalizedText);
|
|
|
|
if (
|
|
!(
|
|
hasLeftTop ||
|
|
hasLeftBottom ||
|
|
hasRightTop ||
|
|
hasRightBottom ||
|
|
hasLeft ||
|
|
hasRight ||
|
|
hasTop ||
|
|
hasBottom ||
|
|
hasCenter
|
|
)
|
|
) {
|
|
return [];
|
|
}
|
|
|
|
const withLabels = refs.map((ref) => ({
|
|
ref,
|
|
label: getPositionLabel(ref, refs),
|
|
}));
|
|
|
|
return withLabels
|
|
.filter(({ label }) => {
|
|
const matchesCompound =
|
|
(hasLeftTop && label.includes("左") && label.includes("上")) ||
|
|
(hasLeftBottom && label.includes("左") && label.includes("下")) ||
|
|
(hasRightTop && label.includes("右") && label.includes("上")) ||
|
|
(hasRightBottom && label.includes("右") && label.includes("下"));
|
|
const matchesAxis =
|
|
(hasLeft && label.includes("左")) ||
|
|
(hasRight && label.includes("右")) ||
|
|
(hasTop && label.includes("上")) ||
|
|
(hasBottom && label.includes("下")) ||
|
|
(hasCenter && (label.includes("中") || label === "中间"));
|
|
return matchesCompound || matchesAxis;
|
|
})
|
|
.map(({ ref }) => ref);
|
|
}
|
|
|
|
function buildResolvedTargetAssets(params: {
|
|
targetRefs: ResolvedImageRef[];
|
|
candidateRefs: ResolvedImageRef[];
|
|
resolutionMode: ResolvedTargetAssets["resolutionMode"];
|
|
confidence: number;
|
|
needsClarification: boolean;
|
|
reason: string;
|
|
fromHistory: boolean;
|
|
}): ResolvedTargetAssets {
|
|
const targetRefs = params.targetRefs;
|
|
const candidateRefs = params.candidateRefs;
|
|
return {
|
|
targetRefs,
|
|
candidateRefs,
|
|
targetImageUrls: targetRefs.map((ref) => ref.url),
|
|
targetAssetIds: normalizeStringArray(targetRefs.map((ref) => ref.assetId || "")),
|
|
candidateImageUrls: candidateRefs.map((ref) => ref.url),
|
|
candidateAssetIds: normalizeStringArray(candidateRefs.map((ref) => ref.assetId || "")),
|
|
resolutionMode: params.resolutionMode,
|
|
confidence: Math.max(0, Math.min(100, Math.round(params.confidence))),
|
|
needsClarification: params.needsClarification,
|
|
reason: params.reason,
|
|
fromHistory: params.fromHistory,
|
|
};
|
|
}
|
|
|
|
function toWorkingMemoryRefs(urls: string[]): ResolvedImageRef[] {
|
|
return urls.map((url) => ({
|
|
url,
|
|
normalizedUrl: url,
|
|
source: "working_memory" as const,
|
|
role: "working_memory" as const,
|
|
}));
|
|
}
|
|
|
|
function buildNormalizedUrlSet(refs: ResolvedImageRef[]): Set<string> {
|
|
return new Set(
|
|
refs
|
|
.map((ref) => String(ref.normalizedUrl || ref.url || "").trim())
|
|
.filter(Boolean),
|
|
);
|
|
}
|
|
|
|
export function resolveTargetAssets(
|
|
params: ResolveTargetAssetsParams,
|
|
): ResolvedTargetAssets {
|
|
const text = String(params.text || "").trim();
|
|
const mediaInput = params.mediaInput;
|
|
const canvasRefs = mediaInput.canvasImageRefs;
|
|
const selectedRefs = mediaInput.selectedImageRefs;
|
|
const currentRefs = mediaInput.currentMessageImageRefs;
|
|
const assistantRefs = mediaInput.historyAssistantImageRefs;
|
|
const userRefs = mediaInput.historyUserImageRefs;
|
|
const candidateRefs = mediaInput.imageRefs;
|
|
const workingState = params.workingState;
|
|
const focusPolicy = params.contextPayload?.focusPolicy || "auto";
|
|
const workingStateGeneratedUrls = normalizeStringArray(workingState?.lastGeneratedImageUrls);
|
|
const workingStateUploadedUrls = normalizeStringArray(workingState?.lastUploadedImageUrls);
|
|
const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls);
|
|
const generatedRefs =
|
|
workingStateGeneratedUrls.length > 0 ? toWorkingMemoryRefs(workingStateGeneratedUrls) : assistantRefs;
|
|
const originalRefs =
|
|
workingStateUploadedUrls.length > 0 ? toWorkingMemoryRefs(workingStateUploadedUrls) : userRefs;
|
|
const selectedLooksLikeOriginal = (() => {
|
|
if (selectedRefs.length === 0 || originalRefs.length === 0) return false;
|
|
const originalKeys = buildNormalizedUrlSet(originalRefs);
|
|
return selectedRefs.every((ref) =>
|
|
originalKeys.has(String(ref.normalizedUrl || ref.url || "").trim()),
|
|
);
|
|
})();
|
|
|
|
const currentOrdinalRefs = pickByOrdinal(text, currentRefs);
|
|
if (currentOrdinalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: currentOrdinalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "current_upload",
|
|
confidence: currentOrdinalRefs.length > 1 ? 96 : 94,
|
|
needsClarification: false,
|
|
reason:
|
|
currentOrdinalRefs.length > 1
|
|
? "根据用户文本中的多个序号定位到本轮上传的多张图片"
|
|
: "根据用户文本中的序号定位到本轮上传图片",
|
|
fromHistory: false,
|
|
});
|
|
}
|
|
|
|
if (currentRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: currentRefs,
|
|
candidateRefs,
|
|
resolutionMode: "current_upload",
|
|
confidence: 93,
|
|
needsClarification: false,
|
|
reason: "本轮消息包含图片输入，默认优先处理当前上传图片",
|
|
fromHistory: false,
|
|
});
|
|
}
|
|
|
|
const ordinalRefs = pickByOrdinal(text, canvasRefs);
|
|
if (ordinalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: ordinalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: ordinalRefs.length > 1 ? 91 : 88,
|
|
needsClarification: false,
|
|
reason:
|
|
ordinalRefs.length > 1
|
|
? "根据用户文本中的多个序号定位到多张画布图片"
|
|
: "根据用户文本中的序号定位到画布图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (ORIGINAL_IMAGE_REGEX.test(text) && originalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: originalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 95,
|
|
needsClarification: false,
|
|
reason: "用户文本明确指向原图/上传图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (LATEST_RESULT_REGEX.test(text) && generatedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "latest_result",
|
|
confidence: 95,
|
|
needsClarification: false,
|
|
reason: "用户文本明确指向上一轮结果/生成图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (focusPolicy === "original" && originalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: originalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 96,
|
|
needsClarification: false,
|
|
reason: "前端上下文指定优先使用原图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (focusPolicy === "latest_result" && generatedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "latest_result",
|
|
confidence: 96,
|
|
needsClarification: false,
|
|
reason: "前端上下文指定优先使用上一轮结果",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
const spatialRefs = pickBySpatialHint(text, canvasRefs);
|
|
if (spatialRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: spatialRefs,
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: spatialRefs.length === 1 ? 90 : 76,
|
|
needsClarification: false,
|
|
reason: "根据用户文本中的方位指向定位到画布图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (focusPolicy === "selected" && selectedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: selectedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "explicit_selection",
|
|
confidence: 97,
|
|
needsClarification: false,
|
|
reason: "前端已明确选中目标图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
const canvasAgentTargetPreference = resolveCanvasAgentTargetPreference({
|
|
text,
|
|
selectedImageCount: selectedRefs.length,
|
|
latestResultImageCount: generatedRefs.length,
|
|
workingMemoryTargetImageCount: workingStateResolvedUrls.length,
|
|
});
|
|
if (canvasAgentTargetPreference.shouldPreferLatestResult) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode:
|
|
workingStateGeneratedUrls.length > 0 ? "working_memory" : "latest_result",
|
|
confidence: canvasAgentTargetPreference.confidence,
|
|
needsClarification: false,
|
|
reason: "用户使用指代表达并提出图片编辑意图，默认续接最近生成结果",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (canvasAgentTargetPreference.shouldPreferWorkingMemoryTarget) {
|
|
const refs = toWorkingMemoryRefs(workingStateResolvedUrls);
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: refs,
|
|
candidateRefs,
|
|
resolutionMode: "working_memory",
|
|
confidence: canvasAgentTargetPreference.confidence,
|
|
needsClarification: false,
|
|
reason: "用户使用指代表达并提出图片编辑意图，使用工作记忆中的上一轮目标图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (hasCanvasAgentDeicticTarget(text)) {
|
|
if (selectedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: selectedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "explicit_selection",
|
|
confidence: 93,
|
|
needsClarification: false,
|
|
reason: "用户使用“这张/这只/这个”等指代表达，且当前存在明确选中目标图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (candidateRefs.length === 1) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [candidateRefs[0]],
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 84,
|
|
needsClarification: false,
|
|
reason: "用户使用指代表达，且当前仅有一张候选图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [],
|
|
candidateRefs,
|
|
resolutionMode: "clarification",
|
|
confidence: 0,
|
|
needsClarification: true,
|
|
reason: "用户使用“这张/这只/这个”等指代表达，但当前没有明确选中唯一目标图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (
|
|
hasCanvasAgentActionableImageEditIntent(text) &&
|
|
currentRefs.length === 0 &&
|
|
generatedRefs.length > 0 &&
|
|
selectedLooksLikeOriginal &&
|
|
!SELECTION_REGEX.test(text) &&
|
|
!ORIGINAL_IMAGE_REGEX.test(text)
|
|
) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "latest_result",
|
|
confidence: 89,
|
|
needsClarification: false,
|
|
reason: "连续对话默认续接上一轮生成结果，避免误回退到原始选中图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if ((SELECTION_REGEX.test(text) || focusPolicy === "selected") && selectedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: selectedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "explicit_selection",
|
|
confidence: 92,
|
|
needsClarification: false,
|
|
reason: "用户文本或上下文明确要求使用当前选中的图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (selectedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: selectedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "explicit_selection",
|
|
confidence: 86,
|
|
needsClarification: false,
|
|
reason: "当前存在画布选中图片，默认优先继续处理选中目标",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (
|
|
canvasRefs.length === 1 &&
|
|
(hasCanvasAgentActionableImageEditIntent(text) || focusPolicy === "auto")
|
|
) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [canvasRefs[0]],
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 84,
|
|
needsClarification: false,
|
|
reason: "用户使用画布/图片指代表达，且当前画布仅有一张可处理图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
const allowsImplicitHistoryTarget =
|
|
IMPLICIT_HISTORY_TARGET_REGEX.test(text) &&
|
|
(hasCanvasAgentActionableImageEditIntent(text) ||
|
|
ACTIONABLE_GENERATION_REGEX.test(text) ||
|
|
LATEST_RESULT_REGEX.test(text) ||
|
|
ORIGINAL_IMAGE_REGEX.test(text) ||
|
|
hasCanvasAgentDeicticTarget(text) ||
|
|
SELECTION_REGEX.test(text));
|
|
|
|
if (allowsImplicitHistoryTarget) {
|
|
if (workingStateResolvedUrls.length > 0) {
|
|
const refs = toWorkingMemoryRefs(workingStateResolvedUrls);
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: refs,
|
|
candidateRefs,
|
|
resolutionMode: "working_memory",
|
|
confidence: 82,
|
|
needsClarification: false,
|
|
reason: "用户表达了连续处理意图，使用会话工作记忆中的上一轮已解析目标图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (workingStateGeneratedUrls.length > 0) {
|
|
const refs = toWorkingMemoryRefs(workingStateGeneratedUrls);
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: refs,
|
|
candidateRefs,
|
|
resolutionMode: "working_memory",
|
|
confidence: 78,
|
|
needsClarification: false,
|
|
reason: "用户表达了连续处理意图，使用会话工作记忆中的最近生成结果",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (generatedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "latest_result",
|
|
confidence: 74,
|
|
needsClarification: false,
|
|
reason: "用户表达了连续处理意图，续接最近一轮生成结果",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (originalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: originalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "working_memory",
|
|
confidence: 68,
|
|
needsClarification: false,
|
|
reason: "用户表达了连续处理意图，回退使用最近一次用户上传图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
}
|
|
|
|
if (hasCanvasAgentActionableImageEditIntent(text)) {
|
|
if (canvasRefs.length === 1) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [canvasRefs[0]],
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 82,
|
|
needsClarification: false,
|
|
reason: "用户表达了图片编辑意图，且当前画布仅有一张可处理图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [],
|
|
candidateRefs,
|
|
resolutionMode: "clarification",
|
|
confidence: 0,
|
|
needsClarification: true,
|
|
reason: "用户表达了编辑意图，但没有明确选中或文本定位目标图；不会自动关联历史图片",
|
|
fromHistory: false,
|
|
});
|
|
}
|
|
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [],
|
|
candidateRefs,
|
|
resolutionMode: "none",
|
|
confidence: 0,
|
|
needsClarification: false,
|
|
reason: "当前请求没有可定位的图片目标",
|
|
fromHistory: false,
|
|
});
|
|
}
|