468 lines
16 KiB
TypeScript
468 lines
16 KiB
TypeScript
import "server-only";
|
|
|
|
import type {
|
|
ResolveTargetAssetsParams,
|
|
ResolvedImageRef,
|
|
ResolvedTargetAssets,
|
|
} from "@/lib/ai/context-memory/types";
|
|
|
|
const ORIGINAL_IMAGE_REGEX =
|
|
/(原图|上传图|参考图|最初那张|第一张图|素材图|源图|原始图片|原始那张|我发的那张|上传的那张|原来的图)/i;
|
|
const LATEST_RESULT_REGEX =
|
|
/(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果|上一版|刚才那张结果|生成的图|生成图|结果图|出图结果|刚出的图|上一张生成图|上一轮结果图|现在这张生成图)/i;
|
|
const DEICTIC_TARGET_REGEX =
|
|
/(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i;
|
|
const SELECTION_REGEX = /(选中的|当前这张|我选的|这几张|这些图|选中图)/i;
|
|
const ACTIONABLE_EDIT_REGEX =
|
|
/(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格|亮一点|暗一点|白底|透明底)/i;
|
|
const ACTIONABLE_GENERATION_REGEX =
|
|
/(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i;
|
|
const IMPLICIT_HISTORY_TARGET_REGEX =
|
|
/(继续|接着|沿用|按刚才|照刚才|上次|上一轮|上一版|刚才|刚生成|最新生成|结果图|生成图|这张|这个|当前|原图|上传图|选中)/i;
|
|
|
|
function normalizeStringArray(values?: string[] | null): string[] {
|
|
if (!Array.isArray(values)) return [];
|
|
return Array.from(
|
|
new Set(
|
|
values
|
|
.filter((value): value is string => typeof value === "string")
|
|
.map((value) => value.trim())
|
|
.filter(Boolean),
|
|
),
|
|
);
|
|
}
|
|
|
|
function chineseNumberToInt(value: string): number | null {
|
|
const normalized = value.trim();
|
|
if (!normalized) return null;
|
|
if (/^\d+$/.test(normalized)) return Number.parseInt(normalized, 10);
|
|
|
|
const map: Record<string, number> = {
|
|
一: 1,
|
|
二: 2,
|
|
两: 2,
|
|
三: 3,
|
|
四: 4,
|
|
五: 5,
|
|
六: 6,
|
|
七: 7,
|
|
八: 8,
|
|
九: 9,
|
|
十: 10,
|
|
};
|
|
|
|
if (normalized === "十") return 10;
|
|
if (normalized.length === 1) return map[normalized] ?? null;
|
|
if (normalized.startsWith("十")) {
|
|
return 10 + (map[normalized.slice(1)] ?? 0);
|
|
}
|
|
if (normalized.endsWith("十")) {
|
|
return (map[normalized.slice(0, 1)] ?? 0) * 10;
|
|
}
|
|
if (normalized.includes("十")) {
|
|
const [head, tail] = normalized.split("十");
|
|
return (map[head] ?? 0) * 10 + (map[tail] ?? 0);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function sortRefsForCanvasOrdering(refs: ResolvedImageRef[]) {
|
|
return [...refs].sort((left, right) => {
|
|
const leftY = Number(left.y ?? 0);
|
|
const rightY = Number(right.y ?? 0);
|
|
if (Math.abs(leftY - rightY) > 12) {
|
|
return leftY - rightY;
|
|
}
|
|
return Number(left.x ?? 0) - Number(right.x ?? 0);
|
|
});
|
|
}
|
|
|
|
function getPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): string {
|
|
if (refs.length <= 1) return "中间";
|
|
const xs = refs.map((item) => Number(item.x ?? 0));
|
|
const ys = refs.map((item) => Number(item.y ?? 0));
|
|
const minX = Math.min(...xs);
|
|
const maxX = Math.max(...xs);
|
|
const minY = Math.min(...ys);
|
|
const maxY = Math.max(...ys);
|
|
const spanX = Math.max(1, maxX - minX);
|
|
const spanY = Math.max(1, maxY - minY);
|
|
const relX = (Number(ref.x ?? 0) - minX) / spanX;
|
|
const relY = (Number(ref.y ?? 0) - minY) / spanY;
|
|
|
|
const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中";
|
|
const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中";
|
|
|
|
if (horizontal === "中" && vertical === "中") return "中间";
|
|
return `${vertical}${horizontal}`;
|
|
}
|
|
|
|
function pickByOrdinal(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] {
|
|
const matched = text.match(/第\s*([一二三四五六七八九十两\d]+)\s*(?:张|个|幅|图)/i);
|
|
const numeric = matched ? chineseNumberToInt(matched[1]) : null;
|
|
if (!numeric || numeric <= 0) return [];
|
|
const ordered = sortRefsForCanvasOrdering(refs);
|
|
return ordered[numeric - 1] ? [ordered[numeric - 1]] : [];
|
|
}
|
|
|
|
function pickBySpatialHint(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] {
|
|
if (!refs.length) return [];
|
|
|
|
const normalizedText = text.replace(/\s+/g, "");
|
|
const hasLeft = /左边|左侧|左面|左上|左下|靠左/.test(normalizedText);
|
|
const hasRight = /右边|右侧|右面|右上|右下|靠右/.test(normalizedText);
|
|
const hasTop = /上面|上方|上边|上侧|顶部|靠上|最上/.test(normalizedText);
|
|
const hasBottom = /下面|下方|下边|下侧|底部|靠下|最下/.test(normalizedText);
|
|
const hasCenter = /中间|居中|正中|中央/.test(normalizedText);
|
|
|
|
if (!(hasLeft || hasRight || hasTop || hasBottom || hasCenter)) return [];
|
|
|
|
const withLabels = refs.map((ref) => ({
|
|
ref,
|
|
label: getPositionLabel(ref, refs),
|
|
}));
|
|
|
|
return withLabels
|
|
.filter(({ label }) => {
|
|
if (hasLeft && !label.includes("左")) return false;
|
|
if (hasRight && !label.includes("右")) return false;
|
|
if (hasTop && !label.includes("上")) return false;
|
|
if (hasBottom && !label.includes("下")) return false;
|
|
if (hasCenter && !(label.includes("中") || label === "中间")) return false;
|
|
return true;
|
|
})
|
|
.map(({ ref }) => ref);
|
|
}
|
|
|
|
function buildResolvedTargetAssets(params: {
|
|
targetRefs: ResolvedImageRef[];
|
|
candidateRefs: ResolvedImageRef[];
|
|
resolutionMode: ResolvedTargetAssets["resolutionMode"];
|
|
confidence: number;
|
|
needsClarification: boolean;
|
|
reason: string;
|
|
fromHistory: boolean;
|
|
}): ResolvedTargetAssets {
|
|
const targetRefs = params.targetRefs;
|
|
const candidateRefs = params.candidateRefs;
|
|
return {
|
|
targetRefs,
|
|
candidateRefs,
|
|
targetImageUrls: targetRefs.map((ref) => ref.url),
|
|
targetAssetIds: normalizeStringArray(targetRefs.map((ref) => ref.assetId || "")),
|
|
candidateImageUrls: candidateRefs.map((ref) => ref.url),
|
|
candidateAssetIds: normalizeStringArray(candidateRefs.map((ref) => ref.assetId || "")),
|
|
resolutionMode: params.resolutionMode,
|
|
confidence: Math.max(0, Math.min(100, Math.round(params.confidence))),
|
|
needsClarification: params.needsClarification,
|
|
reason: params.reason,
|
|
fromHistory: params.fromHistory,
|
|
};
|
|
}
|
|
|
|
function toWorkingMemoryRefs(urls: string[]): ResolvedImageRef[] {
|
|
return urls.map((url) => ({
|
|
url,
|
|
normalizedUrl: url,
|
|
source: "working_memory" as const,
|
|
role: "working_memory" as const,
|
|
}));
|
|
}
|
|
|
|
function buildNormalizedUrlSet(refs: ResolvedImageRef[]): Set<string> {
|
|
return new Set(
|
|
refs
|
|
.map((ref) => String(ref.normalizedUrl || ref.url || "").trim())
|
|
.filter(Boolean),
|
|
);
|
|
}
|
|
|
|
export function resolveTargetAssets(
|
|
params: ResolveTargetAssetsParams,
|
|
): ResolvedTargetAssets {
|
|
const text = String(params.text || "").trim();
|
|
const mediaInput = params.mediaInput;
|
|
const canvasRefs = mediaInput.canvasImageRefs;
|
|
const selectedRefs = mediaInput.selectedImageRefs;
|
|
const currentRefs = mediaInput.currentMessageImageRefs;
|
|
const assistantRefs = mediaInput.historyAssistantImageRefs;
|
|
const userRefs = mediaInput.historyUserImageRefs;
|
|
const candidateRefs = mediaInput.imageRefs;
|
|
const workingState = params.workingState;
|
|
const focusPolicy = params.contextPayload?.focusPolicy || "auto";
|
|
const workingStateGeneratedUrls = normalizeStringArray(workingState?.lastGeneratedImageUrls);
|
|
const workingStateUploadedUrls = normalizeStringArray(workingState?.lastUploadedImageUrls);
|
|
const generatedRefs =
|
|
workingStateGeneratedUrls.length > 0 ? toWorkingMemoryRefs(workingStateGeneratedUrls) : assistantRefs;
|
|
const originalRefs =
|
|
workingStateUploadedUrls.length > 0 ? toWorkingMemoryRefs(workingStateUploadedUrls) : userRefs;
|
|
const selectedLooksLikeOriginal = (() => {
|
|
if (selectedRefs.length === 0 || originalRefs.length === 0) return false;
|
|
const originalKeys = buildNormalizedUrlSet(originalRefs);
|
|
return selectedRefs.every((ref) =>
|
|
originalKeys.has(String(ref.normalizedUrl || ref.url || "").trim()),
|
|
);
|
|
})();
|
|
|
|
if (ORIGINAL_IMAGE_REGEX.test(text) && originalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: originalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 95,
|
|
needsClarification: false,
|
|
reason: "用户文本明确指向原图/上传图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (LATEST_RESULT_REGEX.test(text) && generatedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "latest_result",
|
|
confidence: 95,
|
|
needsClarification: false,
|
|
reason: "用户文本明确指向上一轮结果/生成图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (focusPolicy === "original" && originalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: originalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 96,
|
|
needsClarification: false,
|
|
reason: "前端上下文指定优先使用原图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (focusPolicy === "latest_result" && generatedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "latest_result",
|
|
confidence: 96,
|
|
needsClarification: false,
|
|
reason: "前端上下文指定优先使用上一轮结果",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
const ordinalRefs = pickByOrdinal(text, canvasRefs);
|
|
if (ordinalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: ordinalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 88,
|
|
needsClarification: false,
|
|
reason: "根据用户文本中的序号定位到画布图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
const spatialRefs = pickBySpatialHint(text, canvasRefs);
|
|
if (spatialRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: spatialRefs,
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: spatialRefs.length === 1 ? 90 : 76,
|
|
needsClarification: false,
|
|
reason: "根据用户文本中的方位指向定位到画布图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (focusPolicy === "selected" && selectedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: selectedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "explicit_selection",
|
|
confidence: 97,
|
|
needsClarification: false,
|
|
reason: "前端已明确选中目标图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (currentRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: currentRefs,
|
|
candidateRefs,
|
|
resolutionMode: "current_upload",
|
|
confidence: 93,
|
|
needsClarification: false,
|
|
reason: "本轮消息包含图片输入，默认优先处理当前上传图片",
|
|
fromHistory: false,
|
|
});
|
|
}
|
|
|
|
if (DEICTIC_TARGET_REGEX.test(text)) {
|
|
if (selectedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: selectedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "explicit_selection",
|
|
confidence: 93,
|
|
needsClarification: false,
|
|
reason: "用户使用“这张/这只/这个”等指代表达，且当前存在明确选中目标图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (candidateRefs.length === 1) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [candidateRefs[0]],
|
|
candidateRefs,
|
|
resolutionMode: "text_reference",
|
|
confidence: 84,
|
|
needsClarification: false,
|
|
reason: "用户使用指代表达，且当前仅有一张候选图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [],
|
|
candidateRefs,
|
|
resolutionMode: "clarification",
|
|
confidence: 0,
|
|
needsClarification: true,
|
|
reason: "用户使用“这张/这只/这个”等指代表达，但当前没有明确选中唯一目标图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (
|
|
ACTIONABLE_EDIT_REGEX.test(text) &&
|
|
currentRefs.length === 0 &&
|
|
generatedRefs.length > 0 &&
|
|
selectedLooksLikeOriginal &&
|
|
!SELECTION_REGEX.test(text) &&
|
|
!ORIGINAL_IMAGE_REGEX.test(text)
|
|
) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "latest_result",
|
|
confidence: 89,
|
|
needsClarification: false,
|
|
reason: "连续对话默认续接上一轮生成结果，避免误回退到原始选中图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if ((SELECTION_REGEX.test(text) || focusPolicy === "selected") && selectedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: selectedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "explicit_selection",
|
|
confidence: 92,
|
|
needsClarification: false,
|
|
reason: "用户文本或上下文明确要求使用当前选中的图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (selectedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: selectedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "explicit_selection",
|
|
confidence: 86,
|
|
needsClarification: false,
|
|
reason: "当前存在画布选中图片，默认优先继续处理选中目标",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls);
|
|
const allowsImplicitHistoryTarget =
|
|
IMPLICIT_HISTORY_TARGET_REGEX.test(text) &&
|
|
(ACTIONABLE_EDIT_REGEX.test(text) ||
|
|
ACTIONABLE_GENERATION_REGEX.test(text) ||
|
|
LATEST_RESULT_REGEX.test(text) ||
|
|
ORIGINAL_IMAGE_REGEX.test(text) ||
|
|
DEICTIC_TARGET_REGEX.test(text) ||
|
|
SELECTION_REGEX.test(text));
|
|
|
|
if (allowsImplicitHistoryTarget) {
|
|
if (workingStateResolvedUrls.length > 0) {
|
|
const refs = toWorkingMemoryRefs(workingStateResolvedUrls);
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: refs,
|
|
candidateRefs,
|
|
resolutionMode: "working_memory",
|
|
confidence: 82,
|
|
needsClarification: false,
|
|
reason: "用户表达了连续处理意图，使用会话工作记忆中的上一轮已解析目标图",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (workingStateGeneratedUrls.length > 0) {
|
|
const refs = toWorkingMemoryRefs(workingStateGeneratedUrls);
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: refs,
|
|
candidateRefs,
|
|
resolutionMode: "working_memory",
|
|
confidence: 78,
|
|
needsClarification: false,
|
|
reason: "用户表达了连续处理意图，使用会话工作记忆中的最近生成结果",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (generatedRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: generatedRefs,
|
|
candidateRefs,
|
|
resolutionMode: "latest_result",
|
|
confidence: 74,
|
|
needsClarification: false,
|
|
reason: "用户表达了连续处理意图，续接最近一轮生成结果",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
|
|
if (originalRefs.length > 0) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: originalRefs,
|
|
candidateRefs,
|
|
resolutionMode: "working_memory",
|
|
confidence: 68,
|
|
needsClarification: false,
|
|
reason: "用户表达了连续处理意图，回退使用最近一次用户上传图片",
|
|
fromHistory: true,
|
|
});
|
|
}
|
|
}
|
|
|
|
if (ACTIONABLE_EDIT_REGEX.test(text)) {
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [],
|
|
candidateRefs,
|
|
resolutionMode: "clarification",
|
|
confidence: 0,
|
|
needsClarification: true,
|
|
reason: "用户表达了编辑意图，但没有明确选中或文本定位目标图；不会自动关联历史图片",
|
|
fromHistory: false,
|
|
});
|
|
}
|
|
|
|
return buildResolvedTargetAssets({
|
|
targetRefs: [],
|
|
candidateRefs,
|
|
resolutionMode: "none",
|
|
confidence: 0,
|
|
needsClarification: false,
|
|
reason: "当前请求没有可定位的图片目标",
|
|
fromHistory: false,
|
|
});
|
|
}
|