import "server-only"; import type { ResolveTargetAssetsParams, ResolvedImageRef, ResolvedTargetAssets, } from "@/lib/ai/context-memory/types"; import { hasCanvasAgentActionableImageEditIntent, hasCanvasAgentDeicticTarget, resolveCanvasAgentTargetPreference, } from "@/lib/ai/runtime/canvas-agent-kernel"; import { collectOrdinalImageIndexes } from "@/lib/ai/context-memory/ordinal-image-references"; const ORIGINAL_IMAGE_REGEX = /(原图|上传图|参考图|最初那张|第一张图|素材图|源图|原始图片|原始那张|我发的那张|上传的那张|原来的图)/i; const LATEST_RESULT_REGEX = /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果|上一版|刚才那张结果|生成的图|生成图|结果图|出图结果|刚出的图|上一张生成图|上一轮结果图|现在这张生成图)/i; const SELECTION_REGEX = /(选中的|当前这张|我选的|这几张|这些图|选中图)/i; const ACTIONABLE_GENERATION_REGEX = /(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i; const IMPLICIT_HISTORY_TARGET_REGEX = /(继续|接着|沿用|按刚才|照刚才|上次|上一轮|上一版|刚才|刚生成|最新生成|结果图|生成图|这张|这个|当前|原图|上传图|选中)/i; function normalizeStringArray(values?: string[] | null): string[] { if (!Array.isArray(values)) return []; return Array.from( new Set( values .filter((value): value is string => typeof value === "string") .map((value) => value.trim()) .filter(Boolean), ), ); } function sortRefsForCanvasOrdering(refs: ResolvedImageRef[]) { return [...refs].sort((left, right) => { const leftY = Number(left.y ?? 0); const rightY = Number(right.y ?? 0); if (Math.abs(leftY - rightY) > 12) { return leftY - rightY; } return Number(left.x ?? 0) - Number(right.x ?? 0); }); } function getPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): string { if (refs.length <= 1) return "中间"; const xs = refs.map((item) => Number(item.x ?? 0)); const ys = refs.map((item) => Number(item.y ?? 0)); const minX = Math.min(...xs); const maxX = Math.max(...xs); const minY = Math.min(...ys); const maxY = Math.max(...ys); const spanX = Math.max(1, maxX - minX); const spanY = Math.max(1, maxY - minY); const relX = (Number(ref.x ?? 0) - minX) / spanX; const relY = (Number(ref.y ?? 0) - minY) / spanY; const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中"; const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中"; if (horizontal === "中" && vertical === "中") return "中间"; return `${vertical}${horizontal}`; } function pickByOrdinal(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] { const ordinalIndexes = collectOrdinalImageIndexes(text); if (ordinalIndexes.length === 0) return []; const ordered = sortRefsForCanvasOrdering(refs); return ordinalIndexes .map((index) => ordered[index]) .filter((ref): ref is ResolvedImageRef => Boolean(ref)); } function pickBySpatialHint(text: string, refs: ResolvedImageRef[]): ResolvedImageRef[] { if (!refs.length) return []; const normalizedText = text.replace(/\s+/g, ""); const hasLeftTop = /左上/.test(normalizedText); const hasLeftBottom = /左下/.test(normalizedText); const hasRightTop = /右上/.test(normalizedText); const hasRightBottom = /右下/.test(normalizedText); const hasLeft = /左图|左边|左侧|左面|靠左/.test(normalizedText); const hasRight = /右图|右边|右侧|右面|靠右/.test(normalizedText); const hasTop = /上图|上面|上方|上边|上侧|顶部|靠上|最上/.test(normalizedText); const hasBottom = /下图|下面|下方|下边|下侧|底部|靠下|最下/.test(normalizedText); const hasCenter = /中间|居中|正中|中央/.test(normalizedText); if ( !( hasLeftTop || hasLeftBottom || hasRightTop || hasRightBottom || hasLeft || hasRight || hasTop || hasBottom || hasCenter ) ) { return []; } const withLabels = refs.map((ref) => ({ ref, label: getPositionLabel(ref, refs), })); return withLabels .filter(({ label }) => { const matchesCompound = (hasLeftTop && label.includes("左") && label.includes("上")) || (hasLeftBottom && label.includes("左") && label.includes("下")) || (hasRightTop && label.includes("右") && label.includes("上")) || (hasRightBottom && label.includes("右") && label.includes("下")); const matchesAxis = (hasLeft && label.includes("左")) || (hasRight && label.includes("右")) || (hasTop && label.includes("上")) || (hasBottom && label.includes("下")) || (hasCenter && (label.includes("中") || label === "中间")); return matchesCompound || matchesAxis; }) .map(({ ref }) => ref); } function buildResolvedTargetAssets(params: { targetRefs: ResolvedImageRef[]; candidateRefs: ResolvedImageRef[]; resolutionMode: ResolvedTargetAssets["resolutionMode"]; confidence: number; needsClarification: boolean; reason: string; fromHistory: boolean; }): ResolvedTargetAssets { const targetRefs = params.targetRefs; const candidateRefs = params.candidateRefs; return { targetRefs, candidateRefs, targetImageUrls: targetRefs.map((ref) => ref.url), targetAssetIds: normalizeStringArray(targetRefs.map((ref) => ref.assetId || "")), candidateImageUrls: candidateRefs.map((ref) => ref.url), candidateAssetIds: normalizeStringArray(candidateRefs.map((ref) => ref.assetId || "")), resolutionMode: params.resolutionMode, confidence: Math.max(0, Math.min(100, Math.round(params.confidence))), needsClarification: params.needsClarification, reason: params.reason, fromHistory: params.fromHistory, }; } function toWorkingMemoryRefs(urls: string[]): ResolvedImageRef[] { return urls.map((url) => ({ url, normalizedUrl: url, source: "working_memory" as const, role: "working_memory" as const, })); } function buildNormalizedUrlSet(refs: ResolvedImageRef[]): Set { return new Set( refs .map((ref) => String(ref.normalizedUrl || ref.url || "").trim()) .filter(Boolean), ); } export function resolveTargetAssets( params: ResolveTargetAssetsParams, ): ResolvedTargetAssets { const text = String(params.text || "").trim(); const mediaInput = params.mediaInput; const canvasRefs = mediaInput.canvasImageRefs; const selectedRefs = mediaInput.selectedImageRefs; const currentRefs = mediaInput.currentMessageImageRefs; const assistantRefs = mediaInput.historyAssistantImageRefs; const userRefs = mediaInput.historyUserImageRefs; const candidateRefs = mediaInput.imageRefs; const workingState = params.workingState; const focusPolicy = params.contextPayload?.focusPolicy || "auto"; const workingStateGeneratedUrls = normalizeStringArray(workingState?.lastGeneratedImageUrls); const workingStateUploadedUrls = normalizeStringArray(workingState?.lastUploadedImageUrls); const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls); const generatedRefs = workingStateGeneratedUrls.length > 0 ? toWorkingMemoryRefs(workingStateGeneratedUrls) : assistantRefs; const originalRefs = workingStateUploadedUrls.length > 0 ? toWorkingMemoryRefs(workingStateUploadedUrls) : userRefs; const selectedLooksLikeOriginal = (() => { if (selectedRefs.length === 0 || originalRefs.length === 0) return false; const originalKeys = buildNormalizedUrlSet(originalRefs); return selectedRefs.every((ref) => originalKeys.has(String(ref.normalizedUrl || ref.url || "").trim()), ); })(); const currentOrdinalRefs = pickByOrdinal(text, currentRefs); if (currentOrdinalRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: currentOrdinalRefs, candidateRefs, resolutionMode: "current_upload", confidence: currentOrdinalRefs.length > 1 ? 96 : 94, needsClarification: false, reason: currentOrdinalRefs.length > 1 ? "根据用户文本中的多个序号定位到本轮上传的多张图片" : "根据用户文本中的序号定位到本轮上传图片", fromHistory: false, }); } if (currentRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: currentRefs, candidateRefs, resolutionMode: "current_upload", confidence: 93, needsClarification: false, reason: "本轮消息包含图片输入,默认优先处理当前上传图片", fromHistory: false, }); } const ordinalRefs = pickByOrdinal(text, canvasRefs); if (ordinalRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: ordinalRefs, candidateRefs, resolutionMode: "text_reference", confidence: ordinalRefs.length > 1 ? 91 : 88, needsClarification: false, reason: ordinalRefs.length > 1 ? "根据用户文本中的多个序号定位到多张画布图片" : "根据用户文本中的序号定位到画布图片", fromHistory: true, }); } if (ORIGINAL_IMAGE_REGEX.test(text) && originalRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: originalRefs, candidateRefs, resolutionMode: "text_reference", confidence: 95, needsClarification: false, reason: "用户文本明确指向原图/上传图", fromHistory: true, }); } if (LATEST_RESULT_REGEX.test(text) && generatedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: generatedRefs, candidateRefs, resolutionMode: "latest_result", confidence: 95, needsClarification: false, reason: "用户文本明确指向上一轮结果/生成图", fromHistory: true, }); } if (focusPolicy === "original" && originalRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: originalRefs, candidateRefs, resolutionMode: "text_reference", confidence: 96, needsClarification: false, reason: "前端上下文指定优先使用原图", fromHistory: true, }); } if (focusPolicy === "latest_result" && generatedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: generatedRefs, candidateRefs, resolutionMode: "latest_result", confidence: 96, needsClarification: false, reason: "前端上下文指定优先使用上一轮结果", fromHistory: true, }); } const spatialRefs = pickBySpatialHint(text, canvasRefs); if (spatialRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: spatialRefs, candidateRefs, resolutionMode: "text_reference", confidence: spatialRefs.length === 1 ? 90 : 76, needsClarification: false, reason: "根据用户文本中的方位指向定位到画布图片", fromHistory: true, }); } if (focusPolicy === "selected" && selectedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: selectedRefs, candidateRefs, resolutionMode: "explicit_selection", confidence: 97, needsClarification: false, reason: "前端已明确选中目标图片", fromHistory: true, }); } const canvasAgentTargetPreference = resolveCanvasAgentTargetPreference({ text, selectedImageCount: selectedRefs.length, latestResultImageCount: generatedRefs.length, workingMemoryTargetImageCount: workingStateResolvedUrls.length, }); if (canvasAgentTargetPreference.shouldPreferLatestResult) { return buildResolvedTargetAssets({ targetRefs: generatedRefs, candidateRefs, resolutionMode: workingStateGeneratedUrls.length > 0 ? "working_memory" : "latest_result", confidence: canvasAgentTargetPreference.confidence, needsClarification: false, reason: "用户使用指代表达并提出图片编辑意图,默认续接最近生成结果", fromHistory: true, }); } if (canvasAgentTargetPreference.shouldPreferWorkingMemoryTarget) { const refs = toWorkingMemoryRefs(workingStateResolvedUrls); return buildResolvedTargetAssets({ targetRefs: refs, candidateRefs, resolutionMode: "working_memory", confidence: canvasAgentTargetPreference.confidence, needsClarification: false, reason: "用户使用指代表达并提出图片编辑意图,使用工作记忆中的上一轮目标图", fromHistory: true, }); } if (hasCanvasAgentDeicticTarget(text)) { if (selectedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: selectedRefs, candidateRefs, resolutionMode: "explicit_selection", confidence: 93, needsClarification: false, reason: "用户使用“这张/这只/这个”等指代表达,且当前存在明确选中目标图", fromHistory: true, }); } if (candidateRefs.length === 1) { return buildResolvedTargetAssets({ targetRefs: [candidateRefs[0]], candidateRefs, resolutionMode: "text_reference", confidence: 84, needsClarification: false, reason: "用户使用指代表达,且当前仅有一张候选图", fromHistory: true, }); } return buildResolvedTargetAssets({ targetRefs: [], candidateRefs, resolutionMode: "clarification", confidence: 0, needsClarification: true, reason: "用户使用“这张/这只/这个”等指代表达,但当前没有明确选中唯一目标图", fromHistory: true, }); } if ( hasCanvasAgentActionableImageEditIntent(text) && currentRefs.length === 0 && generatedRefs.length > 0 && selectedLooksLikeOriginal && !SELECTION_REGEX.test(text) && !ORIGINAL_IMAGE_REGEX.test(text) ) { return buildResolvedTargetAssets({ targetRefs: generatedRefs, candidateRefs, resolutionMode: "latest_result", confidence: 89, needsClarification: false, reason: "连续对话默认续接上一轮生成结果,避免误回退到原始选中图", fromHistory: true, }); } if ((SELECTION_REGEX.test(text) || focusPolicy === "selected") && selectedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: selectedRefs, candidateRefs, resolutionMode: "explicit_selection", confidence: 92, needsClarification: false, reason: "用户文本或上下文明确要求使用当前选中的图片", fromHistory: true, }); } if (selectedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: selectedRefs, candidateRefs, resolutionMode: "explicit_selection", confidence: 86, needsClarification: false, reason: "当前存在画布选中图片,默认优先继续处理选中目标", fromHistory: true, }); } if ( canvasRefs.length === 1 && (hasCanvasAgentActionableImageEditIntent(text) || focusPolicy === "auto") ) { return buildResolvedTargetAssets({ targetRefs: [canvasRefs[0]], candidateRefs, resolutionMode: "text_reference", confidence: 84, needsClarification: false, reason: "用户使用画布/图片指代表达,且当前画布仅有一张可处理图片", fromHistory: true, }); } const allowsImplicitHistoryTarget = IMPLICIT_HISTORY_TARGET_REGEX.test(text) && (hasCanvasAgentActionableImageEditIntent(text) || ACTIONABLE_GENERATION_REGEX.test(text) || LATEST_RESULT_REGEX.test(text) || ORIGINAL_IMAGE_REGEX.test(text) || hasCanvasAgentDeicticTarget(text) || SELECTION_REGEX.test(text)); if (allowsImplicitHistoryTarget) { if (workingStateResolvedUrls.length > 0) { const refs = toWorkingMemoryRefs(workingStateResolvedUrls); return buildResolvedTargetAssets({ targetRefs: refs, candidateRefs, resolutionMode: "working_memory", confidence: 82, needsClarification: false, reason: "用户表达了连续处理意图,使用会话工作记忆中的上一轮已解析目标图", fromHistory: true, }); } if (workingStateGeneratedUrls.length > 0) { const refs = toWorkingMemoryRefs(workingStateGeneratedUrls); return buildResolvedTargetAssets({ targetRefs: refs, candidateRefs, resolutionMode: "working_memory", confidence: 78, needsClarification: false, reason: "用户表达了连续处理意图,使用会话工作记忆中的最近生成结果", fromHistory: true, }); } if (generatedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: generatedRefs, candidateRefs, resolutionMode: "latest_result", confidence: 74, needsClarification: false, reason: "用户表达了连续处理意图,续接最近一轮生成结果", fromHistory: true, }); } if (originalRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: originalRefs, candidateRefs, resolutionMode: "working_memory", confidence: 68, needsClarification: false, reason: "用户表达了连续处理意图,回退使用最近一次用户上传图片", fromHistory: true, }); } } if (hasCanvasAgentActionableImageEditIntent(text)) { if (canvasRefs.length === 1) { return buildResolvedTargetAssets({ targetRefs: [canvasRefs[0]], candidateRefs, resolutionMode: "text_reference", confidence: 82, needsClarification: false, reason: "用户表达了图片编辑意图,且当前画布仅有一张可处理图片", fromHistory: true, }); } return buildResolvedTargetAssets({ targetRefs: [], candidateRefs, resolutionMode: "clarification", confidence: 0, needsClarification: true, reason: "用户表达了编辑意图,但没有明确选中或文本定位目标图;不会自动关联历史图片", fromHistory: false, }); } return buildResolvedTargetAssets({ targetRefs: [], candidateRefs, resolutionMode: "none", confidence: 0, needsClarification: false, reason: "当前请求没有可定位的图片目标", fromHistory: false, }); }