fix: preserve selected image context for generation

2026-04-15 18:53:14 +08:00 · 2026-04-15 18:53:14 +08:00 · f75f77105d
commit f75f77105d
parent f112a63c75
5 changed files with 301 additions and 43 deletions
--- a/app/(chat)/api/chat/route.ts
+++ b/app/(chat)/api/chat/route.ts
@ -1622,6 +1622,65 @@ function injectHistoryImagesToLastUserMessage(
  return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
 }

+function isImageInputPart(part: any) {
+  return part?.type === "file" || part?.type === "image" || part?.type === "image_url";
+}
+
+function normalizeResolvedTargetImageUrls(imageUrls?: string[] | null) {
+  return Array.from(
+    new Set(
+      (imageUrls || [])
+        .map((url) => String(url || "").trim())
+        .filter((url) => /^https?:\/\//i.test(url))
+    )
+  ).slice(0, 8);
+}
+
+function summarizeTargetImageUrl(url?: string) {
+  const normalized = String(url || "").trim();
+  if (!normalized) return undefined;
+  try {
+    const parsed = new URL(normalized);
+    return `${parsed.origin}${parsed.pathname}`.slice(0, 160);
+  } catch {
+    return normalized.slice(0, 160);
+  }
+}
+
+function applyResolvedTargetImagesToLastUserMessage(
+  messages: ChatMessage[],
+  targetImageUrls: string[]
+): ChatMessage[] {
+  const normalizedTargetImageUrls = normalizeResolvedTargetImageUrls(targetImageUrls);
+  if (normalizedTargetImageUrls.length === 0) return messages;
+
+  let lastUserIndex = -1;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (messages[i]?.role === "user") {
+      lastUserIndex = i;
+      break;
+    }
+  }
+
+  if (lastUserIndex < 0) return messages;
+  const lastUserMessage = messages[lastUserIndex];
+  const nonImageParts = (lastUserMessage.parts || []).filter((part: any) => !isImageInputPart(part));
+  const imageParts = normalizedTargetImageUrls.map((url, index) => ({
+    type: "file" as const,
+    url,
+    name: `target-image-${index + 1}`,
+    mediaType: guessImageMediaType(url),
+    mimeType: guessImageMediaType(url),
+  }));
+
+  const updatedMessage: ChatMessage = {
+    ...lastUserMessage,
+    parts: [...imageParts, ...nonImageParts],
+  };
+
+  return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg));
+}
+
 function compactAutoDemand(text: string, maxLength = 56): string {
  const normalized = text.replace(/\s+/g, " ").trim();
  if (!normalized) return "按你刚才的描述执行";
@ -9617,6 +9676,21 @@ export async function POST(request: Request) {
                hiddenPromptPrefix,
              );
            }
+
+            const resolvedTargetImageUrlsForProvider = normalizeResolvedTargetImageUrls(
+              requestTargetResolution?.targetImageUrls
+            );
+            if (resolvedTargetImageUrlsForProvider.length > 0) {
+              messagesForAlibaba = applyResolvedTargetImagesToLastUserMessage(
+                messagesForAlibaba,
+                resolvedTargetImageUrlsForProvider
+              );
+              console.log("[Chat API] Image provider target images:", {
+                resolutionMode: requestTargetResolution?.resolutionMode || null,
+                imageCount: resolvedTargetImageUrlsForProvider.length,
+                firstImageUrl: summarizeTargetImageUrl(resolvedTargetImageUrlsForProvider[0]),
+              });
+            }
            
            messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages);
            
--- a/lib/agents/workflow-ui-config.ts
+++ b/lib/agents/workflow-ui-config.ts
@ -13,7 +13,7 @@ import type {

 export const AGENT_WORKFLOW_UI_CONFIG_KEY = "agent_workflow_ui_config";
 export const AGENT_WORKFLOW_MAX_TREE_DEPTH = 8;
-export const AGENT_WORKFLOW_MAX_OPTION_DEPTH = 8;
+export const AGENT_WORKFLOW_MAX_OPTION_DEPTH = 100;
 export const AGENT_WORKFLOW_PROMPT_FRAGMENT_MAX_LENGTH = 20_000;
 export const defaultAgentWorkflowCategoryLabels: AgentWorkflowCategoryLabels = {
  firstCategory: "品牌",
--- a/lib/ai/context-memory/target-resolution.ts
+++ b/lib/ai/context-memory/target-resolution.ts
@ -274,6 +274,18 @@ export function resolveTargetAssets(
    });
  }

+  if (focusPolicy === "selected" && selectedRefs.length > 0) {
+    return buildResolvedTargetAssets({
+      targetRefs: selectedRefs,
+      candidateRefs,
+      resolutionMode: "explicit_selection",
+      confidence: 97,
+      needsClarification: false,
+      reason: "前端已明确选中目标图片",
+      fromHistory: true,
+    });
+  }
+
  if (currentRefs.length > 0) {
    return buildResolvedTargetAssets({
      targetRefs: currentRefs,
@ -341,18 +353,6 @@ export function resolveTargetAssets(
    });
  }

-  if (focusPolicy === "selected" && selectedRefs.length > 0) {
-    return buildResolvedTargetAssets({
-      targetRefs: selectedRefs,
-      candidateRefs,
-      resolutionMode: "explicit_selection",
-      confidence: 97,
-      needsClarification: false,
-      reason: "前端已明确选中目标图片",
-      fromHistory: true,
-    });
-  }
-
  if ((SELECTION_REGEX.test(text) || focusPolicy === "selected") && selectedRefs.length > 0) {
    return buildResolvedTargetAssets({
      targetRefs: selectedRefs,
--- a/lib/ai/providers/krea.ts
+++ b/lib/ai/providers/krea.ts
@ -41,6 +41,22 @@ const IMAGE_POLL_INTERVAL_MS = 2000;
 const VIDEO_POLL_INTERVAL_MS = 5000;
 const IMAGE_TIMEOUT_MS = 5 * 60 * 1000;
 const VIDEO_TIMEOUT_MS = 20 * 60 * 1000;
+const KREA_IMAGE_ASPECT_RATIOS = [
+  "21:9",
+  "16:9",
+  "9:16",
+  "4:3",
+  "3:4",
+  "3:2",
+  "2:3",
+  "5:4",
+  "4:5",
+  "1:1",
+  "auto",
+] as const;
+const KREA_OFFICIAL_IMAGE_ASPECT_RATIOS = KREA_IMAGE_ASPECT_RATIOS.filter(
+  (ratio) => ratio !== "auto"
+);

 function normalizeText(value?: string | null) {
  return String(value || "").replace(/\u00a0/g, " ").trim().normalize("NFKC");
@ -202,23 +218,42 @@ function extractPromptAndAssets(messages: Array<{ role: string; content: any }>)

 function normalizeAspectRatio(value?: string, fallback = "16:9") {
  const normalized = normalizeText(value || fallback);
-  if (
-    [
-      "21:9",
-      "16:9",
-      "9:16",
-      "4:3",
-      "3:4",
-      "1:1",
-      "9:21",
-      "auto",
-    ].includes(normalized)
-  ) {
+  if (KREA_IMAGE_ASPECT_RATIOS.includes(normalized as any)) {
    return normalized;
  }
  return fallback;
 }

+function parseAspectRatioValue(ratio: string) {
+  const [rawW, rawH] = ratio.split(":").map((item) => Number(item));
+  if (!Number.isFinite(rawW) || !Number.isFinite(rawH) || rawW <= 0 || rawH <= 0) {
+    return null;
+  }
+  return rawW / rawH;
+}
+
+function inferNearestKreaImageAspectRatio(width?: number | null, height?: number | null) {
+  const sourceWidth = Number(width);
+  const sourceHeight = Number(height);
+  if (!Number.isFinite(sourceWidth) || !Number.isFinite(sourceHeight) || sourceWidth <= 0 || sourceHeight <= 0) {
+    return undefined;
+  }
+
+  const sourceRatio = sourceWidth / sourceHeight;
+  let bestRatio: string | undefined;
+  let bestDistance = Number.POSITIVE_INFINITY;
+  for (const ratio of KREA_OFFICIAL_IMAGE_ASPECT_RATIOS) {
+    const ratioValue = parseAspectRatioValue(ratio);
+    if (!ratioValue) continue;
+    const distance = Math.abs(Math.log(sourceRatio / ratioValue));
+    if (distance < bestDistance) {
+      bestDistance = distance;
+      bestRatio = ratio;
+    }
+  }
+  return bestRatio;
+}
+
 function clamp(value: number, min: number, max: number) {
  return Math.max(min, Math.min(max, value));
 }
@ -341,6 +376,95 @@ function buildTaggedReferences(imageUrls: string[], tagPrefix = "ref", max = 3)
  }));
 }

+const KREA_SOURCE_IMAGE_FIELDS = new Set([
+  "imageUrls",
+  "imageUrl",
+  "image_url",
+  "referenceImages",
+  "init_image_url",
+  "references",
+]);
+
+function hasKreaSourceImageField(fields: Iterable<string>) {
+  for (const field of fields) {
+    if (KREA_SOURCE_IMAGE_FIELDS.has(field)) return true;
+  }
+  return false;
+}
+
+function buildReferencePreservingPrompt(params: {
+  endpoint: string;
+  kind: KreaModelKind;
+  prompt: string;
+  imageUrls: string[];
+  requestFields: string[];
+}) {
+  const { endpoint, kind, prompt, imageUrls, requestFields } = params;
+  if (kind === "video" || imageUrls.length === 0) return prompt;
+  if (endpoint.includes("/generate/enhance/")) return prompt;
+  if (!hasKreaSourceImageField(requestFields)) return prompt;
+
+  return [
+    "Use the provided reference image(s) as the source image(s). Preserve the original subject, composition, camera angle, geometry, layout, object positions, and spatial relationships. Only apply the user's requested transformation. Do not invent a new unrelated scene.",
+    "请严格以用户上传的参考图为唯一源图，保持原图主体、构图、视角、几何结构、空间布局、物体位置和相对关系，只执行用户要求的转换，不要生成无关的新场景。",
+    `用户要求：${prompt}`,
+  ].join("\n\n");
+}
+
+function summarizeKreaUrl(url?: string | null) {
+  const normalized = normalizeText(url);
+  if (!normalized) return undefined;
+  try {
+    const parsed = new URL(normalized);
+    return `${parsed.origin}${parsed.pathname}`.slice(0, 160);
+  } catch {
+    return normalized.slice(0, 160);
+  }
+}
+
+function summarizeKreaReferenceFields(body: Record<string, any>) {
+  const summary: Record<string, any> = {};
+  const summarizeArray = (value: unknown[]) => ({
+    count: value.length,
+    first:
+      typeof value[0] === "string"
+        ? summarizeKreaUrl(value[0])
+        : value[0] && typeof value[0] === "object"
+          ? summarizeKreaUrl(
+              (value[0] as Record<string, any>).url ||
+                (value[0] as Record<string, any>).uri ||
+                (value[0] as Record<string, any>).image?.[0]
+            )
+          : undefined,
+  });
+
+  for (const field of [
+    "imageUrls",
+    "imageUrl",
+    "image_url",
+    "styleImages",
+    "imageStyleRefs",
+    "characterReferenceImages",
+    "referenceImages",
+    "startImage",
+    "endImage",
+    "init_image_url",
+    "references",
+    "subjects",
+    "styleReferences",
+    "elementReferences",
+  ]) {
+    const value = body[field];
+    if (Array.isArray(value)) {
+      summary[field] = summarizeArray(value);
+    } else if (typeof value === "string") {
+      summary[field] = summarizeKreaUrl(value);
+    }
+  }
+
+  return summary;
+}
+
 function collectHttpUrls(payload: unknown): string[] {
  const urls = new Set<string>();
  const visit = (value: unknown) => {
@ -384,6 +508,23 @@ async function downloadRemoteAsset(url: string) {
  }
 }

+async function inferReferenceAspectRatioFromImageUrl(url?: string | null) {
+  const normalized = normalizeText(url);
+  if (!normalized || !/^https?:\/\//i.test(normalized)) return undefined;
+  try {
+    const { buffer, contentType } = await downloadRemoteAsset(normalized);
+    if (!String(contentType || "").startsWith("image/")) return undefined;
+    const metadata = await sharp(buffer, { failOn: "none" }).metadata();
+    return inferNearestKreaImageAspectRatio(metadata.width, metadata.height);
+  } catch (error) {
+    console.warn("[Krea] Failed to infer reference aspect ratio:", {
+      imageUrl: summarizeKreaUrl(normalized),
+      error,
+    });
+    return undefined;
+  }
+}
+
 async function uploadGeneratedBuffer(buffer: Buffer, extension: string): Promise<UploadedImage> {
  const thumbnailBuffer = await sharp(buffer)
    .resize(400, 400, { fit: "inside", withoutEnlargement: true })
@ -587,6 +728,7 @@ function buildKreaBody(params: {
  prompt: string;
  imageUrls: string[];
  videoUrls: string[];
+  referenceAspectRatio?: string;
  imageParams?: KreaImageParams;
  videoParams?: KreaVideoParams;
 }) {
@ -598,6 +740,7 @@ function buildKreaBody(params: {
    prompt,
    imageUrls,
    videoUrls,
+    referenceAspectRatio,
    imageParams,
    videoParams,
  } = params;
@ -605,9 +748,15 @@ function buildKreaBody(params: {
  const required = new Set(requiredFields);
  const body: Record<string, any> = {};
  const isVideo = kind === "video";
+  const hasImageReference =
+    kind !== "video" && imageUrls.length > 0 && hasKreaSourceImageField(fields);
+  const requestedImageAspectRatio = normalizeText(imageParams?.aspectRatio).toLowerCase();
+  const shouldInferImageGeometryFromReference =
+    hasImageReference && (!requestedImageAspectRatio || requestedImageAspectRatio === "auto");
+  const inferredReferenceAspectRatio = normalizeAspectRatio(referenceAspectRatio, "");
  const ratio = isVideo
    ? normalizeVideoAspectRatio(videoParams?.ratio, endpoint)
-    : normalizeAspectRatio(imageParams?.aspectRatio, "1:1");
+    : inferredReferenceAspectRatio || normalizeAspectRatio(imageParams?.aspectRatio, "1:1");
  const videoResolution = normalizeVideoResolution(videoParams?.resolution, endpoint);
  const imageDimensions = resolveImageDimensions(imageParams?.aspectRatio, imageParams?.imageSize);
  const videoDimensions = resolveVideoDimensions(ratio, videoResolution);
@ -615,9 +764,26 @@ function buildKreaBody(params: {
  if (fields.has("prompt")) body.prompt = prompt;
  if (fields.has("batchSize")) body.batchSize = normalizeBatchSize(imageParams?.imageCount);

-  if (fields.has("width")) body.width = isVideo ? videoDimensions.width : imageDimensions.width;
-  if (fields.has("height")) body.height = isVideo ? videoDimensions.height : imageDimensions.height;
-  if (fields.has("aspectRatio")) body.aspectRatio = ratio === "auto" ? "1:1" : ratio;
+  if (
+    fields.has("width") &&
+    (isVideo || !shouldInferImageGeometryFromReference || required.has("width"))
+  ) {
+    body.width = isVideo ? videoDimensions.width : imageDimensions.width;
+  }
+  if (
+    fields.has("height") &&
+    (isVideo || !shouldInferImageGeometryFromReference || required.has("height"))
+  ) {
+    body.height = isVideo ? videoDimensions.height : imageDimensions.height;
+  }
+  if (fields.has("aspectRatio") && shouldInferImageGeometryFromReference && inferredReferenceAspectRatio) {
+    body.aspectRatio = inferredReferenceAspectRatio;
+  } else if (
+    fields.has("aspectRatio") &&
+    (isVideo || !shouldInferImageGeometryFromReference || required.has("aspectRatio"))
+  ) {
+    body.aspectRatio = ratio === "auto" ? "1:1" : ratio;
+  }
  if (fields.has("ratio")) body.ratio = ratio === "auto" ? "16:9" : ratio;
  if (fields.has("resolution")) {
    body.resolution = isVideo ? videoResolution : normalizeImageSizeLabel(imageParams?.imageSize);
@ -749,21 +915,36 @@ export function createKrea({ apiKey, baseURL }: KreaConfig) {
          const normalizedVideoUrls = (
            await Promise.all(videoUrls.map((url) => normalizeAssetUrl(url)))
          ).filter(Boolean);
-
          const requestFields =
            preset?.requestFields ||
            (kind === "video"
              ? ["prompt", "aspectRatio", "duration", "startImage", "endImage", "referenceImages"]
              : ["prompt", "width", "height", "imageUrls", "imageUrl", "batchSize"]);
          const requiredFields = preset?.requiredFields || ["prompt"];
+          const requestedImageAspectRatio = normalizeText(options.imageParams?.aspectRatio).toLowerCase();
+          const referenceAspectRatio =
+            kind !== "video" &&
+            normalizedImageUrls[0] &&
+            hasKreaSourceImageField(requestFields) &&
+            (!requestedImageAspectRatio || requestedImageAspectRatio === "auto")
+              ? await inferReferenceAspectRatioFromImageUrl(normalizedImageUrls[0])
+              : undefined;
+          const promptForKrea = buildReferencePreservingPrompt({
+            endpoint,
+            kind,
+            prompt,
+            imageUrls: normalizedImageUrls,
+            requestFields,
+          });
          const { body, imageSize } = buildKreaBody({
            endpoint,
            kind,
            requestFields,
            requiredFields,
-            prompt,
+            prompt: promptForKrea,
            imageUrls: normalizedImageUrls,
            videoUrls: normalizedVideoUrls,
+            referenceAspectRatio,
            imageParams: options.imageParams,
            videoParams: options.videoParams,
          });
@ -774,6 +955,9 @@ export function createKrea({ apiKey, baseURL }: KreaConfig) {
            requestFields: Object.keys(body),
            imageCount: normalizedImageUrls.length,
            videoCount: normalizedVideoUrls.length,
+            referenceAspectRatio,
+            referenceFields: summarizeKreaReferenceFields(body),
+            referencePreservingPrompt: promptForKrea !== prompt,
          });

          const job = await requestKreaJob(baseURL, apiKey, endpoint, body);
--- a/lib/ai/runtime/task-boundary-context-pruner.ts
+++ b/lib/ai/runtime/task-boundary-context-pruner.ts
@ -62,7 +62,18 @@ export function pruneTaskBoundaryContext(params: {
  let nextContextPayload = params.contextPayload;
  let nextAutoContextText = params.autoContextText;

-  if (!params.decision.shouldUseCanvasContext) {
+  if (shouldPreferSelectedOnly) {
+    nextContextPayload = params.contextPayload
+      ? {
+          ...params.contextPayload,
+          focusPolicy: "selected",
+          selectedNodeIds: selectedSubset.selectedNodeIds,
+          selectedAssetIds: selectedSubset.selectedAssetIds,
+          canvasImageNodes: selectedSubset.selectedNodes,
+        }
+      : undefined;
+    nextAutoContextText = buildSelectedCanvasSummary(selectedSubset.selectedNodes);
+  } else if (!params.decision.shouldUseCanvasContext) {
    nextContextPayload = params.contextPayload
      ? {
          source: params.contextPayload.source,
@ -74,17 +85,6 @@ export function pruneTaskBoundaryContext(params: {
        }
      : undefined;
    nextAutoContextText = undefined;
-  } else if (shouldPreferSelectedOnly) {
-    nextContextPayload = params.contextPayload
-      ? {
-          ...params.contextPayload,
-          focusPolicy: "selected",
-          selectedNodeIds: selectedSubset.selectedNodeIds,
-          selectedAssetIds: selectedSubset.selectedAssetIds,
-          canvasImageNodes: selectedSubset.selectedNodes,
-        }
-      : undefined;
-    nextAutoContextText = buildSelectedCanvasSummary(selectedSubset.selectedNodes);
  }

  return {