fix: add deterministic scale adjust fallback

lib/canvas/generate-image.ts

@@ -167,6 +167,9 @@ const SCALE_ADJUST_MARKER_CLEANUP_MARGIN = 0.035;
 const SCALE_ADJUST_MARKER_DILATE_RADIUS = 3;
 const SCALE_ADJUST_MARKER_MIN_PIXEL_COUNT = 18;
 const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112;
+const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42;
+const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18;
+const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180;
 
 const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
   "gemini",
@@ -1252,6 +1255,350 @@ async function applyScaleAdjustVerificationToImages(params: {
   return [selected.cleanedImage || selected.image];
 }
 
+function getRawPixelIndex(width: number, x: number, y: number) {
+  return (y * width + x) * 3;
+}
+
+function getLuma(r: number, g: number, b: number) {
+  return r * 0.299 + g * 0.587 + b * 0.114;
+}
+
+function dilateMask(params: {
+  height: number;
+  mask: Uint8Array;
+  radius: number;
+  width: number;
+}) {
+  const { height, mask, radius, width } = params;
+  if (radius <= 0) return mask;
+  const output = new Uint8Array(mask.length);
+  for (let y = 0; y < height; y += 1) {
+    for (let x = 0; x < width; x += 1) {
+      if (mask[y * width + x] === 0) continue;
+      const left = Math.max(0, x - radius);
+      const right = Math.min(width - 1, x + radius);
+      const top = Math.max(0, y - radius);
+      const bottom = Math.min(height - 1, y + radius);
+      for (let yy = top; yy <= bottom; yy += 1) {
+        for (let xx = left; xx <= right; xx += 1) {
+          output[yy * width + xx] = 255;
+        }
+      }
+    }
+  }
+  return output;
+}
+
+function estimateScaleAdjustFallbackBackground(params: {
+  box: ReturnType<typeof rectToPixelBox>;
+  raw: Buffer;
+  width: number;
+}) {
+  const { box, raw, width } = params;
+  const samples: Array<{ b: number; g: number; luma: number; r: number }> = [];
+  for (let y = box.top; y < box.top + box.height; y += 1) {
+    for (let x = box.left; x < box.left + box.width; x += 1) {
+      const index = getRawPixelIndex(width, x, y);
+      const r = raw[index] || 0;
+      const g = raw[index + 1] || 0;
+      const b = raw[index + 2] || 0;
+      samples.push({ b, g, luma: getLuma(r, g, b), r });
+    }
+  }
+  if (samples.length === 0) {
+    return { b: 245, g: 245, luma: 245, r: 245 };
+  }
+  samples.sort((left, right) => right.luma - left.luma);
+  const keepCount = Math.max(16, Math.ceil(samples.length * 0.35));
+  const kept = samples.slice(0, keepCount);
+  const totals = kept.reduce(
+    (acc, sample) => {
+      acc.r += sample.r;
+      acc.g += sample.g;
+      acc.b += sample.b;
+      acc.luma += sample.luma;
+      return acc;
+    },
+    { b: 0, g: 0, luma: 0, r: 0 }
+  );
+  return {
+    b: totals.b / kept.length,
+    g: totals.g / kept.length,
+    luma: totals.luma / kept.length,
+    r: totals.r / kept.length,
+  };
+}
+
+function buildScaleAdjustFallbackSubjectMask(params: {
+  box: ReturnType<typeof rectToPixelBox>;
+  raw: Buffer;
+  width: number;
+}) {
+  const { box, raw, width } = params;
+  const background = estimateScaleAdjustFallbackBackground(params);
+  const mask = new Uint8Array(box.width * box.height);
+  for (let localY = 0; localY < box.height; localY += 1) {
+    const y = box.top + localY;
+    for (let localX = 0; localX < box.width; localX += 1) {
+      if (localX >= box.width * 0.94) continue;
+      const x = box.left + localX;
+      const rawIndex = getRawPixelIndex(width, x, y);
+      const r = raw[rawIndex] || 0;
+      const g = raw[rawIndex + 1] || 0;
+      const b = raw[rawIndex + 2] || 0;
+      const luma = getLuma(r, g, b);
+      const colorDistance = Math.sqrt(
+        (r - background.r) ** 2 +
+          (g - background.g) ** 2 +
+          (b - background.b) ** 2
+      );
+      if (
+        (colorDistance >= SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD &&
+          luma <= background.luma - SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP) ||
+        luma <= Math.min(118, background.luma - 55)
+      ) {
+        mask[localY * box.width + localX] = 255;
+      }
+    }
+  }
+  return dilateMask({
+    height: box.height,
+    mask,
+    radius: Math.max(1, Math.round(Math.min(box.width, box.height) * 0.018)),
+    width: box.width,
+  });
+}
+
+function inpaintScaleAdjustMaskedRegion(params: {
+  background: { b: number; g: number; r: number };
+  box: ReturnType<typeof rectToPixelBox>;
+  mask: Uint8Array;
+  raw: Buffer;
+  width: number;
+}) {
+  const { background, box, mask, raw, width } = params;
+  const working = Buffer.from(raw);
+  const pending = new Uint8Array(mask);
+  let remaining = pending.reduce((sum, value) => sum + (value ? 1 : 0), 0);
+  const offsets = [
+    [-1, 0],
+    [1, 0],
+    [0, -1],
+    [0, 1],
+    [-1, -1],
+    [1, -1],
+    [-1, 1],
+    [1, 1],
+  ];
+
+  for (
+    let iteration = 0;
+    iteration < SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS && remaining > 0;
+    iteration += 1
+  ) {
+    const nextValues: Array<{ b: number; g: number; index: number; r: number }> = [];
+    for (let localY = 0; localY < box.height; localY += 1) {
+      for (let localX = 0; localX < box.width; localX += 1) {
+        const maskIndex = localY * box.width + localX;
+        if (pending[maskIndex] === 0) continue;
+        let count = 0;
+        let totalB = 0;
+        let totalG = 0;
+        let totalR = 0;
+        for (const [offsetX, offsetY] of offsets) {
+          const nx = localX + offsetX;
+          const ny = localY + offsetY;
+          if (nx < 0 || ny < 0 || nx >= box.width || ny >= box.height) continue;
+          if (pending[ny * box.width + nx] !== 0) continue;
+          const rawIndex = getRawPixelIndex(width, box.left + nx, box.top + ny);
+          totalR += working[rawIndex] || 0;
+          totalG += working[rawIndex + 1] || 0;
+          totalB += working[rawIndex + 2] || 0;
+          count += 1;
+        }
+        if (count > 0) {
+          nextValues.push({
+            b: Math.round(totalB / count),
+            g: Math.round(totalG / count),
+            index: maskIndex,
+            r: Math.round(totalR / count),
+          });
+        }
+      }
+    }
+    if (nextValues.length === 0) break;
+    for (const value of nextValues) {
+      const localX = value.index % box.width;
+      const localY = Math.floor(value.index / box.width);
+      const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY);
+      working[rawIndex] = value.r;
+      working[rawIndex + 1] = value.g;
+      working[rawIndex + 2] = value.b;
+      pending[value.index] = 0;
+      remaining -= 1;
+    }
+  }
+
+  if (remaining > 0) {
+    for (let localY = 0; localY < box.height; localY += 1) {
+      for (let localX = 0; localX < box.width; localX += 1) {
+        const maskIndex = localY * box.width + localX;
+        if (pending[maskIndex] === 0) continue;
+        const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY);
+        working[rawIndex] = Math.round(background.r);
+        working[rawIndex + 1] = Math.round(background.g);
+        working[rawIndex + 2] = Math.round(background.b);
+      }
+    }
+  }
+
+  return working;
+}
+
+async function buildScaleAdjustFallbackObjectPng(params: {
+  box: ReturnType<typeof rectToPixelBox>;
+  mask: Uint8Array;
+  sourceBuffer: Buffer;
+  targetBox: ReturnType<typeof rectToPixelBox>;
+}) {
+  const { box, mask, sourceBuffer, targetBox } = params;
+  const cropBuffer = await sharp(sourceBuffer, { failOn: "none" })
+    .extract(box)
+    .removeAlpha()
+    .png({ compressionLevel: 6 })
+    .toBuffer();
+  const alphaBuffer = await sharp(mask, {
+    raw: { channels: 1, height: box.height, width: box.width },
+  })
+    .blur(Math.max(0.7, Math.min(box.width, box.height) * 0.012))
+    .png({ compressionLevel: 6 })
+    .toBuffer();
+
+  return sharp(cropBuffer, { failOn: "none" })
+    .joinChannel(alphaBuffer)
+    .resize(targetBox.width, targetBox.height, { fit: "fill" })
+    .png({ compressionLevel: 6 })
+    .toBuffer();
+}
+
+async function buildScaleAdjustDeterministicFallbackImages(params: {
+  debugLabel: string;
+  verification: CanvasScaleAdjustVerificationConfig;
+}) {
+  const { debugLabel, verification } = params;
+  const sourceImage = await loadScaleAdjustSourceImage(verification);
+  if (!sourceImage) return [];
+  let baseRaw = await sharp(sourceImage.sourceBuffer, { failOn: "none" })
+    .resize(sourceImage.sourceWidth, sourceImage.sourceHeight, {
+      fit: "cover",
+      position: "center",
+    })
+    .removeAlpha()
+    .toColorspace("srgb")
+    .raw()
+    .toBuffer();
+  const overlays: Array<{ buffer: Buffer; left: number; top: number }> = [];
+
+  for (const [index, region] of verification.regions.entries()) {
+    const targetRegion =
+      verification.targetRegions?.[index] ||
+      (typeof verification.scalePercent === "number"
+        ? scaleRegionAroundCenter(region, verification.scalePercent)
+        : null);
+    if (!targetRegion) continue;
+    const sourceBox = rectToPixelBox({
+      height: sourceImage.sourceHeight,
+      region,
+      width: sourceImage.sourceWidth,
+    });
+    const targetBox = rectToPixelBox({
+      height: sourceImage.sourceHeight,
+      region: targetRegion,
+      width: sourceImage.sourceWidth,
+    });
+    const mask = buildScaleAdjustFallbackSubjectMask({
+      box: sourceBox,
+      raw: baseRaw,
+      width: sourceImage.sourceWidth,
+    });
+    const maskedPixels = mask.reduce((sum, value) => sum + (value ? 1 : 0), 0);
+    if (maskedPixels < Math.max(24, sourceBox.width * sourceBox.height * 0.015)) {
+      console.warn("[Canvas Generate Image] Scale adjust deterministic fallback skipped sparse mask:", {
+        debugLabel,
+        maskedPixels,
+        regionIndex: index,
+      });
+      continue;
+    }
+    const background = estimateScaleAdjustFallbackBackground({
+      box: sourceBox,
+      raw: baseRaw,
+      width: sourceImage.sourceWidth,
+    });
+    const objectBuffer = await buildScaleAdjustFallbackObjectPng({
+      box: sourceBox,
+      mask,
+      sourceBuffer: sourceImage.sourceBuffer,
+      targetBox,
+    });
+    baseRaw = inpaintScaleAdjustMaskedRegion({
+      background,
+      box: sourceBox,
+      mask,
+      raw: baseRaw,
+      width: sourceImage.sourceWidth,
+    });
+    overlays.push({
+      buffer: objectBuffer,
+      left: targetBox.left,
+      top: targetBox.top,
+    });
+  }
+
+  if (overlays.length === 0) return [];
+  const baseBuffer = await sharp(baseRaw, {
+    raw: {
+      channels: 3,
+      height: sourceImage.sourceHeight,
+      width: sourceImage.sourceWidth,
+    },
+  })
+    .png({ compressionLevel: 6 })
+    .toBuffer();
+  const mergedBuffer = await sharp(baseBuffer, { failOn: "none" })
+    .composite(overlays)
+    .png({ compressionLevel: 6 })
+    .toBuffer();
+  const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" })
+    .resize(512, 512, { fit: "inside", withoutEnlargement: true })
+    .png({ compressionLevel: 6 })
+    .toBuffer();
+  const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  const [originalUpload, thumbnailUpload] = await Promise.all([
+    uploadFile(
+      `canvas-scale-adjust-deterministic-fallback-${suffix}.png`,
+      bufferToArrayBuffer(mergedBuffer)
+    ),
+    uploadFile(
+      `canvas-scale-adjust-deterministic-fallback-thumb-${suffix}.png`,
+      bufferToArrayBuffer(thumbnailBuffer)
+    ),
+  ]);
+  console.warn("[Canvas Generate Image] Scale adjust deterministic fallback generated:", {
+    debugLabel,
+    originalUrl: originalUpload.url,
+    regionCount: verification.regions.length,
+    thumbnailUrl: thumbnailUpload.url,
+  });
+
+  return [
+    {
+      originalUrl: originalUpload.url,
+      thumbnailUrl: thumbnailUpload.url,
+    },
+  ] satisfies ParsedImage[];
+}
+
 function buildLocalEditCompositeMaskSvg(params: {
   width: number;
   height: number;
@@ -2683,6 +3030,49 @@ export async function generateCanvasImage(
       selectedModelId: selectedModel.model.modelId,
       ...debugMeta,
     });
+    if (scaleAdjustVerification) {
+      try {
+        const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({
+          debugLabel,
+          verification: scaleAdjustVerification,
+        });
+        if (fallbackImages.length > 0) {
+          const verifiedFallbackImages = await applyScaleAdjustVerificationToImages({
+            debugLabel,
+            images: fallbackImages,
+            verification: {
+              ...scaleAdjustVerification,
+              rejectFailedCandidates: true,
+            },
+          });
+          console.warn("[Canvas Generate Image] Returning deterministic scale adjust fallback:", {
+            debugLabel,
+            fallbackImageCount: verifiedFallbackImages.length,
+            rejectedReason: scaleAdjustRejection.reason,
+            selectedModelRef: selectedModel.model.id,
+            selectedModelId: selectedModel.model.modelId,
+            ...debugMeta,
+          });
+          return {
+            images: verifiedFallbackImages,
+            model: selectedModel.model.displayName,
+            modelRef: selectedModel.model.id,
+            prompt: resultPrompt,
+          };
+        }
+      } catch (fallbackError) {
+        console.warn("[Canvas Generate Image] Scale adjust deterministic fallback failed:", {
+          debugLabel,
+          error:
+            fallbackError instanceof Error
+              ? fallbackError.message
+              : String(fallbackError),
+          selectedModelRef: selectedModel.model.id,
+          selectedModelId: selectedModel.model.modelId,
+          ...debugMeta,
+        });
+      }
+    }
     throw caughtError;
   }