diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 20750cab9..64247dcca 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -167,6 +167,9 @@ const SCALE_ADJUST_MARKER_CLEANUP_MARGIN = 0.035; const SCALE_ADJUST_MARKER_DILATE_RADIUS = 3; const SCALE_ADJUST_MARKER_MIN_PIXEL_COUNT = 18; const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112; +const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42; +const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18; +const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180; const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "gemini", @@ -1252,6 +1255,350 @@ async function applyScaleAdjustVerificationToImages(params: { return [selected.cleanedImage || selected.image]; } +function getRawPixelIndex(width: number, x: number, y: number) { + return (y * width + x) * 3; +} + +function getLuma(r: number, g: number, b: number) { + return r * 0.299 + g * 0.587 + b * 0.114; +} + +function dilateMask(params: { + height: number; + mask: Uint8Array; + radius: number; + width: number; +}) { + const { height, mask, radius, width } = params; + if (radius <= 0) return mask; + const output = new Uint8Array(mask.length); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + if (mask[y * width + x] === 0) continue; + const left = Math.max(0, x - radius); + const right = Math.min(width - 1, x + radius); + const top = Math.max(0, y - radius); + const bottom = Math.min(height - 1, y + radius); + for (let yy = top; yy <= bottom; yy += 1) { + for (let xx = left; xx <= right; xx += 1) { + output[yy * width + xx] = 255; + } + } + } + } + return output; +} + +function estimateScaleAdjustFallbackBackground(params: { + box: ReturnType; + raw: Buffer; + width: number; +}) { + const { box, raw, width } = params; + const samples: Array<{ b: number; g: number; luma: number; r: number }> = []; + for (let y = box.top; y < box.top + box.height; y += 1) { + for (let x = box.left; x < box.left + box.width; x += 1) { + const index = getRawPixelIndex(width, x, y); + const r = raw[index] || 0; + const g = raw[index + 1] || 0; + const b = raw[index + 2] || 0; + samples.push({ b, g, luma: getLuma(r, g, b), r }); + } + } + if (samples.length === 0) { + return { b: 245, g: 245, luma: 245, r: 245 }; + } + samples.sort((left, right) => right.luma - left.luma); + const keepCount = Math.max(16, Math.ceil(samples.length * 0.35)); + const kept = samples.slice(0, keepCount); + const totals = kept.reduce( + (acc, sample) => { + acc.r += sample.r; + acc.g += sample.g; + acc.b += sample.b; + acc.luma += sample.luma; + return acc; + }, + { b: 0, g: 0, luma: 0, r: 0 } + ); + return { + b: totals.b / kept.length, + g: totals.g / kept.length, + luma: totals.luma / kept.length, + r: totals.r / kept.length, + }; +} + +function buildScaleAdjustFallbackSubjectMask(params: { + box: ReturnType; + raw: Buffer; + width: number; +}) { + const { box, raw, width } = params; + const background = estimateScaleAdjustFallbackBackground(params); + const mask = new Uint8Array(box.width * box.height); + for (let localY = 0; localY < box.height; localY += 1) { + const y = box.top + localY; + for (let localX = 0; localX < box.width; localX += 1) { + if (localX >= box.width * 0.94) continue; + const x = box.left + localX; + const rawIndex = getRawPixelIndex(width, x, y); + const r = raw[rawIndex] || 0; + const g = raw[rawIndex + 1] || 0; + const b = raw[rawIndex + 2] || 0; + const luma = getLuma(r, g, b); + const colorDistance = Math.sqrt( + (r - background.r) ** 2 + + (g - background.g) ** 2 + + (b - background.b) ** 2 + ); + if ( + (colorDistance >= SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD && + luma <= background.luma - SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP) || + luma <= Math.min(118, background.luma - 55) + ) { + mask[localY * box.width + localX] = 255; + } + } + } + return dilateMask({ + height: box.height, + mask, + radius: Math.max(1, Math.round(Math.min(box.width, box.height) * 0.018)), + width: box.width, + }); +} + +function inpaintScaleAdjustMaskedRegion(params: { + background: { b: number; g: number; r: number }; + box: ReturnType; + mask: Uint8Array; + raw: Buffer; + width: number; +}) { + const { background, box, mask, raw, width } = params; + const working = Buffer.from(raw); + const pending = new Uint8Array(mask); + let remaining = pending.reduce((sum, value) => sum + (value ? 1 : 0), 0); + const offsets = [ + [-1, 0], + [1, 0], + [0, -1], + [0, 1], + [-1, -1], + [1, -1], + [-1, 1], + [1, 1], + ]; + + for ( + let iteration = 0; + iteration < SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS && remaining > 0; + iteration += 1 + ) { + const nextValues: Array<{ b: number; g: number; index: number; r: number }> = []; + for (let localY = 0; localY < box.height; localY += 1) { + for (let localX = 0; localX < box.width; localX += 1) { + const maskIndex = localY * box.width + localX; + if (pending[maskIndex] === 0) continue; + let count = 0; + let totalB = 0; + let totalG = 0; + let totalR = 0; + for (const [offsetX, offsetY] of offsets) { + const nx = localX + offsetX; + const ny = localY + offsetY; + if (nx < 0 || ny < 0 || nx >= box.width || ny >= box.height) continue; + if (pending[ny * box.width + nx] !== 0) continue; + const rawIndex = getRawPixelIndex(width, box.left + nx, box.top + ny); + totalR += working[rawIndex] || 0; + totalG += working[rawIndex + 1] || 0; + totalB += working[rawIndex + 2] || 0; + count += 1; + } + if (count > 0) { + nextValues.push({ + b: Math.round(totalB / count), + g: Math.round(totalG / count), + index: maskIndex, + r: Math.round(totalR / count), + }); + } + } + } + if (nextValues.length === 0) break; + for (const value of nextValues) { + const localX = value.index % box.width; + const localY = Math.floor(value.index / box.width); + const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY); + working[rawIndex] = value.r; + working[rawIndex + 1] = value.g; + working[rawIndex + 2] = value.b; + pending[value.index] = 0; + remaining -= 1; + } + } + + if (remaining > 0) { + for (let localY = 0; localY < box.height; localY += 1) { + for (let localX = 0; localX < box.width; localX += 1) { + const maskIndex = localY * box.width + localX; + if (pending[maskIndex] === 0) continue; + const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY); + working[rawIndex] = Math.round(background.r); + working[rawIndex + 1] = Math.round(background.g); + working[rawIndex + 2] = Math.round(background.b); + } + } + } + + return working; +} + +async function buildScaleAdjustFallbackObjectPng(params: { + box: ReturnType; + mask: Uint8Array; + sourceBuffer: Buffer; + targetBox: ReturnType; +}) { + const { box, mask, sourceBuffer, targetBox } = params; + const cropBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .extract(box) + .removeAlpha() + .png({ compressionLevel: 6 }) + .toBuffer(); + const alphaBuffer = await sharp(mask, { + raw: { channels: 1, height: box.height, width: box.width }, + }) + .blur(Math.max(0.7, Math.min(box.width, box.height) * 0.012)) + .png({ compressionLevel: 6 }) + .toBuffer(); + + return sharp(cropBuffer, { failOn: "none" }) + .joinChannel(alphaBuffer) + .resize(targetBox.width, targetBox.height, { fit: "fill" }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + +async function buildScaleAdjustDeterministicFallbackImages(params: { + debugLabel: string; + verification: CanvasScaleAdjustVerificationConfig; +}) { + const { debugLabel, verification } = params; + const sourceImage = await loadScaleAdjustSourceImage(verification); + if (!sourceImage) return []; + let baseRaw = await sharp(sourceImage.sourceBuffer, { failOn: "none" }) + .resize(sourceImage.sourceWidth, sourceImage.sourceHeight, { + fit: "cover", + position: "center", + }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(); + const overlays: Array<{ buffer: Buffer; left: number; top: number }> = []; + + for (const [index, region] of verification.regions.entries()) { + const targetRegion = + verification.targetRegions?.[index] || + (typeof verification.scalePercent === "number" + ? scaleRegionAroundCenter(region, verification.scalePercent) + : null); + if (!targetRegion) continue; + const sourceBox = rectToPixelBox({ + height: sourceImage.sourceHeight, + region, + width: sourceImage.sourceWidth, + }); + const targetBox = rectToPixelBox({ + height: sourceImage.sourceHeight, + region: targetRegion, + width: sourceImage.sourceWidth, + }); + const mask = buildScaleAdjustFallbackSubjectMask({ + box: sourceBox, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + const maskedPixels = mask.reduce((sum, value) => sum + (value ? 1 : 0), 0); + if (maskedPixels < Math.max(24, sourceBox.width * sourceBox.height * 0.015)) { + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback skipped sparse mask:", { + debugLabel, + maskedPixels, + regionIndex: index, + }); + continue; + } + const background = estimateScaleAdjustFallbackBackground({ + box: sourceBox, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + const objectBuffer = await buildScaleAdjustFallbackObjectPng({ + box: sourceBox, + mask, + sourceBuffer: sourceImage.sourceBuffer, + targetBox, + }); + baseRaw = inpaintScaleAdjustMaskedRegion({ + background, + box: sourceBox, + mask, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + overlays.push({ + buffer: objectBuffer, + left: targetBox.left, + top: targetBox.top, + }); + } + + if (overlays.length === 0) return []; + const baseBuffer = await sharp(baseRaw, { + raw: { + channels: 3, + height: sourceImage.sourceHeight, + width: sourceImage.sourceWidth, + }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const mergedBuffer = await sharp(baseBuffer, { failOn: "none" }) + .composite(overlays) + .png({ compressionLevel: 6 }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + .resize(512, 512, { fit: "inside", withoutEnlargement: true }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-deterministic-fallback-${suffix}.png`, + bufferToArrayBuffer(mergedBuffer) + ), + uploadFile( + `canvas-scale-adjust-deterministic-fallback-thumb-${suffix}.png`, + bufferToArrayBuffer(thumbnailBuffer) + ), + ]); + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback generated:", { + debugLabel, + originalUrl: originalUpload.url, + regionCount: verification.regions.length, + thumbnailUrl: thumbnailUpload.url, + }); + + return [ + { + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + }, + ] satisfies ParsedImage[]; +} + function buildLocalEditCompositeMaskSvg(params: { width: number; height: number; @@ -2683,6 +3030,49 @@ export async function generateCanvasImage( selectedModelId: selectedModel.model.modelId, ...debugMeta, }); + if (scaleAdjustVerification) { + try { + const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({ + debugLabel, + verification: scaleAdjustVerification, + }); + if (fallbackImages.length > 0) { + const verifiedFallbackImages = await applyScaleAdjustVerificationToImages({ + debugLabel, + images: fallbackImages, + verification: { + ...scaleAdjustVerification, + rejectFailedCandidates: true, + }, + }); + console.warn("[Canvas Generate Image] Returning deterministic scale adjust fallback:", { + debugLabel, + fallbackImageCount: verifiedFallbackImages.length, + rejectedReason: scaleAdjustRejection.reason, + selectedModelRef: selectedModel.model.id, + selectedModelId: selectedModel.model.modelId, + ...debugMeta, + }); + return { + images: verifiedFallbackImages, + model: selectedModel.model.displayName, + modelRef: selectedModel.model.id, + prompt: resultPrompt, + }; + } + } catch (fallbackError) { + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback failed:", { + debugLabel, + error: + fallbackError instanceof Error + ? fallbackError.message + : String(fallbackError), + selectedModelRef: selectedModel.model.id, + selectedModelId: selectedModel.model.modelId, + ...debugMeta, + }); + } + } throw caughtError; }