From d058ff32e61f9c653537292215f83509a5e95f7b Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 3 Jul 2026 14:04:49 +0800 Subject: [PATCH] fix: support scale adjust provider mask params --- lib/ai/providers/openai-image.ts | 9 + lib/canvas/generate-image.ts | 896 ++++++++++++++++++++++++++++++- 2 files changed, 895 insertions(+), 10 deletions(-) diff --git a/lib/ai/providers/openai-image.ts b/lib/ai/providers/openai-image.ts index 28eef4685..1dd0ee129 100644 --- a/lib/ai/providers/openai-image.ts +++ b/lib/ai/providers/openai-image.ts @@ -19,6 +19,7 @@ export interface OpenAIImageParams { quality?: string; background?: string; outputFormat?: string; + maskImageUrl?: string; } interface UploadedImage { @@ -643,6 +644,7 @@ async function requestOpenAIImageGeneration(params: { aspectRatio?: string; imageSizeLabel?: "1K" | "2K" | "4K"; imageUrls: string[]; + maskImageUrl?: string; quality?: string; background?: string; outputFormat?: string; @@ -656,6 +658,7 @@ async function requestOpenAIImageGeneration(params: { aspectRatio, imageSizeLabel, imageUrls, + maskImageUrl, quality, background, outputFormat, @@ -723,6 +726,10 @@ async function requestOpenAIImageGeneration(params: { formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename); } } + if (maskImageUrl) { + const mask = await loadInputImage(maskImageUrl); + formData.append("mask", new Blob([mask.buffer], { type: mask.mimeType }), mask.filename); + } const response = await fetch(endpoint, { method: "POST", @@ -898,6 +905,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { timeoutMs: resolveRequestTimeoutMs(baseURL, imageSize, activeModelId), useBltcyImageCompatParams: shouldUseBltcyImageCompatParams(baseURL), imageCount: imageUrls.length, + hasMask: Boolean(options.imageParams?.maskImageUrl), baseURL: normalizeBaseURL(baseURL), }); @@ -910,6 +918,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { aspectRatio: aspectRatio || undefined, imageSizeLabel: imageSize, imageUrls, + maskImageUrl: options.imageParams?.maskImageUrl, quality, background, outputFormat, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 9e5c644e4..49cf5f0bd 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -53,10 +53,20 @@ export interface CanvasLocalEditCompositeConfig { diffCompareBlurPx?: number; } +export interface CanvasScaleAdjustPatchCompositeConfig { + sourceImageUrl: string; + workspaceRegion: CanvasLocalEditCompositeRegion; + featherMinPx?: number; + featherMaxPx?: number; + featherRatio?: number; +} + export interface CanvasScaleAdjustVerificationConfig { + allowDeterministicFallback?: boolean; allowOutsideChangedWhenChangedInside?: boolean; protectedRegions?: CanvasLocalEditCompositeRegion[]; rejectFailedCandidates?: boolean; + regionPrompts?: string[]; sourceImageUrl: string; regions: CanvasLocalEditCompositeRegion[]; targetRegions?: CanvasLocalEditCompositeRegion[]; @@ -75,8 +85,10 @@ export interface CanvasGenerateImageRetryVariant { negativePrompt?: string; prompt?: string; promptExtend?: boolean; + providerImageParams?: Record; referenceImage?: string; referenceImages?: string[]; + scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; } @@ -92,6 +104,7 @@ export interface CanvasGenerateImageRequest { falSystemPrompt?: string; falThinkingLevel?: "minimal" | "high"; promptExtend?: boolean; + providerImageParams?: Record; referenceImage?: string; referenceImages?: string[]; billingFeature?: "multi-view" | "repaint"; @@ -99,6 +112,7 @@ export interface CanvasGenerateImageRequest { debugLabel?: string; debugMeta?: Record; localEditComposite?: CanvasLocalEditCompositeConfig; + scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; retryVariants?: CanvasGenerateImageRetryVariant[]; internalRetryVariantIndex?: number; @@ -168,6 +182,14 @@ const SCALE_ADJUST_MARKER_CLEANUP_MARGIN = 0.035; const SCALE_ADJUST_MARKER_DILATE_RADIUS = 3; const SCALE_ADJUST_MARKER_MIN_PIXEL_COUNT = 18; const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112; +const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42; +const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18; +const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180; +const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO = 0.72; +const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO = 0.84; +const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO = 0.14; +const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60; +const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018; const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "gemini", @@ -344,6 +366,33 @@ function normalizeLocalEditCompositeConfig( : null; } +function normalizeScaleAdjustPatchCompositeConfig( + value: CanvasGenerateImageRequest["scaleAdjustPatchComposite"] +): CanvasScaleAdjustPatchCompositeConfig | null { + if (!value || typeof value !== "object") return null; + const sourceImageUrl = String(value.sourceImageUrl || "").trim(); + if (!/^https?:\/\//i.test(sourceImageUrl)) return null; + const workspaceRegion = normalizeCanvasRegion(value.workspaceRegion); + if (!workspaceRegion) return null; + + return { + sourceImageUrl, + workspaceRegion, + featherMaxPx: normalizeOptionalNumber(value.featherMaxPx, { + max: 180, + min: 0, + }), + featherMinPx: normalizeOptionalNumber(value.featherMinPx, { + max: 100, + min: 0, + }), + featherRatio: normalizeOptionalNumber(value.featherRatio, { + max: 0.12, + min: 0, + }), + }; +} + function normalizeCanvasRegion(region: unknown): CanvasLocalEditCompositeRegion | null { if (!region || typeof region !== "object") return null; const raw = region as Partial; @@ -390,15 +439,22 @@ function normalizeScaleAdjustVerificationConfig( Boolean(region) ) : []; + const regionPrompts = Array.isArray(value.regionPrompts) + ? value.regionPrompts + .map((prompt) => String(prompt || "").trim()) + .slice(0, regions.length) + : []; return regions.length > 0 && (targetRegions.length > 0 || typeof scalePercent === "number") ? { sourceImageUrl, regions, + allowDeterministicFallback: value.allowDeterministicFallback === true, allowOutsideChangedWhenChangedInside: value.allowOutsideChangedWhenChangedInside === true, rejectFailedCandidates: value.rejectFailedCandidates === true, + ...(regionPrompts.length > 0 ? { regionPrompts } : {}), ...(targetRegions.length > 0 ? { targetRegions } : {}), ...(typeof scalePercent === "number" ? { scalePercent } : {}), ...(protectedRegions.length > 0 ? { protectedRegions } : {}), @@ -813,6 +869,138 @@ function isScaleAdjustOutsidePixelChanged(params: { ); } +async function getScaleAdjustFootprintChangeStats(params: { + generatedBuffer: Buffer; + sourceBuffer: Buffer; + sourceWidth: number; + sourceHeight: number; + verification: CanvasScaleAdjustVerificationConfig; +}) { + const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } = + params; + const normalizeToRaw = (buffer: Buffer) => + sharp(buffer, { failOn: "none" }) + .rotate() + .resize(sourceWidth, sourceHeight, { + fit: "cover", + position: "center", + }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(); + const [sourceRaw, generatedRaw] = await Promise.all([ + normalizeToRaw(sourceBuffer), + normalizeToRaw(generatedBuffer), + ]); + const shrinkStats: Array<{ + generatedOutsidePixels: number; + outsideRatio: number; + passed: boolean; + passReason: "low-source-mask" | "strict-ratio" | "removed-old-footprint" | "failed"; + regionIndex: number; + removedOutsidePixels: number; + removedOutsideRatio: number; + sourceOutsidePixels: number; + }> = []; + + for (const [index, region] of verification.regions.entries()) { + const targetRegion = + verification.targetRegions?.[index] || + (typeof verification.scalePercent === "number" + ? scaleRegionAroundCenter(region, verification.scalePercent) + : null); + if (!targetRegion) continue; + const widthFactor = region.width > 0 ? targetRegion.width / region.width : 1; + const heightFactor = region.height > 0 ? targetRegion.height / region.height : 1; + const isShrink = widthFactor < 0.92 || heightFactor < 0.92; + if (!isShrink) continue; + + const sourceBox = rectToPixelBox({ + height: sourceHeight, + region, + width: sourceWidth, + }); + const targetBox = rectToPixelBox({ + height: sourceHeight, + region: expandCanvasRegion( + targetRegion, + SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN + ), + width: sourceWidth, + }); + const sourceMask = buildScaleAdjustFallbackSubjectMask({ + box: sourceBox, + raw: sourceRaw, + width: sourceWidth, + }); + const generatedMask = buildScaleAdjustFallbackSubjectMask({ + box: sourceBox, + raw: generatedRaw, + width: sourceWidth, + }); + let sourceOutsidePixels = 0; + let generatedOutsidePixels = 0; + for (let localY = 0; localY < sourceBox.height; localY += 1) { + const y = sourceBox.top + localY; + for (let localX = 0; localX < sourceBox.width; localX += 1) { + const x = sourceBox.left + localX; + const insideTarget = + x >= targetBox.left && + x < targetBox.left + targetBox.width && + y >= targetBox.top && + y < targetBox.top + targetBox.height; + if (insideTarget) continue; + const maskIndex = localY * sourceBox.width + localX; + if (sourceMask[maskIndex] > 0) sourceOutsidePixels += 1; + if (generatedMask[maskIndex] > 0) generatedOutsidePixels += 1; + } + } + const outsideRatio = + sourceOutsidePixels > 0 ? generatedOutsidePixels / sourceOutsidePixels : 0; + const removedOutsidePixels = sourceOutsidePixels - generatedOutsidePixels; + const removedOutsideRatio = + sourceOutsidePixels > 0 ? removedOutsidePixels / sourceOutsidePixels : 0; + const minRemovedOutsidePixels = Math.max( + SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS, + Math.round( + sourceOutsidePixels * SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO + ) + ); + const lowSourceMask = + sourceOutsidePixels < SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS; + const strictRatio = + outsideRatio <= SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MAX_RATIO; + const removedOldFootprint = + outsideRatio <= SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO && + removedOutsidePixels >= minRemovedOutsidePixels; + const passed = lowSourceMask || strictRatio || removedOldFootprint; + shrinkStats.push({ + generatedOutsidePixels, + outsideRatio, + passed, + passReason: lowSourceMask + ? "low-source-mask" + : strictRatio + ? "strict-ratio" + : removedOldFootprint + ? "removed-old-footprint" + : "failed", + regionIndex: index, + removedOutsidePixels, + removedOutsideRatio, + sourceOutsidePixels, + }); + } + + return { + failedRegionCount: shrinkStats.filter((item) => !item.passed).length, + passed: shrinkStats.every((item) => item.passed), + shrinkRegionCount: shrinkStats.length, + shrinkStats, + }; +} + async function loadScaleAdjustSourceImage( verification: CanvasScaleAdjustVerificationConfig ) { @@ -1036,6 +1224,7 @@ async function applyScaleAdjustVerificationToImages(params: { changedInside: boolean; cleanedImage?: ParsedImage; diffScores: number[]; + footprintStats: Awaited>; image: ParsedImage; index: number; outsideChanged: boolean; @@ -1048,7 +1237,13 @@ async function applyScaleAdjustVerificationToImages(params: { const generatedUrl = image.originalUrl || image.thumbnailUrl; const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); const changeRegions = getScaleAdjustEffectiveChangeRegions(verification); - const [diffScores, changeDiffScores, outsideDiffScores, outsidePixelStats] = await Promise.all([ + const [ + diffScores, + changeDiffScores, + outsideDiffScores, + outsidePixelStats, + footprintStats, + ] = await Promise.all([ getScaleAdjustDiffScores({ generatedBuffer, regions: verification.regions, @@ -1077,10 +1272,18 @@ async function applyScaleAdjustVerificationToImages(params: { sourceWidth: sourceImage.sourceWidth, verification, }), + getScaleAdjustFootprintChangeStats({ + generatedBuffer, + sourceBuffer: sourceImage.sourceBuffer, + sourceHeight: sourceImage.sourceHeight, + sourceWidth: sourceImage.sourceWidth, + verification, + }), ]); - const changedInside = changeDiffScores.some( + const visuallyChangedInside = changeDiffScores.some( (score) => score >= SCALE_ADJUST_VISUAL_DIFF_THRESHOLD ); + const changedInside = visuallyChangedInside && footprintStats.passed; const outsideChanged = isScaleAdjustOutsideChanged(outsideDiffScores) || isScaleAdjustOutsidePixelChanged(outsidePixelStats); @@ -1088,6 +1291,21 @@ async function applyScaleAdjustVerificationToImages(params: { console.info("[Canvas Generate Image] Scale adjust result guard:", { debugLabel, changedInside, + footprintChanged: footprintStats.passed, + footprintStats: { + failedRegionCount: footprintStats.failedRegionCount, + shrinkRegionCount: footprintStats.shrinkRegionCount, + shrinkStats: footprintStats.shrinkStats.map((item) => ({ + generatedOutsidePixels: item.generatedOutsidePixels, + outsideRatio: Number(item.outsideRatio.toFixed(3)), + passed: item.passed, + passReason: item.passReason, + regionIndex: item.regionIndex, + removedOutsidePixels: item.removedOutsidePixels, + removedOutsideRatio: Number(item.removedOutsideRatio.toFixed(3)), + sourceOutsidePixels: item.sourceOutsidePixels, + })), + }, imageIndex: index, outsideChanged, outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))), @@ -1118,6 +1336,21 @@ async function applyScaleAdjustVerificationToImages(params: { imageIndex: index, reasons: guardWarnings, changedInside, + footprintChanged: footprintStats.passed, + footprintStats: { + failedRegionCount: footprintStats.failedRegionCount, + shrinkRegionCount: footprintStats.shrinkRegionCount, + shrinkStats: footprintStats.shrinkStats.map((item) => ({ + generatedOutsidePixels: item.generatedOutsidePixels, + outsideRatio: Number(item.outsideRatio.toFixed(3)), + passed: item.passed, + passReason: item.passReason, + regionIndex: item.regionIndex, + removedOutsidePixels: item.removedOutsidePixels, + removedOutsideRatio: Number(item.removedOutsideRatio.toFixed(3)), + sourceOutsidePixels: item.sourceOutsidePixels, + })), + }, outsideChanged, outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))), outsidePixelStats: { @@ -1153,6 +1386,7 @@ async function applyScaleAdjustVerificationToImages(params: { changedInside, cleanedImage, diffScores, + footprintStats, image, index, outsideChanged, @@ -1174,6 +1408,7 @@ async function applyScaleAdjustVerificationToImages(params: { changedInside, cleanedImage, diffScores, + footprintStats, image, index, outsideChanged, @@ -1253,6 +1488,415 @@ async function applyScaleAdjustVerificationToImages(params: { return [selected.cleanedImage || selected.image]; } +function getRawPixelIndex(width: number, x: number, y: number) { + return (y * width + x) * 3; +} + +function normalizeScaleAdjustFocusText(input: unknown) { + return String(input || "") + .trim() + .toLowerCase() + .normalize("NFKC") + .replace(/[「」『』【】[\]()（）{}]/g, " ") + .replace(/[_/|:：,，.。+]+/g, " ") + .replace(/\s+/g, " "); +} + +function getScaleAdjustFallbackFocusDirection(prompt: unknown) { + const text = normalizeScaleAdjustFocusText(prompt); + const compact = text.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, ""); + if (text.includes("left") || compact.includes("左")) return "left"; + if (text.includes("right") || compact.includes("右")) return "right"; + if (text.includes("top") || compact.includes("上")) return "top"; + if (text.includes("bottom") || compact.includes("下")) return "bottom"; + return ""; +} + +function getScaleAdjustFallbackFocusedSourceBox(params: { + prompt?: string; + sourceBox: ReturnType; +}) { + const { prompt, sourceBox } = params; + const direction = getScaleAdjustFallbackFocusDirection(prompt); + if (!direction) return sourceBox; + + const minWidth = Math.max(1, Math.round(sourceBox.width * 0.42)); + const minHeight = Math.max(1, Math.round(sourceBox.height * 0.42)); + let left = sourceBox.left; + let top = sourceBox.top; + let width = sourceBox.width; + let height = sourceBox.height; + + if (direction === "left") { + width = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); + } else if (direction === "right") { + const nextWidth = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); + left = sourceBox.left + sourceBox.width - nextWidth; + width = nextWidth; + } else if (direction === "top") { + height = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); + } else if (direction === "bottom") { + const nextHeight = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); + top = sourceBox.top + sourceBox.height - nextHeight; + height = nextHeight; + } + + return { + left, + top, + width, + height, + }; +} + +function getLuma(r: number, g: number, b: number) { + return r * 0.299 + g * 0.587 + b * 0.114; +} + +function dilateMask(params: { + height: number; + mask: Uint8Array; + radius: number; + width: number; +}) { + const { height, mask, radius, width } = params; + if (radius <= 0) return mask; + const output = new Uint8Array(mask.length); + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + if (mask[y * width + x] === 0) continue; + const left = Math.max(0, x - radius); + const right = Math.min(width - 1, x + radius); + const top = Math.max(0, y - radius); + const bottom = Math.min(height - 1, y + radius); + for (let yy = top; yy <= bottom; yy += 1) { + for (let xx = left; xx <= right; xx += 1) { + output[yy * width + xx] = 255; + } + } + } + } + return output; +} + +function estimateScaleAdjustFallbackBackground(params: { + box: ReturnType; + raw: Buffer; + width: number; +}) { + const { box, raw, width } = params; + const samples: Array<{ b: number; g: number; luma: number; r: number }> = []; + for (let y = box.top; y < box.top + box.height; y += 1) { + for (let x = box.left; x < box.left + box.width; x += 1) { + const index = getRawPixelIndex(width, x, y); + const r = raw[index] || 0; + const g = raw[index + 1] || 0; + const b = raw[index + 2] || 0; + samples.push({ b, g, luma: getLuma(r, g, b), r }); + } + } + if (samples.length === 0) { + return { b: 245, g: 245, luma: 245, r: 245 }; + } + samples.sort((left, right) => right.luma - left.luma); + const keepCount = Math.max(16, Math.ceil(samples.length * 0.35)); + const kept = samples.slice(0, keepCount); + const totals = kept.reduce( + (acc, sample) => { + acc.r += sample.r; + acc.g += sample.g; + acc.b += sample.b; + acc.luma += sample.luma; + return acc; + }, + { b: 0, g: 0, luma: 0, r: 0 } + ); + return { + b: totals.b / kept.length, + g: totals.g / kept.length, + luma: totals.luma / kept.length, + r: totals.r / kept.length, + }; +} + +function buildScaleAdjustFallbackSubjectMask(params: { + box: ReturnType; + raw: Buffer; + width: number; +}) { + const { box, raw, width } = params; + const background = estimateScaleAdjustFallbackBackground(params); + const mask = new Uint8Array(box.width * box.height); + for (let localY = 0; localY < box.height; localY += 1) { + const y = box.top + localY; + for (let localX = 0; localX < box.width; localX += 1) { + if (localX >= box.width * 0.94) continue; + const x = box.left + localX; + const rawIndex = getRawPixelIndex(width, x, y); + const r = raw[rawIndex] || 0; + const g = raw[rawIndex + 1] || 0; + const b = raw[rawIndex + 2] || 0; + const luma = getLuma(r, g, b); + const colorDistance = Math.sqrt( + (r - background.r) ** 2 + + (g - background.g) ** 2 + + (b - background.b) ** 2 + ); + if ( + (colorDistance >= SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD && + luma <= background.luma - SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP) || + luma <= Math.min(118, background.luma - 55) + ) { + mask[localY * box.width + localX] = 255; + } + } + } + return dilateMask({ + height: box.height, + mask, + radius: Math.max(1, Math.round(Math.min(box.width, box.height) * 0.018)), + width: box.width, + }); +} + +function inpaintScaleAdjustMaskedRegion(params: { + background: { b: number; g: number; r: number }; + box: ReturnType; + mask: Uint8Array; + raw: Buffer; + width: number; +}) { + const { background, box, mask, raw, width } = params; + const working = Buffer.from(raw); + const pending = new Uint8Array(mask); + let remaining = pending.reduce((sum, value) => sum + (value ? 1 : 0), 0); + const offsets = [ + [-1, 0], + [1, 0], + [0, -1], + [0, 1], + [-1, -1], + [1, -1], + [-1, 1], + [1, 1], + ]; + + for ( + let iteration = 0; + iteration < SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS && remaining > 0; + iteration += 1 + ) { + const nextValues: Array<{ b: number; g: number; index: number; r: number }> = []; + for (let localY = 0; localY < box.height; localY += 1) { + for (let localX = 0; localX < box.width; localX += 1) { + const maskIndex = localY * box.width + localX; + if (pending[maskIndex] === 0) continue; + let count = 0; + let totalB = 0; + let totalG = 0; + let totalR = 0; + for (const [offsetX, offsetY] of offsets) { + const nx = localX + offsetX; + const ny = localY + offsetY; + if (nx < 0 || ny < 0 || nx >= box.width || ny >= box.height) continue; + if (pending[ny * box.width + nx] !== 0) continue; + const rawIndex = getRawPixelIndex(width, box.left + nx, box.top + ny); + totalR += working[rawIndex] || 0; + totalG += working[rawIndex + 1] || 0; + totalB += working[rawIndex + 2] || 0; + count += 1; + } + if (count > 0) { + nextValues.push({ + b: Math.round(totalB / count), + g: Math.round(totalG / count), + index: maskIndex, + r: Math.round(totalR / count), + }); + } + } + } + if (nextValues.length === 0) break; + for (const value of nextValues) { + const localX = value.index % box.width; + const localY = Math.floor(value.index / box.width); + const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY); + working[rawIndex] = value.r; + working[rawIndex + 1] = value.g; + working[rawIndex + 2] = value.b; + pending[value.index] = 0; + remaining -= 1; + } + } + + if (remaining > 0) { + for (let localY = 0; localY < box.height; localY += 1) { + for (let localX = 0; localX < box.width; localX += 1) { + const maskIndex = localY * box.width + localX; + if (pending[maskIndex] === 0) continue; + const rawIndex = getRawPixelIndex(width, box.left + localX, box.top + localY); + working[rawIndex] = Math.round(background.r); + working[rawIndex + 1] = Math.round(background.g); + working[rawIndex + 2] = Math.round(background.b); + } + } + } + + return working; +} + +async function buildScaleAdjustFallbackObjectPng(params: { + box: ReturnType; + mask: Uint8Array; + sourceBuffer: Buffer; + targetBox: ReturnType; +}) { + const { box, mask, sourceBuffer, targetBox } = params; + const cropBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .extract(box) + .removeAlpha() + .png({ compressionLevel: 6 }) + .toBuffer(); + const alphaBuffer = await sharp(mask, { + raw: { channels: 1, height: box.height, width: box.width }, + }) + .blur(Math.max(0.7, Math.min(box.width, box.height) * 0.012)) + .png({ compressionLevel: 6 }) + .toBuffer(); + + return sharp(cropBuffer, { failOn: "none" }) + .joinChannel(alphaBuffer) + .resize(targetBox.width, targetBox.height, { fit: "fill" }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + +async function buildScaleAdjustDeterministicFallbackImages(params: { + debugLabel: string; + verification: CanvasScaleAdjustVerificationConfig; +}) { + const { debugLabel, verification } = params; + const sourceImage = await loadScaleAdjustSourceImage(verification); + if (!sourceImage) return []; + let baseRaw = await sharp(sourceImage.sourceBuffer, { failOn: "none" }) + .resize(sourceImage.sourceWidth, sourceImage.sourceHeight, { + fit: "cover", + position: "center", + }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(); + const overlays: Array<{ input: Buffer; left: number; top: number }> = []; + + for (const [index, region] of verification.regions.entries()) { + const targetRegion = + verification.targetRegions?.[index] || + (typeof verification.scalePercent === "number" + ? scaleRegionAroundCenter(region, verification.scalePercent) + : null); + if (!targetRegion) continue; + const sourceBox = rectToPixelBox({ + height: sourceImage.sourceHeight, + region, + width: sourceImage.sourceWidth, + }); + const targetBox = rectToPixelBox({ + height: sourceImage.sourceHeight, + region: targetRegion, + width: sourceImage.sourceWidth, + }); + const prompt = verification.regionPrompts?.[index] || ""; + const focusedSourceBox = getScaleAdjustFallbackFocusedSourceBox({ + prompt, + sourceBox, + }); + const mask = buildScaleAdjustFallbackSubjectMask({ + box: focusedSourceBox, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + const maskedPixels = mask.reduce((sum, value) => sum + (value ? 1 : 0), 0); + if ( + maskedPixels < + Math.max(24, focusedSourceBox.width * focusedSourceBox.height * 0.015) + ) { + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback skipped sparse mask:", { + debugLabel, + maskedPixels, + regionIndex: index, + }); + continue; + } + const background = estimateScaleAdjustFallbackBackground({ + box: focusedSourceBox, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + const objectBuffer = await buildScaleAdjustFallbackObjectPng({ + box: focusedSourceBox, + mask, + sourceBuffer: sourceImage.sourceBuffer, + targetBox, + }); + baseRaw = inpaintScaleAdjustMaskedRegion({ + background, + box: focusedSourceBox, + mask, + raw: baseRaw, + width: sourceImage.sourceWidth, + }); + overlays.push({ + input: objectBuffer, + left: targetBox.left, + top: targetBox.top, + }); + } + + if (overlays.length === 0) return []; + const baseBuffer = await sharp(baseRaw, { + raw: { + channels: 3, + height: sourceImage.sourceHeight, + width: sourceImage.sourceWidth, + }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const mergedBuffer = await sharp(baseBuffer, { failOn: "none" }) + .composite(overlays) + .png({ compressionLevel: 6 }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + .resize(512, 512, { fit: "inside", withoutEnlargement: true }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-deterministic-fallback-${suffix}.png`, + bufferToArrayBuffer(mergedBuffer) + ), + uploadFile( + `canvas-scale-adjust-deterministic-fallback-thumb-${suffix}.png`, + bufferToArrayBuffer(thumbnailBuffer) + ), + ]); + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback generated:", { + debugLabel, + originalUrl: originalUpload.url, + regionCount: verification.regions.length, + thumbnailUrl: thumbnailUpload.url, + }); + + return [ + { + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + }, + ] satisfies ParsedImage[]; +} + function buildLocalEditCompositeMaskSvg(params: { width: number; height: number; @@ -1328,6 +1972,54 @@ function buildLocalEditCompositeMaskSvg(params: { `); } +function buildScaleAdjustPatchCompositeMaskSvg(params: { + width: number; + height: number; + featherPx: number; + touchesLeft: boolean; + touchesTop: boolean; + touchesRight: boolean; + touchesBottom: boolean; +}) { + const { height, width } = params; + const featherPx = Math.max(0, Math.round(params.featherPx)); + if (featherPx <= 0) { + return Buffer.from(` + + + + `); + } + + const spill = Math.max(2, Math.round(featherPx * 2.5)); + const softLeft = params.touchesLeft ? -spill : 0; + const softTop = params.touchesTop ? -spill : 0; + const softRight = params.touchesRight ? width + spill : width; + const softBottom = params.touchesBottom ? height + spill : height; + const coreInset = Math.max(0, Math.round(featherPx * 1.25)); + const coreLeft = params.touchesLeft ? 0 : Math.min(coreInset, Math.floor(width / 3)); + const coreTop = params.touchesTop ? 0 : Math.min(coreInset, Math.floor(height / 3)); + const coreRight = params.touchesRight + ? width + : Math.max(coreLeft + 1, width - Math.min(coreInset, Math.floor(width / 3))); + const coreBottom = params.touchesBottom + ? height + : Math.max(coreTop + 1, height - Math.min(coreInset, Math.floor(height / 3))); + + return Buffer.from(` + + + + + + + + + + + `); +} + function resolveLocalEditCompositeFeatherPx(params: { composite: CanvasLocalEditCompositeConfig; width: number; @@ -1563,6 +2255,133 @@ async function applyLocalEditCompositeToImages(params: { return mergedImages; } +function resolveScaleAdjustPatchCompositeFeatherPx(params: { + composite: CanvasScaleAdjustPatchCompositeConfig; + width: number; + height: number; +}) { + const { composite, height, width } = params; + const minPx = + typeof composite.featherMinPx === "number" ? composite.featherMinPx : 10; + const maxPx = + typeof composite.featherMaxPx === "number" + ? Math.max(minPx, composite.featherMaxPx) + : 42; + const ratio = + typeof composite.featherRatio === "number" ? composite.featherRatio : 0.018; + + return Math.max( + minPx, + Math.min(maxPx, Math.round(Math.min(width, height) * ratio)) + ); +} + +async function applyScaleAdjustPatchCompositeToImages(params: { + images: ParsedImage[]; + composite: CanvasScaleAdjustPatchCompositeConfig | null; + debugLabel: string; +}) { + const { images, composite, debugLabel } = params; + if (!composite || images.length === 0) return images; + + const sourceBuffer = await sharp( + await fetchImageBufferFromUrl(composite.sourceImageUrl), + { failOn: "none" } + ) + .rotate() + .png({ compressionLevel: 6 }) + .toBuffer(); + const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata(); + const sourceWidth = Number(sourceMetadata.width || 0); + const sourceHeight = Number(sourceMetadata.height || 0); + if (sourceWidth <= 0 || sourceHeight <= 0) { + throw new Error("比例调整局部结果合成失败：无法识别原图尺寸"); + } + + const workspaceBox = rectToPixelBox({ + height: sourceHeight, + region: composite.workspaceRegion, + width: sourceWidth, + }); + const featherPx = resolveScaleAdjustPatchCompositeFeatherPx({ + composite, + height: workspaceBox.height, + width: workspaceBox.width, + }); + const touchesLeft = workspaceBox.left <= 0; + const touchesTop = workspaceBox.top <= 0; + const touchesRight = workspaceBox.left + workspaceBox.width >= sourceWidth; + const touchesBottom = workspaceBox.top + workspaceBox.height >= sourceHeight; + const patchMask = buildScaleAdjustPatchCompositeMaskSvg({ + featherPx, + height: workspaceBox.height, + touchesBottom, + touchesLeft, + touchesRight, + touchesTop, + width: workspaceBox.width, + }); + + const mergedImages: ParsedImage[] = []; + for (const [index, image] of images.entries()) { + const generatedUrl = image.originalUrl || image.thumbnailUrl; + const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); + const patchOverlayBuffer = await sharp(generatedBuffer, { failOn: "none" }) + .rotate() + .resize(workspaceBox.width, workspaceBox.height, { + fit: "cover", + position: "center", + }) + .ensureAlpha() + .composite([{ input: patchMask, blend: "dest-in" }]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .ensureAlpha() + .composite([ + { + input: patchOverlayBuffer, + left: workspaceBox.left, + top: workspaceBox.top, + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + .resize(512, 512, { fit: "inside", withoutEnlargement: true }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const suffix = `${Date.now()}-${index + 1}-${Math.random() + .toString(36) + .slice(2, 8)}`; + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-ai-patch-composited-${suffix}.png`, + bufferToArrayBuffer(mergedBuffer) + ), + uploadFile( + `canvas-scale-adjust-ai-patch-composited-thumb-${suffix}.png`, + bufferToArrayBuffer(thumbnailBuffer) + ), + ]); + mergedImages.push({ + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + }); + } + + console.info("[Canvas Generate Image] Scale adjust AI patch composite applied:", { + debugLabel, + featherPx, + imageCount: mergedImages.length, + sourceHeight, + sourceWidth, + workspaceBox, + }); + + return mergedImages; +} + function getProviderBatchLimit(providerType: string) { switch (providerType) { case "ruxa-image": @@ -1981,6 +2800,10 @@ export async function generateCanvasImage( : 1; const promptExtend = typeof request.promptExtend === "boolean" ? request.promptExtend : undefined; + const providerImageParams = + request.providerImageParams && typeof request.providerImageParams === "object" + ? request.providerImageParams + : undefined; const ownerKey = typeof request.ownerKey === "string" && request.ownerKey.trim() ? request.ownerKey.trim() @@ -2002,6 +2825,9 @@ export async function generateCanvasImage( const localEditComposite = normalizeLocalEditCompositeConfig( request.localEditComposite ); + const scaleAdjustPatchComposite = normalizeScaleAdjustPatchCompositeConfig( + request.scaleAdjustPatchComposite + ); const scaleAdjustVerification = normalizeScaleAdjustVerificationConfig( request.scaleAdjustVerification ); @@ -2227,6 +3053,7 @@ export async function generateCanvasImage( providerType: provider.type, ...debugMeta, hasLocalEditComposite: Boolean(localEditComposite), + hasScaleAdjustPatchComposite: Boolean(scaleAdjustPatchComposite), hasScaleAdjustVerification: Boolean(scaleAdjustVerification), }); @@ -2252,6 +3079,7 @@ export async function generateCanvasImage( ]; const imageParams = { + ...(providerImageParams || {}), aspectRatio, safetyTolerance: request.falSafetyTolerance, imageSize, @@ -2553,10 +3381,16 @@ export async function generateCanvasImage( debugLabel, images: baseImages, }); + const scaleAdjustPatchCompositedImages = + await applyScaleAdjustPatchCompositeToImages({ + composite: scaleAdjustPatchComposite, + debugLabel, + images: compositedImages, + }); const finalImages = await applyScaleAdjustVerificationToImages({ debugLabel, - images: compositedImages, + images: scaleAdjustPatchCompositedImages, verification: scaleAdjustVerification, }); console.info("[Canvas Generate Image] Completed:", { @@ -2741,20 +3575,62 @@ export async function generateCanvasImage( } if (scaleAdjustRejection?.images?.length) { - console.warn("[Canvas Generate Image] Returning rejected scale adjust candidate because no fallback remains:", { + console.warn("[Canvas Generate Image] Rejecting scale adjust candidate because no effective edit was detected:", { debugLabel, imageCount: scaleAdjustRejection.images.length, + rejectedImageUrls: scaleAdjustRejection.images.map((image) => ({ + originalUrl: image.originalUrl, + thumbnailUrl: image.thumbnailUrl, + })), reason: scaleAdjustRejection.reason, selectedModelRef: selectedModel.model.id, selectedModelId: selectedModel.model.modelId, ...debugMeta, }); - return { - images: scaleAdjustRejection.images, - model: selectedModel.model.displayName, - modelRef: selectedModel.model.id, - prompt: resultPrompt, - }; + if (scaleAdjustVerification?.allowDeterministicFallback) { + try { + const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({ + debugLabel, + verification: scaleAdjustVerification, + }); + if (fallbackImages.length > 0) { + const verifiedFallbackImages = await applyScaleAdjustVerificationToImages({ + debugLabel, + images: fallbackImages, + verification: { + ...scaleAdjustVerification, + rejectFailedCandidates: true, + }, + }); + console.warn("[Canvas Generate Image] Returning deterministic scale adjust fallback:", { + debugLabel, + fallbackImageCount: verifiedFallbackImages.length, + rejectedReason: scaleAdjustRejection.reason, + selectedModelRef: selectedModel.model.id, + selectedModelId: selectedModel.model.modelId, + ...debugMeta, + }); + return { + images: verifiedFallbackImages, + model: selectedModel.model.displayName, + modelRef: selectedModel.model.id, + prompt: resultPrompt, + }; + } + } catch (fallbackError) { + console.warn("[Canvas Generate Image] Scale adjust deterministic fallback failed:", { + debugLabel, + error: + fallbackError instanceof Error + ? fallbackError.message + : String(fallbackError), + selectedModelRef: selectedModel.model.id, + selectedModelId: selectedModel.model.modelId, + ...debugMeta, + }); + } + } + throw caughtError; } throw caughtError;