diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index e51ac90d5..004fa0011 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2,7 +2,10 @@ import { NextRequest, NextResponse } from "next/server"; import sharp from "sharp"; import { auth } from "@/app/(auth)/auth"; -import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; +import { + executeRmbgJob, + fetchImageBufferFromUrl, +} from "@/lib/ai/rmbg/service"; import { createCanvasImageTask, enqueueCanvasImageTask, @@ -104,8 +107,12 @@ const SCALE_ADJUST_CROP_MAX_EDGE = 1536; const SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN = 0.045; const SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN = 0.18; const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb"; +const SCALE_ADJUST_LAYOUT_GUIDE_WASH_OPACITY = 0.46; const SCALE_ADJUST_STRUCTURE_LOCK_MIN_MASK_RATIO = 0.006; const SCALE_ADJUST_STRUCTURE_LOCK_COMPONENT_MIN_RATIO = 0.00018; +const SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO = 0.012; +const SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO = 0.92; +const SCALE_ADJUST_RMBG_ALPHA_CLEAN_THRESHOLD = 42; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; @@ -128,6 +135,11 @@ type ScaleAdjustPatchReference = { height: number; }; }; +type ScaleAdjustStructureLockedComposite = { + method?: "rmbg-cutout" | "pixel-mask"; + overlay: { input: Buffer; left: number; top: number }; + wash: { input: Buffer; left: number; top: number }; +}; type AiModelWithProviderForSelection = Awaited< ReturnType >[number]; @@ -314,6 +326,28 @@ function isScaleAdjustCoordinateGuideModel( return getScaleAdjustModelFamily(entry) === "nano-banana-2"; } +function isScaleAdjustFalNanoBananaEditModel( + entry: AiModelWithProviderForSelection | null | undefined +) { + if ( + !entry || + entry.provider.type !== "fal-image" || + getScaleAdjustModelFamily(entry) !== "nano-banana-2" + ) { + return false; + } + const label = `${entry.model.displayName || ""} ${entry.model.modelId || ""}` + .toLowerCase() + .replace(/\s+/g, " "); + return label.includes("/edit") || /\bedit\b/.test(label); +} + +function isScaleAdjustHealthyModel( + entry: AiModelWithProviderForSelection | null | undefined +) { + return !entry?.model?.healthStatus || entry.model.healthStatus === "healthy"; +} + function getScaleAdjustProviderPriority(providerType: string) { switch (providerType) { case "ruxa-image": @@ -387,6 +421,9 @@ function compareScaleAdjustProviderCandidates( function resolveScaleAdjustPreferredModelId(params: { configuredModelId: string; honorRequestedModelBeforeGpt?: boolean; + preferFalNanoBananaEdit?: boolean; + preferMaskCapableGpt?: boolean; + preferProviderTypes?: string[]; requestedModelId: string; models: AiModelWithProviderForSelection[]; }) { @@ -416,6 +453,39 @@ function resolveScaleAdjustPreferredModelId(params: { params.requestedModelId || params.configuredModelId ) ); + if (params.preferMaskCapableGpt) { + const maskCapableGptImage2 = gptImage2Candidates + .filter(isScaleAdjustMaskCapableModel) + .filter(isScaleAdjustHealthyModel); + if (maskCapableGptImage2[0]) { + return maskCapableGptImage2[0].model.id; + } + } + for (const providerType of params.preferProviderTypes || []) { + const providerCandidate = gptImage2Candidates.find( + (entry) => + entry.provider.type === providerType && isScaleAdjustHealthyModel(entry) + ); + if (providerCandidate) { + return providerCandidate.model.id; + } + } + if (params.preferFalNanoBananaEdit) { + const falEditCandidate = params.models + .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) + .filter(isScaleAdjustFalNanoBananaEditModel) + .filter(isScaleAdjustHealthyModel) + .sort((left, right) => + compareScaleAdjustProviderCandidates( + left, + right, + params.requestedModelId || params.configuredModelId + ) + )[0]; + if (falEditCandidate) { + return falEditCandidate.model.id; + } + } if (gptImage2Candidates[0]) { return gptImage2Candidates[0].model.id; } @@ -640,6 +710,39 @@ function hasScaleAdjustEnlargeTarget( }); } +function hasScaleAdjustShrinkTarget( + regions: NormalizedLocalEditRegion[], + scalePercent?: ScaleAdjustPercent | null +) { + return regions.some((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const widthFactor = + region.rect.width > 0 ? targetRect.width / region.rect.width : 1; + const heightFactor = + region.rect.height > 0 ? targetRect.height / region.rect.height : 1; + return widthFactor < 0.92 || heightFactor < 0.92; + }); +} + +function hasScaleAdjustStructureTarget( + regions: NormalizedLocalEditRegion[], + scalePercent?: ScaleAdjustPercent | null +) { + return regions.some((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const widthFactor = + region.rect.width > 0 ? targetRect.width / region.rect.width : 1; + const heightFactor = + region.rect.height > 0 ? targetRect.height / region.rect.height : 1; + return ( + widthFactor > 1.08 || + heightFactor > 1.08 || + widthFactor < 0.92 || + heightFactor < 0.92 + ); + }); +} + function expandRectWithMargins( rect: NormalizedLocalEditRegion["rect"], marginX: number, @@ -769,6 +872,39 @@ function buildScaleAdjustCompositeRegions( }); } +function buildScaleAdjustPatchCompositeRegions( + regions: NormalizedLocalEditRegion[], + scalePercent?: ScaleAdjustPercent | null +) { + return regions.flatMap((region) => [ + region.rect, + getScaleAdjustTargetRect(region, scalePercent), + ]); +} + +function buildScaleAdjustProtectedChangeRegions( + regions: NormalizedLocalEditRegion[], + scalePercent?: ScaleAdjustPercent | null +) { + return regions.flatMap((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const unionRect = getRectUnion(region.rect, targetRect); + const marginX = Math.min( + 0.045, + Math.max(0.012, unionRect.width * 0.08) + ); + const marginY = Math.min( + 0.045, + Math.max(0.012, unionRect.height * 0.08) + ); + + return [ + expandRectWithMargins(region.rect, marginX, marginY), + expandRectWithMargins(targetRect, marginX, marginY), + ]; + }); +} + function getScaleAdjustPatchWorkspaceRect( regions: NormalizedLocalEditRegion[], scalePercent?: ScaleAdjustPercent | null @@ -1029,29 +1165,73 @@ function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) ); } +function formatScaleAdjustWashColor( + background?: { b: number; g: number; r: number } | null +) { + if (!background) return SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR; + const r = clamp(Math.round(background.r), 24, 242); + const g = clamp(Math.round(background.g), 24, 242); + const b = clamp(Math.round(background.b), 24, 242); + return `rgb(${r}, ${g}, ${b})`; +} + function buildScaleAdjustSourceFootprintWashSvg(params: { - boxes: Array<{ left: number; top: number; width: number; height: number }>; + boxes: Array<{ + background?: { b: number; g: number; r: number } | null; + left: number; + top: number; + width: number; + height: number; + }>; height: number; width: number; }) { const { boxes, height, width } = params; - const blur = Math.max(4, Math.min(22, Math.round(Math.min(width, height) * 0.016))); + const defs = boxes + .map((box, index) => { + const blur = Math.max( + 14, + Math.min(72, Math.round(Math.min(box.width, box.height) * 0.14)) + ); + return ` + + `; + }) + .join(""); const rects = boxes - .map((box) => { + .map((box, index) => { + const blur = Math.max( + 14, + Math.min(72, Math.round(Math.min(box.width, box.height) * 0.14)) + ); + const pad = Math.max(blur, Math.round(Math.min(box.width, box.height) * 0.055)); const left = Math.max(0, box.left - blur); const top = Math.max(0, box.top - blur); const right = Math.min(width, box.left + box.width + blur); const bottom = Math.min(height, box.top + box.height + blur); - return ``; + const rectWidth = Math.max(1, right - left); + const rectHeight = Math.max(1, bottom - top); + const radius = Math.max( + 8, + Math.min(Math.round(blur * 1.8), Math.round(rectWidth / 3), Math.round(rectHeight / 3)) + ); + const innerLeft = Math.max(0, box.left + pad); + const innerTop = Math.max(0, box.top + pad); + const innerRight = Math.min(width, box.left + box.width - pad); + const innerBottom = Math.min(height, box.top + box.height - pad); + const fill = formatScaleAdjustWashColor(box.background); + const inner = + innerRight > innerLeft + 4 && innerBottom > innerTop + 4 + ? `` + : ""; + return `${inner}`; }) .join(""); return Buffer.from( ` - - - + ${defs} ${rects} ` @@ -1149,6 +1329,51 @@ function isScaleAdjustLikelyWhiteFabricPixel( ); } +function isScaleAdjustPersonLikePrompt(prompt: string) { + const text = normalizeScaleAdjustModelText(prompt); + return /person|people|human|woman|man|girl|boy|model|face|body|hand|arm|leg|clothing|人物|人像|真人|模特|女人|男人|女孩|男孩|小孩|脸|身体|手|胳膊|腿|衣服/.test( + text + ); +} + +function hasScaleAdjustPersonPixelSignal(params: { + box: { left: number; top: number; width: number; height: number }; + raw: Buffer; + width: number; +}) { + const { box, raw, width } = params; + let sampled = 0; + let skin = 0; + let whiteFabric = 0; + const step = Math.max(1, Math.floor(Math.min(box.width, box.height) / 260)); + + for (let localY = 0; localY < box.height; localY += step) { + const y = box.top + localY; + for (let localX = 0; localX < box.width; localX += step) { + const x = box.left + localX; + const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); + const r = raw[rawIndex] || 0; + const g = raw[rawIndex + 1] || 0; + const b = raw[rawIndex + 2] || 0; + const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); + sampled += 1; + if (isScaleAdjustLikelySkinPixel(r, g, b)) { + skin += 1; + } else if ( + isScaleAdjustLikelyWhiteFabricPixel(r, g, b, luma, saturation) && + luma < 242 + ) { + whiteFabric += 1; + } + } + } + + if (sampled <= 0) return false; + const skinRatio = skin / sampled; + const fabricRatio = whiteFabric / sampled; + return skin >= 18 && skinRatio >= 0.0025 && fabricRatio >= 0.035; +} + function estimateScaleAdjustBoxBackground(params: { box: { left: number; top: number; width: number; height: number }; raw: Buffer; @@ -1230,33 +1455,55 @@ function dilateScaleAdjustStructureMask(params: { function buildScaleAdjustStructureInitialMask(params: { box: { left: number; top: number; width: number; height: number }; + includePersonPixels?: boolean; raw: Buffer; width: number; }) { - const { box, raw, width } = params; + const { box, includePersonPixels = false, raw, width } = params; const background = estimateScaleAdjustBoxBackground(params); const mask = new Uint8Array(box.width * box.height); for (let localY = 0; localY < box.height; localY += 1) { const y = box.top + localY; for (let localX = 0; localX < box.width; localX += 1) { - if (localX >= box.width * 0.9) continue; const x = box.left + localX; const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); const r = raw[rawIndex] || 0; const g = raw[rawIndex + 1] || 0; const b = raw[rawIndex + 2] || 0; const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - if (isScaleAdjustLikelySkinPixel(r, g, b)) continue; - if (isScaleAdjustLikelyWhiteFabricPixel(r, g, b, luma, saturation)) { - continue; - } - + const likelySkin = isScaleAdjustLikelySkinPixel(r, g, b); + const likelyWhiteFabric = isScaleAdjustLikelyWhiteFabricPixel( + r, + g, + b, + luma, + saturation + ); const colorDistance = Math.sqrt( (r - background.r) ** 2 + (g - background.g) ** 2 + (b - background.b) ** 2 ); + + if (includePersonPixels && likelySkin) { + mask[localY * box.width + localX] = 255; + continue; + } + if ( + includePersonPixels && + likelyWhiteFabric && + luma < 246 && + (colorDistance > 12 || luma < background.luma - 4) + ) { + mask[localY * box.width + localX] = 255; + continue; + } + if (likelySkin) continue; + if (likelyWhiteFabric) { + continue; + } + const darkStructure = luma < Math.min(136, background.luma - 38); const saturatedStructure = saturation > 0.16 && luma < 218 && colorDistance > 38; @@ -1346,8 +1593,16 @@ function filterScaleAdjustStructureComponents(params: { const componentBoxArea = Math.max(1, componentWidth * componentHeight); const componentDensity = area / componentBoxArea; const centerXRatio = (minX + componentWidth / 2) / width; + const centerYRatio = (minY + componentHeight / 2) / height; + const touchesTop = minY <= height * 0.04; const touchesRight = maxX >= width * 0.88; const edgeSliver = touchesRight && componentWidth < width * 0.18; + const broadTopBackground = + touchesTop && + centerYRatio <= 0.45 && + componentWidth >= width * 0.42 && + componentHeight >= height * 0.18 && + componentDensity >= 0.18; const elongated = componentHeight > componentWidth * 2.4 || componentWidth > componentHeight * 2.4; @@ -1363,7 +1618,12 @@ function filterScaleAdjustStructureComponents(params: { componentDensity >= 0.035 && componentDensity <= 0.72; - if (likelyStructure && centeredEnough && !edgeSliver) { + if ( + likelyStructure && + centeredEnough && + !edgeSliver && + !broadTopBackground + ) { if (fillSurfaceComponent) { for (let fillY = minY; fillY <= maxY; fillY += 1) { for (let fillX = minX; fillX <= maxX; fillX += 1) { @@ -1595,15 +1855,520 @@ async function buildScaleAdjustMaskWashPatch(params: { .toBuffer(); } -async function buildScaleAdjustStructureLockedComposite(params: { +function getScaleAdjustAlphaBoundingBox(params: { + height: number; + rawRgba: Buffer; + threshold?: number; + width: number; +}) { + const { height, rawRgba, threshold = 18, width } = params; + let left = width; + let top = height; + let right = -1; + let bottom = -1; + let count = 0; + + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const alpha = rawRgba[(y * width + x) * 4 + 3] || 0; + if (alpha < threshold) continue; + left = Math.min(left, x); + top = Math.min(top, y); + right = Math.max(right, x); + bottom = Math.max(bottom, y); + count += 1; + } + } + + if (right < left || bottom < top) return null; + return { + count, + left, + top, + width: right - left + 1, + height: bottom - top + 1, + }; +} + +function thresholdScaleAdjustAlphaMask(params: { + maskBuffer: Buffer | Uint8Array; + threshold?: number; +}) { + const { maskBuffer, threshold = 96 } = params; + const output = new Uint8Array(maskBuffer.length); + for (let index = 0; index < maskBuffer.length; index += 1) { + output[index] = (maskBuffer[index] || 0) >= threshold ? 255 : 0; + } + return output; +} + +async function applyScaleAdjustAlphaMaskToRgba(params: { + alphaMask: Uint8Array; + height: number; + rawRgba: Buffer; + width: number; +}) { + const { alphaMask, height, rawRgba, width } = params; + const blur = Math.max(0.45, Math.min(width, height) * 0.0035); + const softenedAlpha = await sharp(Buffer.from(alphaMask), { + raw: { channels: 1, height, width }, + }) + .blur(blur) + .raw() + .toBuffer(); + const output = Buffer.from(rawRgba); + const pixelCount = Math.min(alphaMask.length, Math.floor(output.length / 4)); + for (let index = 0; index < pixelCount; index += 1) { + output[index * 4 + 3] = softenedAlpha[index] || 0; + } + return output; +} + +function getScaleAdjustAlphaBandingStats(params: { + height: number; + maskBuffer: Buffer | Uint8Array; + threshold?: number; + width: number; +}) { + const { height, maskBuffer, threshold = 24, width } = params; + const mask = thresholdScaleAdjustAlphaMask({ maskBuffer, threshold }); + const bbox = getScaleAdjustMaskBoundingBox({ height, mask, width }); + if (!bbox) { + return { + activeRowRatio: 0, + bboxHeightRatio: 0, + bboxWidthRatio: 0, + detected: false, + transitionRatio: 0, + wideRowRatio: 0, + }; + } + + let activeRows = 0; + let transitions = 0; + let wideRows = 0; + let previousCoverage: number | null = null; + for (let y = bbox.top; y < bbox.top + bbox.height; y += 1) { + let rowCount = 0; + for (let x = bbox.left; x < bbox.left + bbox.width; x += 1) { + if (mask[y * width + x] !== 0) rowCount += 1; + } + const coverage = rowCount / Math.max(1, bbox.width); + if (coverage >= 0.08) activeRows += 1; + if (coverage >= 0.45) wideRows += 1; + if ( + previousCoverage !== null && + Math.abs(coverage - previousCoverage) >= 0.28 + ) { + transitions += 1; + } + previousCoverage = coverage; + } + + const bboxHeightRatio = bbox.height / Math.max(1, height); + const bboxWidthRatio = bbox.width / Math.max(1, width); + const activeRowRatio = activeRows / Math.max(1, bbox.height); + const transitionRatio = transitions / Math.max(1, bbox.height - 1); + const wideRowRatio = wideRows / Math.max(1, bbox.height); + const detected = + (bboxWidthRatio >= 0.38 && + bboxHeightRatio >= 0.22 && + wideRowRatio >= 0.18 && + transitionRatio >= 0.12) || + (bboxWidthRatio >= 0.55 && + activeRowRatio >= 0.45 && + transitionRatio >= 0.09); + + return { + activeRowRatio: Number(activeRowRatio.toFixed(3)), + bboxHeightRatio: Number(bboxHeightRatio.toFixed(3)), + bboxWidthRatio: Number(bboxWidthRatio.toFixed(3)), + detected, + transitionRatio: Number(transitionRatio.toFixed(3)), + wideRowRatio: Number(wideRowRatio.toFixed(3)), + }; +} + +async function cleanScaleAdjustAlphaChannel(params: { + height: number; + referenceRawRgb?: Buffer | null; + rawRgba: Buffer; + threshold?: number; + width: number; +}) { + const { + height, + referenceRawRgb = null, + rawRgba, + threshold = SCALE_ADJUST_RMBG_ALPHA_CLEAN_THRESHOLD, + width, + } = params; + const pixelCount = Math.floor(rawRgba.length / 4); + const cleanedRaw = Buffer.from(rawRgba); + const cleanedAlpha = Buffer.alloc(pixelCount); + const range = Math.max(1, 255 - threshold); + const hasReference = + referenceRawRgb && referenceRawRgb.length >= pixelCount * 3; + + for (let index = 0; index < pixelCount; index += 1) { + const colorIndex = index * 4; + const alphaIndex = index * 4 + 3; + const sourceAlpha = rawRgba[alphaIndex] || 0; + let matteArtifact = false; + if (hasReference && sourceAlpha >= threshold) { + const r = rawRgba[colorIndex] || 0; + const g = rawRgba[colorIndex + 1] || 0; + const b = rawRgba[colorIndex + 2] || 0; + const luma = getScaleAdjustPixelLuma(r, g, b); + const channelSpread = Math.max(r, g, b) - Math.min(r, g, b); + const refIndex = index * 3; + const refLuma = getScaleAdjustPixelLuma( + referenceRawRgb?.[refIndex] || 0, + referenceRawRgb?.[refIndex + 1] || 0, + referenceRawRgb?.[refIndex + 2] || 0 + ); + matteArtifact = luma <= 18 && channelSpread <= 18 && refLuma >= 42; + } + const nextAlpha = + sourceAlpha < threshold || matteArtifact + ? 0 + : Math.max(0, Math.min(255, Math.round(((sourceAlpha - threshold) / range) * 255))); + cleanedRaw[alphaIndex] = nextAlpha; + cleanedAlpha[index] = nextAlpha; + } + + return { cleanedAlpha, cleanedRaw }; +} + +async function resizeScaleAdjustAlphaMask(params: { + height: number; + maskBuffer: Buffer; + targetHeight: number; + targetWidth: number; + width: number; +}) { + const { height, maskBuffer, targetHeight, targetWidth, width } = params; + const { data, info } = await sharp(maskBuffer, { + raw: { channels: 1, height, width }, + }) + .resize(targetWidth, targetHeight, { + background: { alpha: 1, b: 0, g: 0, r: 0 }, + fit: "contain", + kernel: sharp.kernel.lanczos3, + }) + .greyscale() + .raw() + .toBuffer({ resolveWithObject: true }); + if (info.channels === 1) return Buffer.from(data); + + const output = Buffer.alloc(targetWidth * targetHeight); + for (let index = 0; index < targetWidth * targetHeight; index += 1) { + output[index] = data[index * info.channels] || 0; + } + return output; +} + +async function buildScaleAdjustRmbgStructureLockedComposite(params: { placement: ReturnType; raw: Buffer; sourceBuffer: Buffer; + sourceImageUrl?: string; sourceWidth: number; -}) { - const { placement, raw, sourceBuffer, sourceWidth } = params; + userId?: string; +}): Promise { + const { + placement, + raw, + sourceBuffer, + sourceImageUrl, + sourceWidth, + userId, + } = params; + if (!userId) return null; + + try { + const sourceCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .extract(placement.sourceBox) + .png({ compressionLevel: 6 }) + .toBuffer(); + const cutoutResult = await executeRmbgJob({ + userId, + imageBase64: sourceCropBuffer.toString("base64"), + imageBuffer: sourceCropBuffer, + prompt: "比例调整主体抠图：只保留框选的目标人物/家具/商品主体，输出透明背景 PNG", + skipGallerySave: true, + sourceImageUrl, + }); + const cutoutBuffer = await fetchImageBufferFromUrl(cutoutResult.original_url); + const normalizedCutout = await sharp(cutoutBuffer, { failOn: "none" }) + .rotate() + .ensureAlpha() + .png({ compressionLevel: 6 }) + .toBuffer(); + const cutoutMetadata = await sharp(normalizedCutout, { failOn: "none" }).metadata(); + const cutoutWidth = Math.round(Number(cutoutMetadata.width || 0)); + const cutoutHeight = Math.round(Number(cutoutMetadata.height || 0)); + if (cutoutWidth <= 0 || cutoutHeight <= 0) return null; + const sameSizeCutout = + Math.abs(cutoutWidth - placement.sourceBox.width) <= 2 && + Math.abs(cutoutHeight - placement.sourceBox.height) <= 2; + const sourceCropRaw = sameSizeCutout + ? await sharp(sourceCropBuffer, { failOn: "none" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer() + : null; + + const cutoutRaw = await sharp(normalizedCutout, { failOn: "none" }) + .ensureAlpha() + .raw() + .toBuffer(); + const { cleanedAlpha, cleanedRaw } = await cleanScaleAdjustAlphaChannel({ + height: cutoutHeight, + referenceRawRgb: sourceCropRaw, + rawRgba: cutoutRaw, + width: cutoutWidth, + }); + const highConfidenceCutoutMask = thresholdScaleAdjustAlphaMask({ + maskBuffer: cleanedAlpha, + threshold: 96, + }); + const highConfidenceCutoutBbox = getScaleAdjustMaskBoundingBox({ + height: cutoutHeight, + mask: highConfidenceCutoutMask, + width: cutoutWidth, + }); + if (!highConfidenceCutoutBbox) return null; + const refinedCutoutRaw = await applyScaleAdjustAlphaMaskToRgba({ + alphaMask: highConfidenceCutoutMask, + height: cutoutHeight, + rawRgba: cleanedRaw, + width: cutoutWidth, + }); + const cleanedCutout = await sharp(refinedCutoutRaw, { + raw: { channels: 4, height: cutoutHeight, width: cutoutWidth }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const alphaBbox = getScaleAdjustAlphaBoundingBox({ + height: cutoutHeight, + rawRgba: refinedCutoutRaw, + width: cutoutWidth, + }); + if (!alphaBbox) return null; + + const alphaRatio = alphaBbox.count / Math.max(1, cutoutWidth * cutoutHeight); + const bboxCoversCanvas = + alphaBbox.width >= cutoutWidth * 0.97 && + alphaBbox.height >= cutoutHeight * 0.97; + if ( + alphaRatio < SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO || + (alphaRatio > SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO && bboxCoversCanvas) + ) { + return null; + } + + const fullAlphaMask = Buffer.from(highConfidenceCutoutMask); + const sourceMaskBuffer = sameSizeCutout + ? fullAlphaMask + : await resizeScaleAdjustAlphaMask({ + height: cutoutHeight, + maskBuffer: fullAlphaMask, + targetHeight: placement.sourceBox.height, + targetWidth: placement.sourceBox.width, + width: cutoutWidth, + }); + const alphaBandingStats = getScaleAdjustAlphaBandingStats({ + height: placement.sourceBox.height, + maskBuffer: sourceMaskBuffer, + width: placement.sourceBox.width, + }); + if (alphaBandingStats.detected) { + console.warn( + "[Canvas Local Edit] Scale adjust RMBG alpha banding detected; fallback to pixel mask:", + alphaBandingStats + ); + return null; + } + const background = estimateScaleAdjustBoxBackground({ + box: placement.sourceBox, + raw, + width: sourceWidth, + }); + const highConfidenceSourceMask = thresholdScaleAdjustAlphaMask({ + maskBuffer: sourceMaskBuffer, + threshold: 96, + }); + const filteredSourceMask = filterScaleAdjustStructureComponents({ + box: placement.sourceBox, + mask: highConfidenceSourceMask, + }); + const filteredSourceBbox = getScaleAdjustMaskBoundingBox({ + height: placement.sourceBox.height, + mask: filteredSourceMask, + width: placement.sourceBox.width, + }); + const sourceWashBaseMask = filteredSourceBbox + ? solidifyScaleAdjustStructureMask({ + box: placement.sourceBox, + mask: filteredSourceMask, + }) + : highConfidenceSourceMask; + const sourceWashMask = dilateScaleAdjustStructureMask({ + height: placement.sourceBox.height, + mask: sourceWashBaseMask, + radius: Math.max( + 2, + Math.round( + Math.min(placement.sourceBox.width, placement.sourceBox.height) * 0.014 + ) + ), + width: placement.sourceBox.width, + }); + const sourceWashPatch = await buildScaleAdjustMaskWashPatch({ + background, + box: placement.sourceBox, + maskBuffer: Buffer.from(sourceWashMask), + }); + + const subjectCutout = await sharp(cleanedCutout, { failOn: "none" }) + .extract(alphaBbox) + .png({ compressionLevel: 6 }) + .toBuffer(); + const widthFactor = + placement.targetBox.width / Math.max(1, placement.sourceFootprintBox.width); + const heightFactor = + placement.targetBox.height / Math.max(1, placement.sourceFootprintBox.height); + const structureScale = Math.max( + 0.05, + Math.min( + 3.5, + Math.sqrt( + Math.max(0.0025, widthFactor) * Math.max(0.0025, heightFactor) + ) + ) + ); + let placedWidth = Math.max(1, Math.round(alphaBbox.width * structureScale)); + let placedHeight = Math.max(1, Math.round(alphaBbox.height * structureScale)); + if (!sameSizeCutout) { + const maxPlacedWidth = Math.max(1, Math.round(placement.targetBox.width * 1.06)); + const maxPlacedHeight = Math.max(1, Math.round(placement.targetBox.height * 1.06)); + const fitRatio = Math.min( + 1, + maxPlacedWidth / Math.max(1, placedWidth), + maxPlacedHeight / Math.max(1, placedHeight) + ); + placedWidth = Math.max(1, Math.round(placedWidth * fitRatio)); + placedHeight = Math.max(1, Math.round(placedHeight * fitRatio)); + } + + const sourceSubjectBox = sameSizeCutout + ? { + left: placement.sourceBox.left + alphaBbox.left, + top: placement.sourceBox.top + alphaBbox.top, + width: alphaBbox.width, + height: alphaBbox.height, + } + : null; + const sourceCenterX = sourceSubjectBox + ? sourceSubjectBox.left + sourceSubjectBox.width / 2 + : placement.sourceBox.left + placement.sourceBox.width / 2; + const sourceCenterY = sourceSubjectBox + ? sourceSubjectBox.top + sourceSubjectBox.height / 2 + : placement.sourceBox.top + placement.sourceBox.height / 2; + const targetCenterX = sourceSubjectBox + ? placement.targetBox.left + + ((sourceCenterX - placement.sourceFootprintBox.left) / + Math.max(1, placement.sourceFootprintBox.width)) * + placement.targetBox.width + : placement.targetBox.left + placement.targetBox.width / 2; + const targetCenterY = sourceSubjectBox + ? placement.targetBox.top + + ((sourceCenterY - placement.sourceFootprintBox.top) / + Math.max(1, placement.sourceFootprintBox.height)) * + placement.targetBox.height + : placement.targetBox.top + placement.targetBox.height / 2; + const structureOverlay = await sharp(subjectCutout, { failOn: "none" }) + .resize(placedWidth, placedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + + console.info("[Canvas Local Edit] Scale adjust RMBG structure draft applied:", { + alphaRatio: Number(alphaRatio.toFixed(3)), + cutoutHeight, + cutoutWidth, + placedHeight, + placedWidth, + sameSizeCutout, + alphaBandingStats, + sourceHeight: placement.sourceBox.height, + sourceWidth: placement.sourceBox.width, + }); + + return { + method: "rmbg-cutout", + overlay: { + input: structureOverlay, + left: Math.round(targetCenterX - placedWidth / 2), + top: Math.round(targetCenterY - placedHeight / 2), + }, + wash: { + input: sourceWashPatch, + left: placement.sourceBox.left, + top: placement.sourceBox.top, + }, + }; + } catch (error) { + console.warn("[Canvas Local Edit] Scale adjust RMBG structure draft failed; fallback to pixel mask:", { + error: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +async function buildScaleAdjustStructureLockedComposite(params: { + includePersonPixels?: boolean; + placement: ReturnType; + raw: Buffer; + sourceBuffer: Buffer; + sourceImageUrl?: string; + sourceWidth: number; + userId?: string; +}): Promise { + const { + includePersonPixels = false, + placement, + raw, + sourceBuffer, + sourceImageUrl, + sourceWidth, + userId, + } = params; + // The current RMBG provider can bake horizontal matte stripes into furniture cutouts. + const allowRmbgStructureLock = + process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_STRUCTURE === "1" || + (includePersonPixels && + process.env.CANVAS_SCALE_ADJUST_DISABLE_PERSON_RMBG !== "1"); + const rmbgComposite = !allowRmbgStructureLock + ? null + : await buildScaleAdjustRmbgStructureLockedComposite({ + placement, + raw, + sourceBuffer, + sourceImageUrl, + sourceWidth, + userId, + }); + if (rmbgComposite) return rmbgComposite; + if (includePersonPixels) return null; + const initialMask = buildScaleAdjustStructureInitialMask({ box: placement.sourceBox, + includePersonPixels, raw, width: sourceWidth, }); @@ -1708,6 +2473,7 @@ async function buildScaleAdjustStructureLockedComposite(params: { .toBuffer(); return { + method: "pixel-mask", overlay: { input: structureOverlay, left: Math.round(targetCenterX - placedWidth / 2), @@ -1809,25 +2575,33 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { scalePercent?: ScaleAdjustPercent | null; sourceBuffer: Buffer; sourceHeight: number; + sourceImageUrl?: string; sourceWidth: number; + structureLockAppliedRef?: { value: boolean }; structureLockedDraft?: boolean; + userId?: string; }) { const { regions, scalePercent, sourceBuffer, sourceHeight, + sourceImageUrl, sourceWidth, + structureLockAppliedRef, structureLockedDraft = false, + userId, } = params; - const sourceRaw = structureLockedDraft - ? await sharp(sourceBuffer, { failOn: "none" }) - .removeAlpha() - .toColorspace("srgb") - .raw() - .toBuffer() - : null; + const sourceRaw = + structureLockedDraft || regions.length > 0 + ? await sharp(sourceBuffer, { failOn: "none" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer() + : null; const sourceFootprintBoxes: Array<{ + background?: { b: number; g: number; r: number } | null; left: number; top: number; width: number; @@ -1857,23 +2631,70 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { const shouldUseStructureLock = structureLockedDraft && sourceRaw && - (targetWidthFactor > 1.08 || targetHeightFactor > 1.08); + (targetWidthFactor > 1.08 || + targetHeightFactor > 1.08 || + targetWidthFactor < 0.92 || + targetHeightFactor < 0.92); const structureLockedComposite = shouldUseStructureLock ? await buildScaleAdjustStructureLockedComposite({ + includePersonPixels: + isScaleAdjustPersonLikePrompt(region.prompt) || + hasScaleAdjustPersonPixelSignal({ + box: placement.sourceBox, + raw: sourceRaw, + width: sourceWidth, + }), placement, raw: sourceRaw, sourceBuffer, + sourceImageUrl, sourceWidth, + userId, }) : null; if (structureLockedComposite) { + if (structureLockAppliedRef) { + structureLockAppliedRef.value = true; + } subjectWashComposites.push(structureLockedComposite.wash); composites.push(structureLockedComposite.overlay); continue; } - sourceFootprintBoxes.push(placement.sourceFootprintBox); + const shouldUseShrinkCutoutDraft = + process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_SHRINK_CUTOUT === "1" && + !structureLockedDraft && + sourceRaw && + userId && + (targetWidthFactor < 0.92 || targetHeightFactor < 0.92); + const shrinkCutoutComposite = shouldUseShrinkCutoutDraft + ? await buildScaleAdjustRmbgStructureLockedComposite({ + placement, + raw: sourceRaw, + sourceBuffer, + sourceImageUrl, + sourceWidth, + userId, + }) + : null; + + if (shrinkCutoutComposite) { + subjectWashComposites.push(shrinkCutoutComposite.wash); + composites.push(shrinkCutoutComposite.overlay); + continue; + } + + sourceFootprintBoxes.push({ + ...placement.sourceFootprintBox, + background: sourceRaw + ? estimateScaleAdjustBoxBackground({ + box: placement.sourceFootprintBox, + raw: sourceRaw, + width: sourceWidth, + }) + : null, + }); const resizedCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) .extract(placement.sourceBox) @@ -1947,20 +2768,36 @@ async function createScaleAdjustPatchReference(params: { scalePercent?: ScaleAdjustPercent | null; sourceBuffer: Buffer; sourceHeight: number; + sourceImageUrl?: string; sourceWidth: number; + userId?: string; }): Promise { - const { regions, scalePercent, sourceBuffer, sourceHeight, sourceWidth } = - params; + const { + regions, + scalePercent, + sourceBuffer, + sourceHeight, + sourceImageUrl, + sourceWidth, + userId, + } = params; const workspaceRect = getScaleAdjustPatchWorkspaceRect(regions, scalePercent); const workspacePixelRect = rectToPixels(workspaceRect, sourceWidth, sourceHeight); - const structureLockedDraft = hasScaleAdjustEnlargeTarget(regions, scalePercent); + const shouldUseStructureLockedDraft = hasScaleAdjustStructureTarget( + regions, + scalePercent + ); + const structureLockAppliedRef = { value: false }; const targetDraftBuffer = await buildScaleAdjustVisualLayoutGuideBuffer({ regions, scalePercent, sourceBuffer, sourceHeight, + sourceImageUrl, sourceWidth, - structureLockedDraft, + structureLockAppliedRef, + structureLockedDraft: shouldUseStructureLockedDraft, + userId, }); const [sourceCropBuffer, targetDraftCropBuffer] = await Promise.all([ sharp(sourceBuffer, { failOn: "none" }) @@ -1986,7 +2823,7 @@ async function createScaleAdjustPatchReference(params: { return { sourceCropUrl: sourceCropUpload.url, - structureLockedDraft, + structureLockedDraft: structureLockAppliedRef.value, targetDraftUrl: targetDraftUpload.url, workspacePixelRect, workspaceRect, @@ -2567,38 +3404,38 @@ function buildScaleAdjustCoordinatePrompt( const isEnlarge = widthFactor > 1.08 || heightFactor > 1.08; const scaleDirection = isShrink - ? "make the furniture clearly smaller" + ? "make the selected subject clearly smaller" : isEnlarge - ? "make the furniture clearly larger" - : "move/fit the furniture to the target box"; + ? "make the selected subject clearly larger" + : "move/fit the selected subject to the target box"; const footprintRule = isShrink - ? "After shrinking, the original OLD footprint outside the TARGET box must be cleared and rebuilt as natural background/floor/rug/wall; the furniture must not still touch the OLD box edges." + ? "After shrinking, the original OLD footprint outside the TARGET box must be cleared and rebuilt as natural background/floor/rug/wall; the selected subject must not still touch the OLD box edges." : isEnlarge - ? "After enlarging, the same furniture should visibly occupy more of the TARGET footprint, with natural shadows and occlusion; do not just move it or stretch one axis." + ? "After enlarging, the same selected subject should visibly occupy more of the TARGET footprint, with natural shadows and occlusion; do not just move it or stretch one axis." : "Keep the subject centered on the TARGET box without changing the full canvas."; const cropNote = cropIndex >= 0 ? options.falSafe ? ` Input image ${cropIndex + 3} is an extra identity and silhouette reference.` - : ` Input image ${cropIndex + 3} is a clean crop reference for this same furniture identity and silhouette; use it only to recognize the subject, never paste it, and do not preserve its old size.` + : ` Input image ${cropIndex + 3} is a clean crop reference for this same target subject identity and silhouette; use it only to recognize the subject, never paste it, and do not preserve its old size.` : ""; if (options.falSafe) { const actionPhrase = isShrink - ? "resize the furniture proportionally smaller and place it inside the final footprint" + ? "resize the selected subject proportionally smaller and place it inside the final footprint" : isEnlarge - ? "resize the same furniture proportionally larger toward the final footprint" - : "place the same furniture subject on the final footprint"; + ? "resize the same selected subject proportionally larger toward the final footprint" + : "place the same selected subject on the final footprint"; const blendPhrase = isShrink ? "Blend the newly visible floor, rug, wall, window area, shadow, and background around it naturally." : isEnlarge ? "Blend edges, contact shadows, and occlusion with the surrounding room naturally." : "Keep the surrounding room stable."; - return `Region ${region.index}: current furniture box in source photo left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; requested final footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; scale factor width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. Edit direction: ${actionPhrase}. ${blendPhrase}${cropNote}`; + return `Region ${region.index}: current target-subject search box in source photo left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; requested final footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; scale factor width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. Edit direction: ${actionPhrase}. ${blendPhrase}${cropNote}`; } - return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}.${userInstructionLine} The final furniture silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged furniture footprint is not acceptable. ${footprintRule}${cropNote}`; + return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}.${userInstructionLine} The final selected-subject silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged target subject footprint is not acceptable. ${footprintRule}${cropNote}`; }) .join("\n"); const cropLines = cropReferences @@ -2610,13 +3447,13 @@ function buildScaleAdjustCoordinatePrompt( if (options.falSafe) { return [ - "Interior furniture photo edit.", + "Interior selected-subject scale photo edit.", `Return one realistic photo with the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, "Input image 1 is the source photo. Input image 2 is a marked copy of the source photo for location guidance only and should not appear in the final photo.", - "In the guide, the cyan dashed box marks the existing furniture area and the blue solid box marks the requested final footprint.", - "Resize only the furniture subject indicated by each current box. Keep the camera, room layout, walls, windows, floor, rug, sofa, lamp, text, and background stable.", - "Preserve the furniture identity, material, perspective, objects on top, and natural aspect ratio.", - "For a smaller final footprint, show the furniture smaller and blend newly visible floor, rug, wall, window area, shadow, and background around it naturally. For a larger final footprint, scale the same furniture proportionally with natural edges and contact shadows.", + "In the guide, the cyan dashed box marks the existing selected target subject area and the blue solid box marks the requested final footprint.", + "Resize only the selected target subject indicated by each current box. The target may be a person/model/child/body part, furniture, product, or another object. Keep the camera, room layout, walls, windows, floor, rug, sofa, lamp, text, background, and all non-target subjects stable.", + "Preserve the target identity, person anatomy/face/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions.", + "For a smaller final footprint, show the selected subject smaller and blend newly visible floor, rug, wall, window area, shadow, and background around it naturally. For a larger final footprint, scale the same selected subject proportionally with natural edges and contact shadows.", "Return a clean photo without guide marks, boxes, arrows, overlays, patch edges, color mismatch, or full-frame zoom.", cropLines ? `${cropLines}` : "", "", @@ -2629,31 +3466,31 @@ function buildScaleAdjustCoordinatePrompt( ? "输入图片2是一张完整画面的粗略目标布局草稿，不是输出底图。它把 OLD 源框旧主体占位洗淡，表示旧尺寸主体必须从原尺寸位置移除并自然修复背景；同时把目标主体粗略缩放/移动到 TARGET 最终占位附近，这个缩小后的位置和尺寸是最终结果必须匹配的几何目标。请把 input 2 当成最终构图/比例的强约束：如果 input 2 中目标主体比 input 1 小，最终输出也必须明显变小，不能保留 input 1 的原尺寸主体。不要复制 input 2 的淡化痕迹、粗糙拼接、局部透明边、矩形补丁、亮度变化或参考图画质；最终必须以 input 1 的真实照片为唯一画面基底重新完成无缝图片编辑。" : options.markedPhotoGuide ? "输入图片2是一张原图浅化后的完整画面位置参考图，不是输出底图。青色虚线框表示当前主体搜索框（对应下方 OLD），蓝色实线框表示目标最终占位（对应下方 TARGET）。请用输入图片2理解主体在真实室内画面中的位置关系，但最终图不能出现任何框线、色块、标注或参考图的浅化效果。" - : "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current furniture; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size furniture area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics."; + : "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current selected subject; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size subject area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics."; return [ - "Local furniture scale edit. Use input image 1 as the clean original photo and the locked camera frame.", + "Local selected-subject scale edit. Use input image 1 as the clean original photo and the locked camera frame.", "One-shot cost constraint: produce the requested edit in this generation without relying on later retries or alternate candidates; an unchanged result is not acceptable.", - "中文硬约束：只调整 OLD 框中的主要家具主体。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧家具占位必须改为自然背景；如果是放大，同一件家具必须明显变大。禁止整图缩放、整图裁切、整图重绘。", - "一次成功硬约束：不要保守返回原尺寸。目标家具的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；不接受看起来和原图几乎相同的结果。", + "中文硬约束：只调整 OLD 框中的主要目标主体，目标可以是人物、模特、小孩、手臂身体、商品或家具。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧主体占位必须改为自然背景；如果是放大，同一目标主体必须明显变大。禁止整图缩放、整图裁切、整图重绘。", + "一次成功硬约束：不要保守返回原尺寸。目标主体的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；不接受看起来和原图几乎相同的结果。", `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, coordinateGuideLine, cropLines ? `${cropLines}` : "", "", "Task:", - "Find the main movable furniture subject inside each OLD box in input image 1. Resize and, if needed, move that subject only so its final visible footprint matches the TARGET box shown in input image 2 and the numeric TARGET coordinates below.", - "Do a real subject-scale edit, not a full image zoom. The camera, room layout, windows, wall decor, text, floor, rug, and all non-target furniture must remain stable.", - "If an OLD box contains more than one object, choose the dominant furniture/product inside that box, usually the largest or most central movable item. Keep adjacent lamps, plants, side tables, pillows, wall decor, windows, text, and background objects at their original size unless they are clearly part of the same furniture subject.", - "Keep the furniture identity, material, perspective, objects on top, and natural aspect ratio from input image 1. Do not create a second copy or extra parts. Do not preserve the original furniture size just because the scene looks realistic; the user explicitly wants a visible scale change.", - "For beds, sofas, tables, chairs, cabinets, benches, and other large furniture, still perform the scale edit on that furniture. Do not treat large furniture as locked architecture.", - "If the target is smaller, the furniture must visibly become smaller and be mostly contained by the TARGET footprint. Rebuild the OLD area outside the TARGET footprint as newly exposed floor, rug, wall, bedding, shadows, and background. If the target is larger, enlarge the same furniture naturally without stretching it.", - "缩小时请优先让目标家具明显缩小，而不是为了保守而保持原样。不接受原尺寸保留、只轻微变形、或只改变整张图构图。", - "Do not keep the selected furniture at its original footprint. The edited furniture should fit the TARGET rectangle much better than the OLD rectangle.", + "Find the main selected target subject inside each OLD box in input image 1. The target can be a person/model/child/body part, furniture, product, or another movable object. Resize and, if needed, move that subject only so its final visible footprint matches the TARGET box shown in input image 2 and the numeric TARGET coordinates below.", + "Do a real subject-scale edit, not a full image zoom. The camera, room layout, windows, wall decor, text, floor, rug, and all non-target subjects must remain stable.", + "If an OLD box contains more than one object/person, choose the dominant target subject inside that box, usually the largest or most central selected item/person. Keep adjacent lamps, plants, side tables, pillows, wall decor, windows, text, background objects, and non-target people at their original size unless they are clearly part of the same target subject.", + "Keep the target identity, person face/anatomy/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions from input image 1. Do not create a second copy or extra parts. Do not preserve the original target size just because the scene looks realistic; the user explicitly wants a visible scale change.", + "For beds, sofas, tables, chairs, cabinets, benches, people, models, children, hands, and other large visible subjects, still perform the scale edit on that selected subject. Do not treat the selected subject as locked background.", + "If the target is smaller, the selected subject must visibly become smaller and be mostly contained by the TARGET footprint. Rebuild the OLD area outside the TARGET footprint as newly exposed floor, rug, wall, bedding, shadows, or background. If the target is larger, enlarge the same selected subject naturally without stretching it.", + "缩小时请优先让目标主体明显缩小，而不是为了保守而保持原样。不接受原尺寸保留、只轻微变形、或只改变整张图构图。", + "Do not keep the selected subject at its original footprint. The edited subject should fit the TARGET rectangle much better than the OLD rectangle.", "", "Region requirements:", regionLines, "", - "Success check: compare the final furniture footprint with input image 1 and the neutral coordinate map in input image 2. The selected furniture size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", + "Success check: compare the final selected-subject footprint with input image 1 and the neutral coordinate map in input image 2. The selected subject size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", "Do not paste rectangular image areas. Do not output coordinate-map lines, labels, boxes, arrows, scale text, muted wash, overlays, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", "Return only the clean edited photo.", ].join("\n"); @@ -2661,12 +3498,12 @@ function buildScaleAdjustCoordinatePrompt( function buildScaleAdjustCoordinateSystemPrompt() { return [ - "You are an image editing model for local furniture scale adjustment.", + "You are an image editing model for local selected-subject scale adjustment.", "Input image 1 is the only final photo base. Input image 2 is a zoomed neutral coordinate instruction map, not a photo layer.", - "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding unchanged furniture and full-canvas zoom.", - "Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.", - "Prefer a clear visible size change over a conservative unchanged result; unchanged furniture is not acceptable.", - "When shrinking, rebuild the old full-size furniture footprint outside the target footprint as natural background. Large furniture is editable and must not remain locked at original size.", + "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding an unchanged selected target subject and full-canvas zoom.", + "Generate a clean realistic final photo by editing the selected target subject itself and naturally rebuilding only the tiny surrounding background that becomes visible.", + "The target subject can be a person/model/child/body part, furniture, product, or another movable object. Prefer a clear visible size change over a conservative unchanged result; an unchanged target subject is not acceptable.", + "When shrinking, rebuild the old full-size target-subject footprint outside the target footprint as natural background. Large selected subjects are editable and must not remain locked at original size.", "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", "Never output guide artifacts from the edit map: no faded wash, placeholder shadow, overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", @@ -2676,13 +3513,13 @@ function buildScaleAdjustCoordinateSystemPrompt() { function buildScaleAdjustTextOnlyPrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, - cropReferences: ScaleAdjustCropReference[] = [], + cropReferences: ScaleAdjustCropReference[], sourceSize: { width: number; height: number } ) { const cropLines = cropReferences .map( (reference, index) => - `Input image ${index + 2}: clean crop reference for Region ${reference.regionIndex}; use it only to identify the furniture subject and its natural aspect ratio, never paste the crop into the output.` + `Input image ${index + 2}: clean crop reference for Region ${reference.regionIndex}; use it only to identify the selected target subject and its natural aspect ratio/proportions, never paste the crop into the output.` ) .join("\n"); const regionLines = regions @@ -2704,27 +3541,27 @@ function buildScaleAdjustTextOnlyPrompt( const targetBottom = targetPixelRect.top + targetPixelRect.height; const cropNote = cropIndex >= 0 - ? ` Input image ${cropIndex + 2} is the clean crop reference for this furniture identity and silhouette.` + ? ` Input image ${cropIndex + 2} is the clean crop reference for this target-subject identity and silhouette.` : ""; - return `Region ${region.index}: hidden coordinate set A current area x0=${region.pixelRect.left}, y0=${region.pixelRect.top}, x1=${sourceRight}, y1=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; hidden coordinate set B final size/location x0=${targetPixelRect.left}, y0=${targetPixelRect.top}, x1=${targetRight}, y1=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; final visual scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || "make the same furniture match set B"}.${cropNote}`; + return `Region ${region.index}: hidden coordinate set A current area x0=${region.pixelRect.left}, y0=${region.pixelRect.top}, x1=${sourceRight}, y1=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; hidden coordinate set B final size/location x0=${targetPixelRect.left}, y0=${targetPixelRect.top}, x1=${targetRight}, y1=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; final visual scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || "make the same selected target subject match set B"}.${cropNote}`; }) .join("\n"); return [ - "Local furniture scale edit. Input image 1 is the locked source photo and the only final canvas base.", + "Local selected-subject scale edit. Input image 1 is the locked source photo and the only final canvas base.", `Return one clean realistic photo with exactly the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, cropLines ? cropLines : "", "The final photo must contain only natural scene content. Never add artificial graphic overlays, visible coordinate aids, labels, lines, UI elements, or text.", - "Only edit the main movable furniture subject inside each current pixel area. Keep all other furniture, sofa, lamps, wall, windows, rug, floor, decor, lighting, and camera frame stable.", + "Only edit the main selected target subject inside each current pixel area. The target can be a person/model/child/body part, furniture, product, or another object. Keep all other non-target subjects, furniture, sofa, lamps, wall, windows, rug, floor, decor, lighting, and camera frame stable.", "Use the numeric pixel sets below as hidden instructions only; they are not visual content and must not be drawn.", - "When set B is smaller than set A, make the same furniture visibly smaller and naturally rebuild the newly visible floor, rug, wall, shadow, or background around it. Do not leave the old full-size furniture silhouette.", - "Keep the furniture identity, materials, perspective, objects on top, contact shadows, and natural aspect ratio. Do not stretch, squash, duplicate, or paste a rectangular crop.", - "中文硬约束：输入图1是锁定底图，只能调整 A 像素范围里的主要家具主体，让它变成 B 像素范围对应的大小和位置。A/B 坐标只是隐藏指令，最终照片里绝对不能画出任何线条、框、文字、标注或 UI 元素。", + "When set B is smaller than set A, make the same selected target subject visibly smaller and naturally rebuild the newly visible floor, rug, wall, shadow, or background around it. Do not leave the old full-size subject silhouette.", + "Keep the target identity, person anatomy/face/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions. Do not stretch, squash, duplicate, or paste a rectangular crop.", + "中文硬约束：输入图1是锁定底图，只能调整 A 像素范围里的主要目标主体，目标可以是人物、模特、小孩、手臂身体、商品或家具，让它变成 B 像素范围对应的大小和位置。A/B 坐标只是隐藏指令，最终照片里绝对不能画出任何线条、框、文字、标注或 UI 元素。", "", "Region instructions:", regionLines, "", - "Success check before returning: the selected furniture must be clearly closer to set B than to set A, while the rest of the photo remains visually unchanged and clean.", + "Success check before returning: the selected target subject must be clearly closer to set B than to set A, while the rest of the photo remains visually unchanged and clean.", ].filter(Boolean).join("\n"); } @@ -2771,21 +3608,21 @@ function buildScaleAdjustPatchPrompt( const firstInputRole = targetDraftPrimary ? structureLockedDraft ? includeSourceReference - ? "Input image 1 is a rough product-structure-locked target crop. The selected furniture/product was moved and scaled using source-photo pixels, so its final size, position, rods, panels, weave, wheels, material, and perspective are the hard target. Repair rough background holes, edge halos, blur, occlusion, contact shadows, and surface artifacts without redesigning the product. Input image 2 is only the clean background/person reference, never the size reference." - : "Input image 1 is a rough product-structure-locked target crop and the only visual reference. The selected furniture/product was moved and scaled using source-photo pixels, so its final size, position, rods, panels, weave, wheels, material, and perspective are the hard target. Repair rough background holes, edge halos, blur, occlusion, contact shadows, and surface artifacts without redesigning the product." + ? "Input image 1 is a rough subject-structure-locked target crop. The selected target subject was moved and scaled using source-photo pixels, so its final size, position, silhouette, material/skin/clothing, perspective, and visible structure are the hard target. If the subject is a person, keep the same person identity, face, body proportions, pose, hair, hands, and clothing while cleaning rough edges. If the subject is furniture/product, keep rods, panels, weave, wheels, material, and product structure. Repair rough background holes, edge halos, blur, occlusion, contact shadows, and surface artifacts without redesigning the subject. Input image 2 is only the clean background and non-target reference, never the old size reference." + : "Input image 1 is a rough subject-structure-locked target crop and the only visual reference. The selected target subject was moved and scaled using source-photo pixels, so its final size, position, silhouette, material/skin/clothing, perspective, and visible structure are the hard target. If the subject is a person, keep the same person identity, face, body proportions, pose, hair, hands, and clothing while cleaning rough edges. If the subject is furniture/product, keep rods, panels, weave, wheels, material, and product structure. Repair rough background holes, edge halos, blur, occlusion, contact shadows, and surface artifacts without redesigning the subject." : includeSourceReference - ? "Input image 1 is the rough edited target-layout crop. It is the hard layout target: the selected furniture size and center in the final output must match this draft, but the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts must be cleaned." - : "Input image 1 is the rough edited target-layout crop and the only visual reference. It is the hard layout target: the selected furniture size and center in the final output must match this draft. Clean away the rough wash, ghosted old footprint, placeholder shadow, blur, and patch artifacts." + ? "Input image 1 is the rough edited target-layout crop. It is the hard layout target: the selected subject size and center in the final output must match this draft, but the rough wash, ghosted old footprint, pale wipe, placeholder shadow, blur, and patch artifacts are only edit instructions and must be cleaned. Never reproduce the pale wipe or any rectangular haze as photo content." + : "Input image 1 is the rough edited target-layout crop and the only visual reference. It is the hard layout target: the selected subject size and center in the final output must match this draft. Clean away the rough wash, ghosted old footprint, pale wipe, placeholder shadow, blur, and patch artifacts. Never reproduce the pale wipe or any rectangular haze as photo content." : "Input image 1 is the exact crop from the original photo and is the locked crop base."; const secondInputRole = targetDraftPrimary ? includeSourceReference - ? "Input image 2 is the original clean crop. Use it for stable crop framing, background texture, lighting, perspective, furniture identity, and non-target human/person preservation. Do not restore the selected furniture to its old size from this image." + ? "Input image 2 is the original clean crop. Use it for stable crop framing, background texture, lighting, perspective, target identity, and non-target person/object preservation. Do not restore the selected subject to its old size from this image." : "" - : "Input image 2 is a rough target-layout draft of the same crop. It is the hard layout target for the selected furniture size and position. Use it to understand the final footprint and newly visible background area, then remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft."; + : "Input image 2 is a rough target-layout draft of the same crop. It is the hard layout target for the selected subject size and position. Use it to understand the final footprint and newly visible background area, then remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft."; const cropLines = cropReferences .map( (reference, index) => - `Input image ${index + cropReferenceStartIndex}: clean identity crop for Region ${reference.regionIndex}; use it only to recognize the target furniture shape and materials, not its old size.` + `Input image ${index + cropReferenceStartIndex}: clean identity crop for Region ${reference.regionIndex}; use it only to recognize the target subject shape, identity, anatomy/materials, and natural aspect ratio, not its old size.` ) .join("\n"); const regionLines = regions @@ -2811,15 +3648,15 @@ function buildScaleAdjustPatchPrompt( region.rect.height > 0 ? targetRect.height / region.rect.height : 1; const action = widthFactor < 0.92 || heightFactor < 0.92 - ? "make the same furniture visibly smaller" + ? "make the same selected subject visibly smaller" : widthFactor > 1.08 || heightFactor > 1.08 - ? "make the same furniture visibly larger" - : "move the same furniture to the final footprint"; + ? "make the same selected subject visibly larger" + : "move the same selected subject to the final footprint"; const cropNote = cropIndex >= 0 - ? ` Input image ${cropIndex + cropReferenceStartIndex} is the clean identity reference for this exact furniture.` + ? ` Input image ${cropIndex + cropReferenceStartIndex} is the clean identity reference for this exact target subject.` : ""; - return `Region ${region.index}: current search area inside input image ${sourceInputIndex} crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside input image ${draftInputIndex} / output crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}. The final object must be much closer to the final footprint than to the current search area.${cropNote}`; + return `Region ${region.index}: current search area inside input image ${sourceInputIndex} crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside input image ${draftInputIndex} / output crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}. Treat the main selected subject inside the search area as the target; it may be a person, model, child, body part, furniture, product, or other object. The final subject must be much closer to the final footprint than to the current search area.${cropNote}`; }) .join("\n"); const chineseHardConstraint = (() => { @@ -2828,30 +3665,31 @@ function buildScaleAdjustPatchPrompt( } if (structureLockedDraft) { return includeSourceReference - ? "中文硬约束：输入图1已经用原图产品像素把目标家具/产品放到目标尺寸和目标位置，产品结构、杆件、面板、编织纹理、轮子、材质和透视都要按输入图1保留。输入图2只能作为干净背景、人物、光影参考，绝对不能把产品恢复成输入图2里的旧尺寸。不能重新设计或替换产品，但要清理产品表面的草稿黑斑、破洞、脏点、灰块和藤编断裂，让藤编面板连续干净、轮子只出现在底部。" - : "中文硬约束：输入图1已经用原图产品像素把目标家具/产品放到目标尺寸和目标位置，产品结构、杆件、面板、编织纹理、轮子、材质和透视都要按输入图1保留。不能重新设计或替换产品，但要清理产品表面的草稿黑斑、破洞、脏点、灰块和藤编断裂，让藤编面板连续干净、轮子只出现在底部。"; + ? "中文硬约束：输入图1已经用原图像素把目标主体放到目标尺寸和目标位置，目标主体可以是人物、模特、小孩、手臂身体、商品或家具。人物目标必须按输入图1保留同一身份、脸部、身体比例、姿态、发型、手部和服装；商品/家具目标必须保留产品结构、杆件、面板、编织纹理、轮子、材质和透视。输入图2只能作为干净背景、光影和非目标元素参考，绝对不能把目标主体恢复成输入图2里的旧尺寸。不能重新设计或替换目标主体，只清理草稿边缘、黑斑、破洞、脏点、灰块、皮肤/服装/产品表面伪影。" + : "中文硬约束：输入图1已经用原图像素把目标主体放到目标尺寸和目标位置，目标主体可以是人物、模特、小孩、手臂身体、商品或家具。人物目标必须按输入图1保留同一身份、脸部、身体比例、姿态、发型、手部和服装；商品/家具目标必须保留产品结构、杆件、面板、编织纹理、轮子、材质和透视。不能重新设计或替换目标主体，只清理草稿边缘、黑斑、破洞、脏点、灰块、皮肤/服装/产品表面伪影。"; } return includeSourceReference - ? "中文硬约束：输入图1是目标大小/位置草稿，必须按它的目标尺寸和目标位置输出；输入图2只是原始质感、人物和背景参考，绝对不要把家具恢复成输入图2里的旧尺寸。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。" - : "中文硬约束：输入图1是唯一视觉参考和目标大小/位置草稿，必须按它的目标尺寸和目标位置输出。不要找回旧尺寸家具。只输出同一裁切范围的干净局部照片，去掉草稿中的泛白、残影和拼接感。"; + ? "中文硬约束：输入图1是目标大小/位置草稿，必须按它的目标尺寸和目标位置输出；输入图2只是原始质感、非目标人物和背景参考，绝对不要把目标主体恢复成输入图2里的旧尺寸。草稿里的淡色擦除层/雾块只是提示旧位置需要补背景，不是画面内容，最终不能出现矩形雾块、白块、残影或拼接感。" + : "中文硬约束：输入图1是唯一视觉参考和目标大小/位置草稿，必须按它的目标尺寸和目标位置输出。不要找回旧尺寸目标主体。草稿里的淡色擦除层/雾块只是提示旧位置需要补背景，不是画面内容，最终不能出现矩形雾块、白块、残影或拼接感。"; })(); return [ - "Local furniture scale edit inside a cropped workspace.", + "Local selected-subject scale edit inside a cropped workspace.", `Return exactly one clean realistic crop with the same framing and aspect as input image 1. The crop represents full-photo pixels left=${patchReference.workspacePixelRect.left}, top=${patchReference.workspacePixelRect.top}, width=${patchReference.workspacePixelRect.width}, height=${patchReference.workspacePixelRect.height}.`, firstInputRole, secondInputRole, cropLines, - "Only edit the selected movable furniture subject. Keep the crop borders, wall, window frame, floor, rug, sofa, lamps, decor, lighting, camera perspective, and all non-target furniture stable.", - "If people, faces, arms, hands, bodies, hair, or worn clothing appear in the crop, they are not furniture. Preserve the human scale, identity, pose, face, hand shape, and clothing position as much as possible; do not enlarge, shrink, duplicate, or redraw the person as part of the furniture scale edit.", - "When the final footprint is smaller, the selected furniture must clearly shrink; erase the old larger furniture footprint outside the final footprint and replace it with natural background/floor/rug/wall/shadow that matches the original crop.", + "Only edit the selected target subject inside the search area. The target subject can be a person/model/child/body part, furniture, product, or another movable object. Keep the crop borders, wall, window frame, floor, rug, sofa, lamps, decor, lighting, camera perspective, and all non-target subjects stable.", + "If the selected target is a person, scale the whole visible person/body coherently to the final footprint while preserving identity, face, pose, anatomy, hand shape, hair, clothing, and natural human proportions. If a person appears outside the selected target, preserve that non-target person without resizing or redrawing them.", + "When the final footprint is smaller, the selected subject must clearly shrink; erase the old larger subject footprint outside the final footprint and replace it with natural background/floor/rug/wall/shadow that matches the original crop.", + "If input image 1 still shows any faint, transparent, blurred, or ghosted old-size copy of the selected subject outside the final footprint, treat it as an erase marker only. Do not keep it. The only selected target subject that may remain is the smaller clean subject inside the final footprint.", structureLockedDraft - ? "When the final footprint is larger, the target draft already places the same furniture/product at the required larger size. Do not redesign the product geometry; keep the locked rods, panels, weave, wheels, material, and perspective while cleaning background gaps, boundary blending, rough surface artifacts, natural occlusion, and contact shadows." - : "When the final footprint is larger, enlarge the selected furniture itself to match the target draft while keeping nearby people and hands natural. If a hand touches or holds an item on the furniture, keep the hand anatomically plausible and use the original crop as the person-preservation reference.", - "Keep the same furniture identity, material, perspective, object details, contact shadows, and natural aspect ratio. Do not stretch, squash, duplicate, or replace it with a different item.", - "Hard failure condition: if the selected furniture remains near its original old size, the answer is wrong. The output must visibly change the furniture footprint to match the target draft footprint.", + ? "When the final footprint is larger, the target draft already places the same selected subject at the required larger size. Do not redesign the subject geometry/anatomy; keep the locked person identity/anatomy/clothing or product structure/material/perspective while cleaning background gaps, boundary blending, rough surface artifacts, natural occlusion, and contact shadows." + : "When the final footprint is larger, enlarge the selected subject itself to match the target draft while keeping nearby non-target people and hands natural. If the target is a person, preserve anatomy and identity instead of inventing a new person.", + "Keep the same target identity, perspective, visible details, contact shadows, and natural aspect ratio/proportions. Do not stretch, squash, duplicate, or replace it with a different subject.", + "Hard failure condition: if the selected target subject remains near its original old size, the answer is wrong. The output must visibly change the selected subject footprint to match the target draft footprint.", structureLockedDraft - ? "Remove every rough draft artifact from input image 1: gray fill patches, beige wash, flat painted areas, jagged mask holes, broken woven-panel cells, black speckled stains, dark blotches on cane panels, rectangular crop edges, and ghost fragments. Woven panels must look like continuous clean cane weave; wheels belong only under the bottom frame, not as black stains inside the woven panel. Rods, wheels, and hooks must remain the same product structure, with natural material texture." + ? "Remove every rough draft artifact from input image 1: gray fill patches, beige wash, flat painted areas, jagged mask holes, black speckled stains, dark blotches, rectangular crop edges, and ghost fragments. Human skin, face, hands, hair, and clothing must look continuous and natural when the target is a person. Woven panels, wheels, rods, hooks, and product surfaces must remain continuous and clean when the target is furniture/product." : "", "Do not output boxes, lines, labels, coordinate marks, UI marks, text, arrows, overlays, rough layout artifacts, rectangular patch edges, or a full-room image.", chineseHardConstraint, @@ -2859,7 +3697,7 @@ function buildScaleAdjustPatchPrompt( "Region instructions:", regionLines, "", - "Success check before returning: the selected furniture footprint must be clearly closer to the final footprint than the current search area, and the crop must look like one natural photograph.", + "Success check before returning: the selected target subject footprint must be clearly closer to the final footprint than the current search area, and the crop must look like one natural photograph.", ].filter(Boolean).join("\n"); } @@ -2984,6 +3822,18 @@ export async function POST(request: NextRequest) { const isScaleAdjustOperation = operation === SCALE_ADJUST_OPERATION && (hasScaleAdjustTargetRects || scalePercent !== null); + const hasScaleAdjustEnlarge = isScaleAdjustOperation + ? hasScaleAdjustEnlargeTarget(regions, scalePercent) + : false; + const hasScaleAdjustShrink = isScaleAdjustOperation + ? hasScaleAdjustShrinkTarget(regions, scalePercent) + : false; + const preferScaleAdjustMaskCapableModel = + hasScaleAdjustShrink && !hasScaleAdjustEnlarge; + const preferScaleAdjustFalNanoBananaEdit = + hasScaleAdjustShrink && !hasScaleAdjustEnlarge; + const preferScaleAdjustProviderTypes = + hasScaleAdjustShrink && !hasScaleAdjustEnlarge ? ["grsai-gpt-image"] : []; const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); const resolvedAspectRatio = resolveLocalEditAspectRatio( body.aspectRatio, @@ -3001,10 +3851,10 @@ export async function POST(request: NextRequest) { const detailFixedModel = isScaleAdjustOperation ? resolveScaleAdjustPreferredModelId({ configuredModelId: configuredDetailModel, - honorRequestedModelBeforeGpt: hasScaleAdjustEnlargeTarget( - regions, - scalePercent - ), + honorRequestedModelBeforeGpt: hasScaleAdjustEnlarge, + preferFalNanoBananaEdit: preferScaleAdjustFalNanoBananaEdit, + preferMaskCapableGpt: preferScaleAdjustMaskCapableModel, + preferProviderTypes: preferScaleAdjustProviderTypes, requestedModelId: requestedModel || "", models: scaleAdjustModelCandidates, }) @@ -3043,14 +3893,22 @@ export async function POST(request: NextRequest) { !useProviderTextOnlyCoordinatePrompt; const useScaleAdjustMarkedPhotoGuide = useScaleAdjustCoordinatePrompt && !useScaleAdjustVisualLayoutGuide; + const useScaleAdjustFullFrameFalEdit = + isScaleAdjustOperation && + isScaleAdjustFalNanoBananaEditModel(resolvedScaleAdjustModelEntry); const useScaleAdjustStructureLockedPatchWorkflow = - isScaleAdjustOperation && hasScaleAdjustEnlargeTarget(regions, scalePercent); + isScaleAdjustOperation && + !useScaleAdjustFullFrameFalEdit && + hasScaleAdjustStructureTarget(regions, scalePercent) && + (!preferScaleAdjustMaskCapableModel || + !isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry)); const useScaleAdjustEditMask = isScaleAdjustOperation && !useScaleAdjustStructureLockedPatchWorkflow && isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry); const useScaleAdjustPatchWorkflow = isScaleAdjustOperation && + !useScaleAdjustFullFrameFalEdit && (useScaleAdjustStructureLockedPatchWorkflow || !useScaleAdjustEditMask); const sendScaleAdjustAnnotatedReferenceToModel = isScaleAdjustOperation && @@ -3133,7 +3991,9 @@ export async function POST(request: NextRequest) { scalePercent, sourceBuffer, sourceHeight, + sourceImageUrl, sourceWidth, + userId: session.user.id, }) : null; const shouldCreateScaleAdjustCropReferences = @@ -3154,7 +4014,7 @@ export async function POST(request: NextRequest) { : []; const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const includeScaleAdjustPatchSourceReference = - scaleAdjustPatchReference?.structureLockedDraft === true; + Boolean(scaleAdjustPatchReference); const prompt = isScaleAdjustOperation ? scaleAdjustPatchReference ? buildScaleAdjustPatchPrompt( @@ -3198,12 +4058,12 @@ export async function POST(request: NextRequest) { ) : buildLocalEditPrompt(regions); const scaleAdjustCoordinateNegativePrompt = useFalScaleAdjustCoordinatePrompt - ? "guide marks, labels, boxes, arrows, overlays, patch edges, color mismatch, full-frame zoom, duplicate furniture, stretched furniture, distorted furniture" - : "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, old-size target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; + ? "guide marks, labels, boxes, arrows, overlays, patch edges, color mismatch, full-frame zoom, unchanged target subject, duplicate subject, duplicate furniture, duplicate person, stretched furniture, distorted furniture, deformed person, changed face, extra limbs" + : "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target subject, old-size target subject, unchanged target person, old-size target person, unchanged target furniture, old-size target furniture, stretched subject, squashed subject, distorted subject, stretched furniture, squashed furniture, duplicate subject, duplicate person, duplicate furniture, second copy, changed face, deformed person, broken anatomy, extra limbs, extra fingers, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; const scaleAdjustTextOnlyNegativePrompt = - "artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, full image redesign, full-frame zoom, crop, pan, pasted crop, hard patch edge, color mismatch, changed sofa, changed lamp, changed wall, changed rug, duplicate furniture, stretched furniture, distorted furniture"; + "artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, full image redesign, full-frame zoom, crop, pan, pasted crop, hard patch edge, color mismatch, changed sofa, changed lamp, changed wall, changed rug, duplicate subject, duplicate person, duplicate furniture, changed face, deformed person, extra limbs, stretched subject, stretched furniture, distorted furniture"; const scaleAdjustPatchNegativePrompt = - "full room image, changed crop border, artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, guide mark, pasted crop, hard patch edge, color mismatch, gray draft patch, beige wash patch, flat painted area, broken woven panel, jagged mask hole, black speckled stain, dark blotch on cane panel, dirty woven panel, wheel inside woven panel, duplicate furniture, unchanged target furniture, changed locked product, redrawn product interior, different furniture, stretched furniture, distorted furniture"; + "full room image, changed crop border, artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, guide mark, pasted crop, hard patch edge, rectangular haze, white haze block, pale wipe block, color mismatch, gray draft patch, beige wash patch, flat painted area, jagged mask hole, black speckled stain, dark blotch, duplicate subject, duplicate person, duplicate furniture, unchanged target subject, unchanged target person, unchanged target furniture, old-size target subject, changed target identity, changed face, deformed person, broken anatomy, extra limbs, extra fingers, changed locked product, redrawn product interior, different furniture, broken woven panel, dirty woven panel, wheel inside woven panel, stretched subject, stretched furniture, distorted furniture"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -3239,9 +4099,18 @@ export async function POST(request: NextRequest) { const scaleAdjustCompositeRegions = isScaleAdjustOperation ? buildScaleAdjustCompositeRegions(regions, scalePercent) : []; + const scaleAdjustPatchCompositeRegions = isScaleAdjustOperation + ? buildScaleAdjustPatchCompositeRegions(regions, scalePercent) + : []; + const scaleAdjustProtectedChangeRegions = + isScaleAdjustOperation && scaleAdjustPatchReference + ? buildScaleAdjustProtectedChangeRegions(regions, scalePercent) + : []; const scaleAdjustProtectedRegions = isScaleAdjustOperation && scaleAdjustPatchReference - ? [...scaleAdjustCompositeRegions, scaleAdjustPatchReference.workspaceRect] + ? scaleAdjustPatchReference.structureLockedDraft + ? [scaleAdjustPatchReference.workspaceRect] + : scaleAdjustProtectedChangeRegions : scaleAdjustCompositeRegions; const scaleAdjustVerificationConfig = isScaleAdjustOperation ? { @@ -3328,6 +4197,7 @@ export async function POST(request: NextRequest) { scaleAdjustPatchWorkflowApplied: Boolean(scaleAdjustPatchReference), scaleAdjustPatchStructureLockedDraftApplied: scaleAdjustPatchReference?.structureLockedDraft === true, + scaleAdjustFullFrameFalEditApplied: useScaleAdjustFullFrameFalEdit, scaleAdjustPatchReferenceOrder: scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary ? includeScaleAdjustPatchSourceReference @@ -3393,7 +4263,9 @@ export async function POST(request: NextRequest) { ? scaleAdjustPatchReference ? scaleAdjustPatchReference.structureLockedDraft ? "ai-patch-workspace-structure-locked-draft-v85-one-shot" - : "ai-patch-workspace-draft-v70-one-shot" + : "ai-patch-workspace-draft-v75-soft-wipe-erase-ghost-one-shot" + : useScaleAdjustEditMask + ? "mask-edit-gpt-image-2-scale-shrink-v74-one-shot" : useFalScaleAdjustCoordinatePrompt ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" : useScaleAdjustVisualLayoutGuide @@ -3439,9 +4311,18 @@ export async function POST(request: NextRequest) { referenceImages, scaleAdjustPatchComposite: scaleAdjustPatchReference ? { + diffCompareBlurPx: 0.8, + diffHighThreshold: 42, + diffLowThreshold: 14, featherMaxPx: 42, featherMinPx: 10, featherRatio: 0.018, + maskMode: scaleAdjustPatchReference.structureLockedDraft + ? "workspace" + : "diff", + regions: scaleAdjustPatchReference.structureLockedDraft + ? undefined + : scaleAdjustPatchCompositeRegions, sourceImageUrl, workspaceRegion: scaleAdjustPatchReference.workspaceRect, } diff --git a/lib/ai/rmbg/service.ts b/lib/ai/rmbg/service.ts index 97438fe0b..0916f9150 100644 --- a/lib/ai/rmbg/service.ts +++ b/lib/ai/rmbg/service.ts @@ -63,6 +63,7 @@ const AOS_MAX_RESOLUTION = 4096; const AOS_CREATE_TIMEOUT_MS = 120_000; const AOS_POLL_TIMEOUT_MS = 60_000; const AOS_DOWNLOAD_TIMEOUT_MS = 60_000; +const IMAGE_URL_FETCH_TIMEOUT_MS = 60_000; async function createThumbnail(buffer: Buffer, mimeType: string): Promise { try { @@ -499,14 +500,16 @@ function getModelId(provider: RmbgProvider) { export async function fetchImageBufferFromUrl(imageUrl: string): Promise { try { - const imageResponse = await fetch(imageUrl); - if (!imageResponse.ok) { - throw new RmbgServiceError( - `无法获取图片: ${imageResponse.status} ${imageResponse.statusText}`, - 400 - ); - } - return Buffer.from(await imageResponse.arrayBuffer()); + return await withTimeout(async (signal) => { + const imageResponse = await fetch(imageUrl, { signal }); + if (!imageResponse.ok) { + throw new RmbgServiceError( + `无法获取图片: ${imageResponse.status} ${imageResponse.statusText}`, + 400 + ); + } + return Buffer.from(await imageResponse.arrayBuffer()); + }, IMAGE_URL_FETCH_TIMEOUT_MS); } catch (error) { if (error instanceof RmbgServiceError) { throw error; diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 49cf5f0bd..e29df494e 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -56,6 +56,11 @@ export interface CanvasLocalEditCompositeConfig { export interface CanvasScaleAdjustPatchCompositeConfig { sourceImageUrl: string; workspaceRegion: CanvasLocalEditCompositeRegion; + regions?: CanvasLocalEditCompositeRegion[]; + maskMode?: "workspace" | "diff"; + diffLowThreshold?: number; + diffHighThreshold?: number; + diffCompareBlurPx?: number; featherMinPx?: number; featherMaxPx?: number; featherRatio?: number; @@ -153,6 +158,8 @@ class ScaleAdjustOutputRejectedError extends Error { const message = params.reason === "outside-frame-changed" ? "比例调整结果改变了框选区域外的画面，已拒绝该模型结果" + : params.reason === "rectangular-seam-detected" + ? "比例调整结果出现明显矩形拼接硬边，已拒绝该模型结果" : "比例调整结果没有产生有效变化，已拒绝该模型结果"; super(message); this.name = "ScaleAdjustOutputRejectedError"; @@ -182,6 +189,7 @@ const SCALE_ADJUST_MARKER_CLEANUP_MARGIN = 0.035; const SCALE_ADJUST_MARKER_DILATE_RADIUS = 3; const SCALE_ADJUST_MARKER_MIN_PIXEL_COUNT = 18; const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112; +const SCALE_ADJUST_FOOTPRINT_FALLBACK_DIFF_THRESHOLD = 18; const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42; const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18; const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180; @@ -190,6 +198,16 @@ const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO = 0.84; const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO = 0.14; const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60; const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018; +const SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD = 18; +const SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF = 5; +const SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF = 54; +const SCALE_ADJUST_RECTANGULAR_SEAM_EDGE_INCREASE = 2.5; +const SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO = 0.11; +const SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO = 0.18; +const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO = 0.1; +const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO = 0.26; +const SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO = 0.55; +const SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO = 0.5; const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "gemini", @@ -374,10 +392,31 @@ function normalizeScaleAdjustPatchCompositeConfig( if (!/^https?:\/\//i.test(sourceImageUrl)) return null; const workspaceRegion = normalizeCanvasRegion(value.workspaceRegion); if (!workspaceRegion) return null; + const regions = Array.isArray(value.regions) + ? value.regions + .map((region) => normalizeCanvasRegion(region)) + .filter((region): region is CanvasLocalEditCompositeRegion => + Boolean(region) + ) + : []; return { sourceImageUrl, workspaceRegion, + ...(regions.length > 0 ? { regions } : {}), + maskMode: value.maskMode === "diff" ? "diff" : "workspace", + diffHighThreshold: normalizeOptionalNumber(value.diffHighThreshold, { + max: 120, + min: 1, + }), + diffLowThreshold: normalizeOptionalNumber(value.diffLowThreshold, { + max: 80, + min: 0, + }), + diffCompareBlurPx: normalizeOptionalNumber(value.diffCompareBlurPx, { + max: 12, + min: 0, + }), featherMaxPx: normalizeOptionalNumber(value.featherMaxPx, { max: 180, min: 0, @@ -869,6 +908,274 @@ function isScaleAdjustOutsidePixelChanged(params: { ); } +function getScaleAdjustRawRgbDiff(raw: Buffer, leftIndex: number, rightIndex: number) { + return ( + Math.abs(raw[leftIndex] - raw[rightIndex]) * 0.299 + + Math.abs(raw[leftIndex + 1] - raw[rightIndex + 1]) * 0.587 + + Math.abs(raw[leftIndex + 2] - raw[rightIndex + 2]) * 0.114 + ); +} + +function getScaleAdjustPixelChanged(params: { + generatedRaw: Buffer; + index: number; + sourceRaw: Buffer; +}) { + return ( + Math.abs(params.generatedRaw[params.index] - params.sourceRaw[params.index]) * + 0.299 + + Math.abs( + params.generatedRaw[params.index + 1] - + params.sourceRaw[params.index + 1] + ) * + 0.587 + + Math.abs( + params.generatedRaw[params.index + 2] - + params.sourceRaw[params.index + 2] + ) * + 0.114 + ) >= SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD; +} + +function summarizeScaleAdjustSeamLine(params: { + axis: "horizontal" | "vertical"; + boxIndex: number; + changeMask: Uint8Array; + generatedRaw: Buffer; + height: number; + position: number; + rangeEnd: number; + rangeStart: number; + sourceRaw: Buffer; + width: number; +}) { + const { + axis, + boxIndex, + changeMask, + generatedRaw, + height, + position, + rangeEnd, + rangeStart, + sourceRaw, + width, + } = params; + let transitionCount = 0; + let currentRun = 0; + let maxRun = 0; + let edgeDiffSum = 0; + const sampleCount = Math.max(1, rangeEnd - rangeStart); + + for (let offset = rangeStart; offset < rangeEnd; offset += 1) { + const leftX = axis === "vertical" ? position - 1 : offset; + const leftY = axis === "vertical" ? offset : position - 1; + const rightX = axis === "vertical" ? position : offset; + const rightY = axis === "vertical" ? offset : position; + if ( + leftX < 0 || + rightX < 0 || + leftY < 0 || + rightY < 0 || + leftX >= width || + rightX >= width || + leftY >= height || + rightY >= height + ) { + currentRun = 0; + continue; + } + + const leftPixel = leftY * width + leftX; + const rightPixel = rightY * width + rightX; + const changedBoundary = + changeMask[leftPixel] !== changeMask[rightPixel]; + const leftRawIndex = leftPixel * 3; + const rightRawIndex = rightPixel * 3; + const generatedEdgeDiff = getScaleAdjustRawRgbDiff( + generatedRaw, + leftRawIndex, + rightRawIndex + ); + const sourceEdgeDiff = getScaleAdjustRawRgbDiff( + sourceRaw, + leftRawIndex, + rightRawIndex + ); + const likelyBackgroundSeam = + generatedEdgeDiff >= SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF && + generatedEdgeDiff <= SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF && + generatedEdgeDiff - sourceEdgeDiff >= + SCALE_ADJUST_RECTANGULAR_SEAM_EDGE_INCREASE; + + if (changedBoundary && likelyBackgroundSeam) { + transitionCount += 1; + currentRun += 1; + maxRun = Math.max(maxRun, currentRun); + edgeDiffSum += generatedEdgeDiff; + } else { + currentRun = 0; + } + } + + return { + axis, + boxIndex, + edgeDiffMean: transitionCount > 0 ? edgeDiffSum / transitionCount : 0, + maxRun, + maxRunRatio: maxRun / sampleCount, + position, + sampleCount, + transitionCount, + transitionRatio: transitionCount / sampleCount, + }; +} + +async function getScaleAdjustRectangularSeamStats(params: { + generatedBuffer: Buffer; + sourceBuffer: Buffer; + sourceWidth: number; + sourceHeight: number; + verification: CanvasScaleAdjustVerificationConfig; +}) { + const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } = + params; + const normalizeToRaw = (buffer: Buffer) => + sharp(buffer, { failOn: "none" }) + .rotate() + .resize(sourceWidth, sourceHeight, { + fit: "cover", + position: "center", + }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(); + const [sourceRaw, generatedRaw] = await Promise.all([ + normalizeToRaw(sourceBuffer), + normalizeToRaw(generatedBuffer), + ]); + const changeMask = new Uint8Array(sourceWidth * sourceHeight); + for (let pixel = 0; pixel < sourceWidth * sourceHeight; pixel += 1) { + const rawIndex = pixel * 3; + changeMask[pixel] = getScaleAdjustPixelChanged({ + generatedRaw, + index: rawIndex, + sourceRaw, + }) + ? 1 + : 0; + } + + const protectedBoxes = getScaleAdjustProtectedRegions(verification).map((region) => + rectToPixelBox({ + height: sourceHeight, + region, + width: sourceWidth, + }) + ); + const lineStep = Math.max( + 1, + Math.floor(Math.min(sourceWidth, sourceHeight) / 1400) + ); + let bestVertical: ReturnType | null = null; + let bestHorizontal: ReturnType | null = null; + + for (const [boxIndex, box] of protectedBoxes.entries()) { + const left = Math.max(2, box.left + 2); + const top = Math.max(2, box.top + 2); + const right = Math.min(sourceWidth - 2, box.left + box.width - 2); + const bottom = Math.min(sourceHeight - 2, box.top + box.height - 2); + if (right <= left || bottom <= top) continue; + + for (let x = left; x <= right; x += lineStep) { + const stats = summarizeScaleAdjustSeamLine({ + axis: "vertical", + boxIndex, + changeMask, + generatedRaw, + height: sourceHeight, + position: x, + rangeEnd: bottom, + rangeStart: top, + sourceRaw, + width: sourceWidth, + }); + if (!bestVertical || stats.maxRunRatio > bestVertical.maxRunRatio) { + bestVertical = stats; + } + } + + for (let y = top; y <= bottom; y += lineStep) { + const stats = summarizeScaleAdjustSeamLine({ + axis: "horizontal", + boxIndex, + changeMask, + generatedRaw, + height: sourceHeight, + position: y, + rangeEnd: right, + rangeStart: left, + sourceRaw, + width: sourceWidth, + }); + if (!bestHorizontal || stats.maxRunRatio > bestHorizontal.maxRunRatio) { + bestHorizontal = stats; + } + } + } + + const verticalPair = + bestVertical && + bestVertical.maxRunRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO && + bestVertical.transitionRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO; + const horizontalPair = + bestHorizontal && + bestHorizontal.maxRunRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO && + bestHorizontal.transitionRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO; + const verticalStrong = + bestVertical && + bestVertical.maxRunRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO && + bestVertical.transitionRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO; + const horizontalStrong = + bestHorizontal && + bestHorizontal.maxRunRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO && + bestHorizontal.transitionRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO; + const verticalVeryStrong = + bestVertical && + bestVertical.maxRunRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO && + bestVertical.transitionRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO; + const horizontalVeryStrong = + bestHorizontal && + bestHorizontal.maxRunRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO && + bestHorizontal.transitionRatio >= + SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO; + + return { + detected: Boolean( + (verticalPair && horizontalPair) || + (verticalStrong && horizontalStrong) || + verticalVeryStrong || + horizontalVeryStrong + ), + horizontal: bestHorizontal, + lineStep, + protectedBoxCount: protectedBoxes.length, + vertical: bestVertical, + }; +} + async function getScaleAdjustFootprintChangeStats(params: { generatedBuffer: Buffer; sourceBuffer: Buffer; @@ -1232,6 +1539,10 @@ async function applyScaleAdjustVerificationToImages(params: { outsidePixelStats: Awaited< ReturnType >; + rectangularSeamDetected: boolean; + rectangularSeamStats: Awaited< + ReturnType + >; }> = []; for (const [index, image] of images.entries()) { const generatedUrl = image.originalUrl || image.thumbnailUrl; @@ -1243,6 +1554,7 @@ async function applyScaleAdjustVerificationToImages(params: { outsideDiffScores, outsidePixelStats, footprintStats, + rectangularSeamStats, ] = await Promise.all([ getScaleAdjustDiffScores({ generatedBuffer, @@ -1279,18 +1591,35 @@ async function applyScaleAdjustVerificationToImages(params: { sourceWidth: sourceImage.sourceWidth, verification, }), + getScaleAdjustRectangularSeamStats({ + generatedBuffer, + sourceBuffer: sourceImage.sourceBuffer, + sourceHeight: sourceImage.sourceHeight, + sourceWidth: sourceImage.sourceWidth, + verification, + }), ]); const visuallyChangedInside = changeDiffScores.some( (score) => score >= SCALE_ADJUST_VISUAL_DIFF_THRESHOLD ); - const changedInside = visuallyChangedInside && footprintStats.passed; + const footprintFallbackAccepted = + visuallyChangedInside && + !footprintStats.passed && + changeDiffScores.some( + (score) => score >= SCALE_ADJUST_FOOTPRINT_FALLBACK_DIFF_THRESHOLD + ); + const changedInside = + (visuallyChangedInside && footprintStats.passed) || + footprintFallbackAccepted; const outsideChanged = isScaleAdjustOutsideChanged(outsideDiffScores) || isScaleAdjustOutsidePixelChanged(outsidePixelStats); + const rectangularSeamDetected = rectangularSeamStats.detected; console.info("[Canvas Generate Image] Scale adjust result guard:", { debugLabel, changedInside, + footprintFallbackAccepted, footprintChanged: footprintStats.passed, footprintStats: { failedRegionCount: footprintStats.failedRegionCount, @@ -1320,6 +1649,38 @@ async function applyScaleAdjustVerificationToImages(params: { Number(score.toFixed(2)) ), protectedRegionCount: verification.protectedRegions?.length || 0, + rectangularSeamDetected, + rectangularSeamStats: { + horizontal: rectangularSeamStats.horizontal + ? { + edgeDiffMean: Number( + rectangularSeamStats.horizontal.edgeDiffMean.toFixed(2) + ), + maxRunRatio: Number( + rectangularSeamStats.horizontal.maxRunRatio.toFixed(3) + ), + position: rectangularSeamStats.horizontal.position, + transitionRatio: Number( + rectangularSeamStats.horizontal.transitionRatio.toFixed(3) + ), + } + : null, + lineStep: rectangularSeamStats.lineStep, + vertical: rectangularSeamStats.vertical + ? { + edgeDiffMean: Number( + rectangularSeamStats.vertical.edgeDiffMean.toFixed(2) + ), + maxRunRatio: Number( + rectangularSeamStats.vertical.maxRunRatio.toFixed(3) + ), + position: rectangularSeamStats.vertical.position, + transitionRatio: Number( + rectangularSeamStats.vertical.transitionRatio.toFixed(3) + ), + } + : null, + }, regionCount: verification.regions.length, regionDiffScores: diffScores.map((score) => Number(score.toFixed(2))), scalePercent: verification.scalePercent ?? null, @@ -1329,6 +1690,7 @@ async function applyScaleAdjustVerificationToImages(params: { const guardWarnings = [ ...(outsideChanged ? ["outside-frame-changed"] : []), ...(!changedInside ? ["no-effective-scale-change"] : []), + ...(rectangularSeamDetected ? ["rectangular-seam-detected"] : []), ]; if (guardWarnings.length > 0) { console.warn("[Canvas Generate Image] Scale adjust result guard warning; keeping result for preview:", { @@ -1336,6 +1698,7 @@ async function applyScaleAdjustVerificationToImages(params: { imageIndex: index, reasons: guardWarnings, changedInside, + footprintFallbackAccepted, footprintChanged: footprintStats.passed, footprintStats: { failedRegionCount: footprintStats.failedRegionCount, @@ -1360,6 +1723,37 @@ async function applyScaleAdjustVerificationToImages(params: { sampledPixelCount: outsidePixelStats.sampledPixelCount, step: outsidePixelStats.step, }, + rectangularSeamDetected, + rectangularSeamStats: { + horizontal: rectangularSeamStats.horizontal + ? { + edgeDiffMean: Number( + rectangularSeamStats.horizontal.edgeDiffMean.toFixed(2) + ), + maxRunRatio: Number( + rectangularSeamStats.horizontal.maxRunRatio.toFixed(3) + ), + position: rectangularSeamStats.horizontal.position, + transitionRatio: Number( + rectangularSeamStats.horizontal.transitionRatio.toFixed(3) + ), + } + : null, + vertical: rectangularSeamStats.vertical + ? { + edgeDiffMean: Number( + rectangularSeamStats.vertical.edgeDiffMean.toFixed(2) + ), + maxRunRatio: Number( + rectangularSeamStats.vertical.maxRunRatio.toFixed(3) + ), + position: rectangularSeamStats.vertical.position, + transitionRatio: Number( + rectangularSeamStats.vertical.transitionRatio.toFixed(3) + ), + } + : null, + }, changeRegionDiffScores: changeDiffScores.map((score) => Number(score.toFixed(2)) ), @@ -1392,6 +1786,8 @@ async function applyScaleAdjustVerificationToImages(params: { outsideChanged, outsideDiffScores, outsidePixelStats, + rectangularSeamDetected, + rectangularSeamStats, }); continue; } @@ -1414,6 +1810,8 @@ async function applyScaleAdjustVerificationToImages(params: { outsideChanged, outsideDiffScores, outsidePixelStats, + rectangularSeamDetected, + rectangularSeamStats, }); } @@ -1424,11 +1822,20 @@ async function applyScaleAdjustVerificationToImages(params: { const outsideMean = item.outsidePixelStats.meanDiff; const outsidePenalty = item.outsideChanged ? 100 : 0; const noChangePenalty = item.changedInside ? 0 : 60; - return insideScore - outsideMean * 0.35 - outsidePenalty - noChangePenalty; + const seamPenalty = item.rectangularSeamDetected ? 140 : 0; + return ( + insideScore - + outsideMean * 0.35 - + outsidePenalty - + noChangePenalty - + seamPenalty + ); }; const rankedImages = [...evaluatedImages].sort((left, right) => { - const leftPassed = left.changedInside && !left.outsideChanged; - const rightPassed = right.changedInside && !right.outsideChanged; + const leftPassed = + left.changedInside && !left.outsideChanged && !left.rectangularSeamDetected; + const rightPassed = + right.changedInside && !right.outsideChanged && !right.rectangularSeamDetected; if (leftPassed !== rightPassed) return leftPassed ? -1 : 1; return scoreEvaluation(right) - scoreEvaluation(left); }); @@ -1438,9 +1845,13 @@ async function applyScaleAdjustVerificationToImages(params: { debugLabel, candidateCount: evaluatedImages.length, imageIndex: selected.index, - passed: selected.changedInside && !selected.outsideChanged, + passed: + selected.changedInside && + !selected.outsideChanged && + !selected.rectangularSeamDetected, changedInside: selected.changedInside, outsideChanged: selected.outsideChanged, + rectangularSeamDetected: selected.rectangularSeamDetected, changeRegionDiffScores: selected.changeDiffScores.map((score) => Number(score.toFixed(2)) ), @@ -1473,12 +1884,16 @@ async function applyScaleAdjustVerificationToImages(params: { if ( verification.rejectFailedCandidates && - (!selected.changedInside || (selected.outsideChanged && !outsideChangeAccepted)) + (!selected.changedInside || + selected.rectangularSeamDetected || + (selected.outsideChanged && !outsideChangeAccepted)) ) { throw new ScaleAdjustOutputRejectedError({ - reason: selected.outsideChanged - ? "outside-frame-changed" - : "no-effective-scale-change", + reason: selected.rectangularSeamDetected + ? "rectangular-seam-detected" + : selected.outsideChanged + ? "outside-frame-changed" + : "no-effective-scale-change", diffScores: selected.changeDiffScores, images: [selected.cleanedImage || selected.image], outsideDiffScores: selected.outsideDiffScores, @@ -1976,6 +2391,7 @@ function buildScaleAdjustPatchCompositeMaskSvg(params: { width: number; height: number; featherPx: number; + regions?: Array<{ left: number; top: number; width: number; height: number }>; touchesLeft: boolean; touchesTop: boolean; touchesRight: boolean; @@ -1983,6 +2399,46 @@ function buildScaleAdjustPatchCompositeMaskSvg(params: { }) { const { height, width } = params; const featherPx = Math.max(0, Math.round(params.featherPx)); + const regions = params.regions?.filter( + (region) => region.width > 0 && region.height > 0 + ); + if (regions?.length) { + const blur = Math.max(1, featherPx); + const softPad = Math.max(2, Math.round(featherPx * 1.35)); + const corePad = Math.max(0, Math.round(featherPx * 0.22)); + const softRects = regions + .map((region) => { + const left = Math.max(0, region.left - softPad); + const top = Math.max(0, region.top - softPad); + const right = Math.min(width, region.left + region.width + softPad); + const bottom = Math.min(height, region.top + region.height + softPad); + return ``; + }) + .join(""); + const coreRects = regions + .map((region) => { + const left = Math.max(0, region.left - corePad); + const top = Math.max(0, region.top - corePad); + const right = Math.min(width, region.left + region.width + corePad); + const bottom = Math.min(height, region.top + region.height + corePad); + return ``; + }) + .join(""); + + return Buffer.from(` + + + + + + + + ${softRects} + ${coreRects} + + `); + } + if (featherPx <= 0) { return Buffer.from(` @@ -2148,6 +2604,75 @@ async function buildLocalEditDiffCompositeMask(params: { .toBuffer(); } +async function normalizeAlphaMaskToRaw(params: { + maskBuffer: Buffer; + width: number; + height: number; +}) { + const { height, maskBuffer, width } = params; + return sharp(maskBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .extractChannel(0) + .raw() + .toBuffer(); +} + +async function normalizeOpacityMaskToRaw(params: { + maskBuffer: Buffer; + width: number; + height: number; +}) { + const { height, maskBuffer, width } = params; + return sharp(maskBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .raw() + .toBuffer(); +} + +function multiplyAlphaMasks(primaryMask: Buffer, clipMask: Buffer) { + if (primaryMask.length !== clipMask.length) return primaryMask; + const multiplied = Buffer.allocUnsafe(primaryMask.length); + for (let index = 0; index < primaryMask.length; index += 1) { + multiplied[index] = Math.round((primaryMask[index] * clipMask[index]) / 255); + } + return multiplied; +} + +function boostAlphaMask(mask: Buffer, cutoff: number, multiplier: number) { + const boosted = Buffer.allocUnsafe(mask.length); + for (let index = 0; index < mask.length; index += 1) { + const value = mask[index]; + boosted[index] = + value < cutoff ? 0 : Math.min(255, Math.round(value * multiplier)); + } + return boosted; +} + +async function applyRawAlphaToImage(params: { + imageBuffer: Buffer; + alphaMaskRaw: Buffer; + width: number; + height: number; +}) { + const { alphaMaskRaw, height, imageBuffer, width } = params; + const rgbRaw = await sharp(imageBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(); + + return sharp(rgbRaw, { raw: { channels: 3, height, width } }) + .joinChannel(alphaMaskRaw, { + raw: { channels: 1, height, width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + async function applyLocalEditCompositeToImages(params: { images: ParsedImage[]; composite: CanvasLocalEditCompositeConfig | null; @@ -2187,19 +2712,22 @@ async function applyLocalEditCompositeToImages(params: { .toBuffer(); const overlayBuffer = composite.maskMode === "diff" - ? await sharp(resizedGeneratedBuffer, { failOn: "none" }) - .removeAlpha() - .joinChannel( - await buildLocalEditDiffCompositeMask({ + ? await applyRawAlphaToImage({ + alphaMaskRaw: await normalizeAlphaMaskToRaw({ + height: sourceHeight, + maskBuffer: await buildLocalEditDiffCompositeMask({ composite, generatedBuffer: resizedGeneratedBuffer, height: sourceHeight, sourceBuffer, width: sourceWidth, - }) - ) - .png({ compressionLevel: 6 }) - .toBuffer() + }), + width: sourceWidth, + }), + height: sourceHeight, + imageBuffer: resizedGeneratedBuffer, + width: sourceWidth, + }) : await sharp(resizedGeneratedBuffer, { failOn: "none" }) .ensureAlpha() .composite([ @@ -2312,30 +2840,139 @@ async function applyScaleAdjustPatchCompositeToImages(params: { const touchesTop = workspaceBox.top <= 0; const touchesRight = workspaceBox.left + workspaceBox.width >= sourceWidth; const touchesBottom = workspaceBox.top + workspaceBox.height >= sourceHeight; - const patchMask = buildScaleAdjustPatchCompositeMaskSvg({ - featherPx, - height: workspaceBox.height, - touchesBottom, - touchesLeft, - touchesRight, - touchesTop, - width: workspaceBox.width, - }); + const maskRegionBoxes = composite.regions + ?.map((region) => + rectToPixelBox({ + height: sourceHeight, + region, + width: sourceWidth, + }) + ) + .map((box) => { + const left = Math.max(0, box.left - workspaceBox.left); + const top = Math.max(0, box.top - workspaceBox.top); + const right = Math.min( + workspaceBox.width, + box.left + box.width - workspaceBox.left + ); + const bottom = Math.min( + workspaceBox.height, + box.top + box.height - workspaceBox.top + ); + return { + left, + top, + width: Math.max(0, right - left), + height: Math.max(0, bottom - top), + }; + }) + .filter((box) => box.width > 0 && box.height > 0); + const staticPatchMask = + composite.maskMode === "diff" + ? null + : buildScaleAdjustPatchCompositeMaskSvg({ + featherPx, + height: workspaceBox.height, + regions: maskRegionBoxes, + touchesBottom, + touchesLeft, + touchesRight, + touchesTop, + width: workspaceBox.width, + }); + const regionClipPatchMask = maskRegionBoxes?.length + ? buildScaleAdjustPatchCompositeMaskSvg({ + featherPx, + height: workspaceBox.height, + regions: maskRegionBoxes, + touchesBottom, + touchesLeft, + touchesRight, + touchesTop, + width: workspaceBox.width, + }) + : null; + const regionClipPatchMaskRaw = regionClipPatchMask + ? await normalizeOpacityMaskToRaw({ + height: workspaceBox.height, + maskBuffer: regionClipPatchMask, + width: workspaceBox.width, + }) + : null; + const sourcePatchBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .extract(workspaceBox) + .png({ compressionLevel: 6 }) + .toBuffer(); + const diffRegions = (maskRegionBoxes?.length + ? maskRegionBoxes + : [{ left: 0, top: 0, width: workspaceBox.width, height: workspaceBox.height }] + ).map((box) => ({ + x: box.left / Math.max(1, workspaceBox.width), + y: box.top / Math.max(1, workspaceBox.height), + width: box.width / Math.max(1, workspaceBox.width), + height: box.height / Math.max(1, workspaceBox.height), + })); const mergedImages: ParsedImage[] = []; for (const [index, image] of images.entries()) { const generatedUrl = image.originalUrl || image.thumbnailUrl; const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); - const patchOverlayBuffer = await sharp(generatedBuffer, { failOn: "none" }) + const resizedGeneratedBuffer = await sharp(generatedBuffer, { failOn: "none" }) .rotate() .resize(workspaceBox.width, workspaceBox.height, { fit: "cover", position: "center", }) - .ensureAlpha() - .composite([{ input: patchMask, blend: "dest-in" }]) .png({ compressionLevel: 6 }) .toBuffer(); + const diffPatchMask = + composite.maskMode === "diff" + ? await buildLocalEditDiffCompositeMask({ + composite: { + diffCompareBlurPx: composite.diffCompareBlurPx, + diffHighThreshold: composite.diffHighThreshold, + diffLowThreshold: composite.diffLowThreshold, + featherMaxPx: composite.featherMaxPx, + featherMinPx: composite.featherMinPx, + featherRatio: composite.featherRatio, + maskMode: "diff", + regions: diffRegions, + sourceImageUrl: composite.sourceImageUrl, + }, + generatedBuffer: resizedGeneratedBuffer, + height: workspaceBox.height, + sourceBuffer: sourcePatchBuffer, + width: workspaceBox.width, + }) + : null; + const patchMask = diffPatchMask || staticPatchMask; + if (!patchMask) { + throw new Error("比例调整局部结果合成失败：无法生成合成蒙版"); + } + const patchOverlayBuffer = diffPatchMask + ? await applyRawAlphaToImage({ + alphaMaskRaw: boostAlphaMask( + multiplyAlphaMasks( + await normalizeAlphaMaskToRaw({ + height: workspaceBox.height, + maskBuffer: diffPatchMask, + width: workspaceBox.width, + }), + regionClipPatchMaskRaw || + Buffer.alloc(workspaceBox.width * workspaceBox.height, 255) + ), + 8, + 3 + ), + height: workspaceBox.height, + imageBuffer: resizedGeneratedBuffer, + width: workspaceBox.width, + }) + : await sharp(resizedGeneratedBuffer, { failOn: "none" }) + .ensureAlpha() + .composite([{ input: patchMask, blend: "dest-in" }]) + .png({ compressionLevel: 6 }) + .toBuffer(); const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" }) .ensureAlpha() .composite([ @@ -2374,6 +3011,9 @@ async function applyScaleAdjustPatchCompositeToImages(params: { debugLabel, featherPx, imageCount: mergedImages.length, + diffRegionClipApplied: Boolean(regionClipPatchMask), + maskMode: composite.maskMode || "workspace", + maskRegionCount: maskRegionBoxes?.length || 0, sourceHeight, sourceWidth, workspaceBox,