lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: reject scale adjust old footprint remnants

Browse Source
This commit is contained in:
134 2026-07-08 20:33:10 +08:00
parent 7ac4a25a5f
commit ff71e3935c
2 changed files with 293 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
app/api/canvas/local-edit/route.ts
View File

@ -126,6 +126,7 @@ type ScaleAdjustCropReference = {
type ScaleAdjustPatchReference = {
backgroundPlateCropUrl?: string | null;
backgroundPlateFullUrl?: string | null;
compositeClipMaskCropUrl?: string | null;
compositeClipMaskUrl?: string | null;
repairBlendMaskUrl?: string | null;
productLockMethod?: string | null;
@ -4066,6 +4067,12 @@ async function createScaleAdjustPatchReference(params: {
.png({ compressionLevel: 6 })
.toBuffer()
: null;
const compositeClipMaskCropBuffer = targetDraftReference.compositeClipMaskBuffer
? await sharp(targetDraftReference.compositeClipMaskBuffer, { failOn: "none" })
.extract(workspacePixelRect)
.png({ compressionLevel: 6 })
.toBuffer()
: null;
const [sourceCropUpload, targetDraftUpload, targetDraftFullUpload] =
await Promise.all([
uploadScaleAdjustPrepAsset(
@ -4087,6 +4094,7 @@ async function createScaleAdjustPatchReference(params: {
productLockOverlayUpload,
productLockProtectMaskUpload,
compositeClipMaskUpload,
compositeClipMaskCropUpload,
repairBlendMaskUpload,
productLockEditMaskUpload,
productLockFalEditMaskUpload,
@ -4111,6 +4119,10 @@ async function createScaleAdjustPatchReference(params: {
`canvas-scale-adjust-composite-clip-mask-${suffix}.png`,
targetDraftReference.compositeClipMaskBuffer
),
uploadOptionalScaleAdjustAsset(
`canvas-scale-adjust-composite-clip-mask-crop-${suffix}.png`,
compositeClipMaskCropBuffer
),
uploadOptionalScaleAdjustAsset(
`canvas-scale-adjust-repair-blend-mask-${suffix}.png`,
targetDraftReference.repairBlendMaskBuffer
@ -4134,6 +4146,7 @@ async function createScaleAdjustPatchReference(params: {
return {
backgroundPlateCropUrl: backgroundPlateCropUpload?.url || null,
backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null,
compositeClipMaskCropUrl: compositeClipMaskCropUpload?.url || null,
compositeClipMaskUrl: compositeClipMaskUpload?.url || null,
repairBlendMaskUrl: repairBlendMaskUpload?.url || null,
productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null,
@ -5201,7 +5214,9 @@ function buildScaleAdjustProductLockBackgroundCropPrompt(
return [
"Cropped background repair for product-locked scale adjustment.",
`Return exactly one clean realistic crop with the same framing, aspect ratio, and pixel extent as input image 1. This crop maps to full-photo pixels left=${workspace.left}, top=${workspace.top}, width=${workspace.width}, height=${workspace.height}.`,
"Input image 1 is the original crop from the source photo. Input image 2 is a rough empty-background repair guide crop where the selected furniture/product footprint has been wiped.",
patchReference.compositeClipMaskCropUrl
? "Input image 1 is the original crop from the source photo. Input image 2 is a black/white alpha-mask guide only: white/bright pixels mark the old selected product footprint and target repair boundary to erase or keep empty; black pixels are context to preserve. Do not render the mask itself."
: "Input image 1 is the original crop from the source photo. Input image 2 is a rough empty-background repair guide crop where the selected furniture/product footprint has been wiped.",
"Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in your output. The system will composite the locked original product pixels into the target footprint after your crop is returned.",
"Your only job is to repair the old footprint into natural wall, curtain, floor, rug, shadow, sofa edge, and background, while keeping non-target people and non-target objects stable.",
"If input image 2 contains haze, gray wash, ghost silhouettes, crop edges, black holes, rough mask shapes, or pasted fragments, treat them as repair hints only. Do not preserve them as visible content.",
@ -5578,7 +5593,10 @@ export async function POST(request: NextRequest) {
const useScaleAdjustProductLockCropBackground =
useScaleAdjustProductLockComposite &&
!useScaleAdjustFullFrameStructurePatch &&
Boolean(scaleAdjustPatchReference?.backgroundPlateCropUrl);
Boolean(
scaleAdjustPatchReference?.compositeClipMaskCropUrl ||
scaleAdjustPatchReference?.backgroundPlateCropUrl
);
const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference);
const includeScaleAdjustPatchSourceReference =
Boolean(scaleAdjustPatchReference);
@ -5696,7 +5714,8 @@ export async function POST(request: NextRequest) {
: useScaleAdjustProductLockCropBackground
? [
scaleAdjustPatchReference.sourceCropUrl,
scaleAdjustPatchReference.backgroundPlateCropUrl as string,
(scaleAdjustPatchReference.compositeClipMaskCropUrl ||
scaleAdjustPatchReference.backgroundPlateCropUrl) as string,
...scaleAdjustCropReferenceImageUrls,
...manualRegionReferenceImageUrls,
]
@ -5762,6 +5781,12 @@ export async function POST(request: NextRequest) {
),
}
: {}),
...(scaleAdjustPatchReference?.compositeClipMaskUrl
? {
oldFootprintMaskImageUrl:
scaleAdjustPatchReference.compositeClipMaskUrl,
}
: {}),
}
: undefined;
const requestedImageSize =
@ -5877,6 +5902,8 @@ export async function POST(request: NextRequest) {
scaleAdjustPatchReference?.productLockProtectMaskUrl || null,
scaleAdjustPatchCompositeClipMaskUrl:
scaleAdjustPatchReference?.compositeClipMaskUrl || null,
scaleAdjustPatchCompositeClipMaskCropUrl:
scaleAdjustPatchReference?.compositeClipMaskCropUrl || null,
scaleAdjustRepairBlendMaskUrl:
scaleAdjustPatchReference?.repairBlendMaskUrl || null,
scaleAdjustProviderMaskImageUrl: useScaleAdjustProductLockMaskEdit

267
lib/canvas/generate-image.ts
View File

@ -83,6 +83,7 @@ export interface CanvasScaleAdjustProductLockCompositeConfig {
export interface CanvasScaleAdjustVerificationConfig {
allowDeterministicFallback?: boolean;
allowOutsideChangedWhenChangedInside?: boolean;
oldFootprintMaskImageUrl?: string;
protectedRegions?: CanvasLocalEditCompositeRegion[];
rejectFailedCandidates?: boolean;
regionPrompts?: string[];
@ -177,6 +178,8 @@ class ScaleAdjustOutputRejectedError extends Error {
? "比例调整结果改变了框选区域外的画面，已拒绝该模型结果"
: params.reason === "rectangular-seam-detected"
? "比例调整结果出现明显矩形拼接硬边，已拒绝该模型结果"
: params.reason === "old-footprint-remnant-detected"
? "比例调整结果保留了旧尺寸残影，已拒绝该模型结果"
: "比例调整结果没有产生有效变化，已拒绝该模型结果";
super(message);
this.name = "ScaleAdjustOutputRejectedError";
@ -215,6 +218,10 @@ const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO = 0.84;
const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO = 0.14;
const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60;
const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018;
const SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_DIFF_THRESHOLD = 56;
const SCALE_ADJUST_OLD_FOOTPRINT_MIN_PIXELS = 80;
const SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_RATIO_THRESHOLD = 0.18;
const SCALE_ADJUST_OLD_FOOTPRINT_DARK_RATIO_THRESHOLD = 0.22;
const SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD = 18;
const SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF = 5;
const SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF = 54;
@ -612,6 +619,10 @@ function normalizeScaleAdjustVerificationConfig(
if (!value || typeof value !== "object") return null;
const sourceImageUrl = String(value.sourceImageUrl || "").trim();
if (!/^https?:\/\//i.test(sourceImageUrl)) return null;
const oldFootprintMaskImageUrl = String(
(value as { oldFootprintMaskImageUrl?: unknown }).oldFootprintMaskImageUrl ||
""
).trim();
if (!Array.isArray(value.regions) || value.regions.length === 0) return null;
const rawScalePercent = Number(value.scalePercent);
const scalePercent =
@ -651,6 +662,9 @@ function normalizeScaleAdjustVerificationConfig(
allowOutsideChangedWhenChangedInside:
value.allowOutsideChangedWhenChangedInside === true,
rejectFailedCandidates: value.rejectFailedCandidates === true,
...(/^https?:\/\//i.test(oldFootprintMaskImageUrl)
? { oldFootprintMaskImageUrl }
: {}),
...(regionPrompts.length > 0 ? { regionPrompts } : {}),
...(targetRegions.length > 0 ? { targetRegions } : {}),
...(typeof scalePercent === "number" ? { scalePercent } : {}),
@ -1466,6 +1480,196 @@ async function getScaleAdjustFootprintChangeStats(params: {
};
}
async function loadScaleAdjustOldFootprintMask(params: {
height: number;
url?: string;
width: number;
}) {
const { height, url, width } = params;
if (!url) return null;
try {
return await sharp(await fetchImageBufferFromUrl(url), { failOn: "none" })
.rotate()
.resize(width, height, { fit: "fill" })
.ensureAlpha()
.extractChannel(3)
.greyscale()
.raw()
.toBuffer();
} catch (error) {
console.warn("[Canvas Generate Image] Failed to load scale adjust old footprint mask:", {
error: error instanceof Error ? error.message : String(error),
url,
});
return null;
}
}
async function getScaleAdjustOldFootprintGhostStats(params: {
generatedBuffer: Buffer;
sourceBuffer: Buffer;
sourceWidth: number;
sourceHeight: number;
verification: CanvasScaleAdjustVerificationConfig;
}) {
const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } =
params;
const normalizeToRaw = (buffer: Buffer) =>
sharp(buffer, { failOn: "none" })
.rotate()
.resize(sourceWidth, sourceHeight, {
fit: "cover",
position: "center",
})
.removeAlpha()
.toColorspace("srgb")
.raw()
.toBuffer();
const [sourceRaw, generatedRaw, explicitMaskRaw] = await Promise.all([
normalizeToRaw(sourceBuffer),
normalizeToRaw(generatedBuffer),
loadScaleAdjustOldFootprintMask({
height: sourceHeight,
url: verification.oldFootprintMaskImageUrl,
width: sourceWidth,
}),
]);
const shrinkStats: Array<{
detected: boolean;
darkResidualPixels: number;
darkResidualRatio: number;
maskSource: "explicit" | "fallback";
regionIndex: number;
similarPixels: number;
similarRatio: number;
sourceOutsidePixels: number;
}> = [];
for (const [index, region] of verification.regions.entries()) {
const targetRegion =
verification.targetRegions?.[index] ||
(typeof verification.scalePercent === "number"
? scaleRegionAroundCenter(region, verification.scalePercent)
: null);
if (!targetRegion) continue;
const widthFactor = region.width > 0 ? targetRegion.width / region.width : 1;
const heightFactor = region.height > 0 ? targetRegion.height / region.height : 1;
const isShrink = widthFactor < 0.92 || heightFactor < 0.92;
if (!isShrink) continue;
const sourceBox = rectToPixelBox({
height: sourceHeight,
region,
width: sourceWidth,
});
const targetBox = rectToPixelBox({
height: sourceHeight,
region: expandCanvasRegion(
targetRegion,
SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN * 1.35
),
width: sourceWidth,
});
const fallbackMask = buildScaleAdjustFallbackSubjectMask({
box: sourceBox,
raw: sourceRaw,
width: sourceWidth,
});
const background = estimateScaleAdjustFallbackBackground({
box: sourceBox,
raw: sourceRaw,
width: sourceWidth,
});
let sourceOutsidePixels = 0;
let similarPixels = 0;
let darkResidualPixels = 0;
let explicitPixels = 0;
for (let localY = 0; localY < sourceBox.height; localY += 1) {
const y = sourceBox.top + localY;
for (let localX = 0; localX < sourceBox.width; localX += 1) {
const x = sourceBox.left + localX;
const insideTarget =
x >= targetBox.left &&
x < targetBox.left + targetBox.width &&
y >= targetBox.top &&
y < targetBox.top + targetBox.height;
if (insideTarget) continue;
const localIndex = localY * sourceBox.width + localX;
const absoluteIndex = y * sourceWidth + x;
const explicitMaskValue = explicitMaskRaw?.[absoluteIndex] || 0;
const maskHit = explicitMaskValue > 18 || fallbackMask[localIndex] > 0;
if (!maskHit) continue;
if (explicitMaskValue > 18) explicitPixels += 1;
const rawIndex = getRawPixelIndex(sourceWidth, x, y);
const sourceR = sourceRaw[rawIndex] || 0;
const sourceG = sourceRaw[rawIndex + 1] || 0;
const sourceB = sourceRaw[rawIndex + 2] || 0;
const generatedR = generatedRaw[rawIndex] || 0;
const generatedG = generatedRaw[rawIndex + 1] || 0;
const generatedB = generatedRaw[rawIndex + 2] || 0;
const sourceLuma = getLuma(sourceR, sourceG, sourceB);
const generatedLuma = getLuma(generatedR, generatedG, generatedB);
const sourceBackgroundDistance = Math.sqrt(
(sourceR - background.r) ** 2 +
(sourceG - background.g) ** 2 +
(sourceB - background.b) ** 2
);
if (sourceBackgroundDistance < 24 && explicitMaskValue <= 18) continue;
sourceOutsidePixels += 1;
const sourceGeneratedDiff = Math.sqrt(
(sourceR - generatedR) ** 2 +
(sourceG - generatedG) ** 2 +
(sourceB - generatedB) ** 2
);
if (
sourceGeneratedDiff <=
SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_DIFF_THRESHOLD
) {
similarPixels += 1;
}
if (
generatedLuma <= background.luma - 32 &&
sourceLuma <= background.luma - 18
) {
darkResidualPixels += 1;
}
}
}
const similarRatio =
sourceOutsidePixels > 0 ? similarPixels / sourceOutsidePixels : 0;
const darkResidualRatio =
sourceOutsidePixels > 0 ? darkResidualPixels / sourceOutsidePixels : 0;
const enoughPixels =
sourceOutsidePixels >= SCALE_ADJUST_OLD_FOOTPRINT_MIN_PIXELS;
const detected =
enoughPixels &&
(similarRatio >= SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_RATIO_THRESHOLD ||
darkResidualRatio >= SCALE_ADJUST_OLD_FOOTPRINT_DARK_RATIO_THRESHOLD ||
(similarRatio >= 0.12 && darkResidualRatio >= 0.14));
shrinkStats.push({
detected,
darkResidualPixels,
darkResidualRatio,
maskSource: explicitPixels > 0 ? "explicit" : "fallback",
regionIndex: index,
similarPixels,
similarRatio,
sourceOutsidePixels,
});
}
return {
detected: shrinkStats.some((item) => item.detected),
shrinkRegionCount: shrinkStats.length,
shrinkStats,
};
}
async function loadScaleAdjustSourceImage(
verification: CanvasScaleAdjustVerificationConfig
) {
@ -1690,6 +1894,10 @@ async function applyScaleAdjustVerificationToImages(params: {
cleanedImage?: ParsedImage;
diffScores: number[];
footprintStats: Awaited<ReturnType<typeof getScaleAdjustFootprintChangeStats>>;
oldFootprintGhostDetected: boolean;
oldFootprintGhostStats: Awaited<
ReturnType<typeof getScaleAdjustOldFootprintGhostStats>
>;
image: ParsedImage;
index: number;
outsideChanged: boolean;
@ -1712,6 +1920,7 @@ async function applyScaleAdjustVerificationToImages(params: {
outsideDiffScores,
outsidePixelStats,
footprintStats,
oldFootprintGhostStats,
rectangularSeamStats,
] = await Promise.all([
getScaleAdjustDiffScores({
@ -1749,6 +1958,13 @@ async function applyScaleAdjustVerificationToImages(params: {
sourceWidth: sourceImage.sourceWidth,
verification,
}),
getScaleAdjustOldFootprintGhostStats({
generatedBuffer,
sourceBuffer: sourceImage.sourceBuffer,
sourceHeight: sourceImage.sourceHeight,
sourceWidth: sourceImage.sourceWidth,
verification,
}),
getScaleAdjustRectangularSeamStats({
generatedBuffer,
sourceBuffer: sourceImage.sourceBuffer,
@ -1772,6 +1988,7 @@ async function applyScaleAdjustVerificationToImages(params: {
const outsideChanged =
isScaleAdjustOutsideChanged(outsideDiffScores) ||
isScaleAdjustOutsidePixelChanged(outsidePixelStats);
const oldFootprintGhostDetected = oldFootprintGhostStats.detected;
const rectangularSeamDetected = rectangularSeamStats.detected;
console.info("[Canvas Generate Image] Scale adjust result guard:", {
@ -1795,6 +2012,18 @@ async function applyScaleAdjustVerificationToImages(params: {
},
imageIndex: index,
outsideChanged,
oldFootprintGhostDetected,
oldFootprintGhostStats: {
shrinkRegionCount: oldFootprintGhostStats.shrinkRegionCount,
shrinkStats: oldFootprintGhostStats.shrinkStats.map((item) => ({
darkResidualRatio: Number(item.darkResidualRatio.toFixed(3)),
detected: item.detected,
maskSource: item.maskSource,
regionIndex: item.regionIndex,
similarRatio: Number(item.similarRatio.toFixed(3)),
sourceOutsidePixels: item.sourceOutsidePixels,
})),
},
outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))),
outsidePixelStats: {
changedPixelCount: outsidePixelStats.changedPixelCount,
@ -1849,6 +2078,7 @@ async function applyScaleAdjustVerificationToImages(params: {
...(outsideChanged ? ["outside-frame-changed"] : []),
...(!changedInside ? ["no-effective-scale-change"] : []),
...(rectangularSeamDetected ? ["rectangular-seam-detected"] : []),
...(oldFootprintGhostDetected ? ["old-footprint-remnant-detected"] : []),
];
if (guardWarnings.length > 0) {
console.warn("[Canvas Generate Image] Scale adjust result guard warning; keeping result for preview:", {
@ -1873,6 +2103,18 @@ async function applyScaleAdjustVerificationToImages(params: {
})),
},
outsideChanged,
oldFootprintGhostDetected,
oldFootprintGhostStats: {
shrinkRegionCount: oldFootprintGhostStats.shrinkRegionCount,
shrinkStats: oldFootprintGhostStats.shrinkStats.map((item) => ({
darkResidualRatio: Number(item.darkResidualRatio.toFixed(3)),
detected: item.detected,
maskSource: item.maskSource,
regionIndex: item.regionIndex,
similarRatio: Number(item.similarRatio.toFixed(3)),
sourceOutsidePixels: item.sourceOutsidePixels,
})),
},
outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))),
outsidePixelStats: {
changedPixelCount: outsidePixelStats.changedPixelCount,
@ -1939,6 +2181,8 @@ async function applyScaleAdjustVerificationToImages(params: {
cleanedImage,
diffScores,
footprintStats,
oldFootprintGhostDetected,
oldFootprintGhostStats,
image,
index,
outsideChanged,
@ -1963,6 +2207,8 @@ async function applyScaleAdjustVerificationToImages(params: {
cleanedImage,
diffScores,
footprintStats,
oldFootprintGhostDetected,
oldFootprintGhostStats,
image,
index,
outsideChanged,
@ -1981,19 +2227,27 @@ async function applyScaleAdjustVerificationToImages(params: {
const outsidePenalty = item.outsideChanged ? 100 : 0;
const noChangePenalty = item.changedInside ? 0 : 60;
const seamPenalty = item.rectangularSeamDetected ? 140 : 0;
const ghostPenalty = item.oldFootprintGhostDetected ? 160 : 0;
return (
insideScore -
outsideMean * 0.35 -
outsidePenalty -
noChangePenalty -
seamPenalty
seamPenalty -
ghostPenalty
);
};
const rankedImages = [...evaluatedImages].sort((left, right) => {
const leftPassed =
left.changedInside && !left.outsideChanged && !left.rectangularSeamDetected;
left.changedInside &&
!left.outsideChanged &&
!left.rectangularSeamDetected &&
!left.oldFootprintGhostDetected;
const rightPassed =
right.changedInside && !right.outsideChanged && !right.rectangularSeamDetected;
right.changedInside &&
!right.outsideChanged &&
!right.rectangularSeamDetected &&
!right.oldFootprintGhostDetected;
if (leftPassed !== rightPassed) return leftPassed ? -1 : 1;
return scoreEvaluation(right) - scoreEvaluation(left);
});
@ -2006,8 +2260,10 @@ async function applyScaleAdjustVerificationToImages(params: {
passed:
selected.changedInside &&
!selected.outsideChanged &&
!selected.rectangularSeamDetected,
!selected.rectangularSeamDetected &&
!selected.oldFootprintGhostDetected,
changedInside: selected.changedInside,
oldFootprintGhostDetected: selected.oldFootprintGhostDetected,
outsideChanged: selected.outsideChanged,
rectangularSeamDetected: selected.rectangularSeamDetected,
changeRegionDiffScores: selected.changeDiffScores.map((score) =>
@ -2044,11 +2300,14 @@ async function applyScaleAdjustVerificationToImages(params: {
verification.rejectFailedCandidates &&
(!selected.changedInside ||
selected.rectangularSeamDetected ||
selected.oldFootprintGhostDetected ||
(selected.outsideChanged && !outsideChangeAccepted))
) {
throw new ScaleAdjustOutputRejectedError({
reason: selected.rectangularSeamDetected
? "rectangular-seam-detected"
: selected.oldFootprintGhostDetected
? "old-footprint-remnant-detected"
: selected.outsideChanged
? "outside-frame-changed"
: "no-effective-scale-change",