lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: stabilize scale adjust subject resizing

Browse Source 
# Conflicts:
#	app/api/canvas/local-edit/route.ts
#	lib/canvas/generate-image.ts
This commit is contained in:
134 2026-07-04 13:26:58 +08:00
commit 83e911f698
3 changed files with 1676 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1111
app/api/canvas/local-edit/route.ts
View File

File diff suppressed because it is too large Load Diff

19
lib/ai/rmbg/service.ts
View File

@ -63,6 +63,7 @@ const AOS_MAX_RESOLUTION = 4096;
const AOS_CREATE_TIMEOUT_MS = 120_000;
const AOS_POLL_TIMEOUT_MS = 60_000;
const AOS_DOWNLOAD_TIMEOUT_MS = 60_000;
const IMAGE_URL_FETCH_TIMEOUT_MS = 60_000;
async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer> {
try {
@ -499,14 +500,16 @@ function getModelId(provider: RmbgProvider) {
export async function fetchImageBufferFromUrl(imageUrl: string): Promise<Buffer> {
try {
const imageResponse = await fetch(imageUrl);
if (!imageResponse.ok) {
throw new RmbgServiceError(
`无法获取图片: ${imageResponse.status} ${imageResponse.statusText}`,
400
);
}
return Buffer.from(await imageResponse.arrayBuffer());
return await withTimeout(async (signal) => {
const imageResponse = await fetch(imageUrl, { signal });
if (!imageResponse.ok) {
throw new RmbgServiceError(
`无法获取图片: ${imageResponse.status} ${imageResponse.statusText}`,
400
);
}
return Buffer.from(await imageResponse.arrayBuffer());
}, IMAGE_URL_FETCH_TIMEOUT_MS);
} catch (error) {
if (error instanceof RmbgServiceError) {
throw error;

698
lib/canvas/generate-image.ts
View File

@ -56,6 +56,11 @@ export interface CanvasLocalEditCompositeConfig {
export interface CanvasScaleAdjustPatchCompositeConfig {
sourceImageUrl: string;
workspaceRegion: CanvasLocalEditCompositeRegion;
regions?: CanvasLocalEditCompositeRegion[];
maskMode?: "workspace" | "diff";
diffLowThreshold?: number;
diffHighThreshold?: number;
diffCompareBlurPx?: number;
featherMinPx?: number;
featherMaxPx?: number;
featherRatio?: number;
@ -153,6 +158,8 @@ class ScaleAdjustOutputRejectedError extends Error {
const message =
params.reason === "outside-frame-changed"
? "比例调整结果改变了框选区域外的画面，已拒绝该模型结果"
: params.reason === "rectangular-seam-detected"
? "比例调整结果出现明显矩形拼接硬边，已拒绝该模型结果"
: "比例调整结果没有产生有效变化，已拒绝该模型结果";
super(message);
this.name = "ScaleAdjustOutputRejectedError";
@ -182,6 +189,7 @@ const SCALE_ADJUST_MARKER_CLEANUP_MARGIN = 0.035;
const SCALE_ADJUST_MARKER_DILATE_RADIUS = 3;
const SCALE_ADJUST_MARKER_MIN_PIXEL_COUNT = 18;
const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112;
const SCALE_ADJUST_FOOTPRINT_FALLBACK_DIFF_THRESHOLD = 18;
const SCALE_ADJUST_FALLBACK_MASK_DISTANCE_THRESHOLD = 42;
const SCALE_ADJUST_FALLBACK_MASK_LUMA_GAP = 18;
const SCALE_ADJUST_FALLBACK_INPAINT_ITERATIONS = 180;
@ -190,6 +198,16 @@ const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO = 0.84;
const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO = 0.14;
const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60;
const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018;
const SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD = 18;
const SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF = 5;
const SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF = 54;
const SCALE_ADJUST_RECTANGULAR_SEAM_EDGE_INCREASE = 2.5;
const SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO = 0.11;
const SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO = 0.18;
const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO = 0.1;
const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO = 0.26;
const SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO = 0.55;
const SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO = 0.5;
const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
"gemini",
@ -374,10 +392,31 @@ function normalizeScaleAdjustPatchCompositeConfig(
if (!/^https?:\/\//i.test(sourceImageUrl)) return null;
const workspaceRegion = normalizeCanvasRegion(value.workspaceRegion);
if (!workspaceRegion) return null;
const regions = Array.isArray(value.regions)
? value.regions
.map((region) => normalizeCanvasRegion(region))
.filter((region): region is CanvasLocalEditCompositeRegion =>
Boolean(region)
)
: [];
return {
sourceImageUrl,
workspaceRegion,
...(regions.length > 0 ? { regions } : {}),
maskMode: value.maskMode === "diff" ? "diff" : "workspace",
diffHighThreshold: normalizeOptionalNumber(value.diffHighThreshold, {
max: 120,
min: 1,
}),
diffLowThreshold: normalizeOptionalNumber(value.diffLowThreshold, {
max: 80,
min: 0,
}),
diffCompareBlurPx: normalizeOptionalNumber(value.diffCompareBlurPx, {
max: 12,
min: 0,
}),
featherMaxPx: normalizeOptionalNumber(value.featherMaxPx, {
max: 180,
min: 0,
@ -869,6 +908,274 @@ function isScaleAdjustOutsidePixelChanged(params: {
);
}
function getScaleAdjustRawRgbDiff(raw: Buffer, leftIndex: number, rightIndex: number) {
return (
Math.abs(raw[leftIndex] - raw[rightIndex]) * 0.299 +
Math.abs(raw[leftIndex + 1] - raw[rightIndex + 1]) * 0.587 +
Math.abs(raw[leftIndex + 2] - raw[rightIndex + 2]) * 0.114
);
}
function getScaleAdjustPixelChanged(params: {
generatedRaw: Buffer;
index: number;
sourceRaw: Buffer;
}) {
return (
Math.abs(params.generatedRaw[params.index] - params.sourceRaw[params.index]) *
0.299 +
Math.abs(
params.generatedRaw[params.index + 1] -
params.sourceRaw[params.index + 1]
) *
0.587 +
Math.abs(
params.generatedRaw[params.index + 2] -
params.sourceRaw[params.index + 2]
) *
0.114
) >= SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD;
}
function summarizeScaleAdjustSeamLine(params: {
axis: "horizontal" | "vertical";
boxIndex: number;
changeMask: Uint8Array;
generatedRaw: Buffer;
height: number;
position: number;
rangeEnd: number;
rangeStart: number;
sourceRaw: Buffer;
width: number;
}) {
const {
axis,
boxIndex,
changeMask,
generatedRaw,
height,
position,
rangeEnd,
rangeStart,
sourceRaw,
width,
} = params;
let transitionCount = 0;
let currentRun = 0;
let maxRun = 0;
let edgeDiffSum = 0;
const sampleCount = Math.max(1, rangeEnd - rangeStart);
for (let offset = rangeStart; offset < rangeEnd; offset += 1) {
const leftX = axis === "vertical" ? position - 1 : offset;
const leftY = axis === "vertical" ? offset : position - 1;
const rightX = axis === "vertical" ? position : offset;
const rightY = axis === "vertical" ? offset : position;
if (
leftX < 0 ||
rightX < 0 ||
leftY < 0 ||
rightY < 0 ||
leftX >= width ||
rightX >= width ||
leftY >= height ||
rightY >= height
) {
currentRun = 0;
continue;
}
const leftPixel = leftY * width + leftX;
const rightPixel = rightY * width + rightX;
const changedBoundary =
changeMask[leftPixel] !== changeMask[rightPixel];
const leftRawIndex = leftPixel * 3;
const rightRawIndex = rightPixel * 3;
const generatedEdgeDiff = getScaleAdjustRawRgbDiff(
generatedRaw,
leftRawIndex,
rightRawIndex
);
const sourceEdgeDiff = getScaleAdjustRawRgbDiff(
sourceRaw,
leftRawIndex,
rightRawIndex
);
const likelyBackgroundSeam =
generatedEdgeDiff >= SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF &&
generatedEdgeDiff <= SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF &&
generatedEdgeDiff - sourceEdgeDiff >=
SCALE_ADJUST_RECTANGULAR_SEAM_EDGE_INCREASE;
if (changedBoundary && likelyBackgroundSeam) {
transitionCount += 1;
currentRun += 1;
maxRun = Math.max(maxRun, currentRun);
edgeDiffSum += generatedEdgeDiff;
} else {
currentRun = 0;
}
}
return {
axis,
boxIndex,
edgeDiffMean: transitionCount > 0 ? edgeDiffSum / transitionCount : 0,
maxRun,
maxRunRatio: maxRun / sampleCount,
position,
sampleCount,
transitionCount,
transitionRatio: transitionCount / sampleCount,
};
}
async function getScaleAdjustRectangularSeamStats(params: {
generatedBuffer: Buffer;
sourceBuffer: Buffer;
sourceWidth: number;
sourceHeight: number;
verification: CanvasScaleAdjustVerificationConfig;
}) {
const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } =
params;
const normalizeToRaw = (buffer: Buffer) =>
sharp(buffer, { failOn: "none" })
.rotate()
.resize(sourceWidth, sourceHeight, {
fit: "cover",
position: "center",
})
.removeAlpha()
.toColorspace("srgb")
.raw()
.toBuffer();
const [sourceRaw, generatedRaw] = await Promise.all([
normalizeToRaw(sourceBuffer),
normalizeToRaw(generatedBuffer),
]);
const changeMask = new Uint8Array(sourceWidth * sourceHeight);
for (let pixel = 0; pixel < sourceWidth * sourceHeight; pixel += 1) {
const rawIndex = pixel * 3;
changeMask[pixel] = getScaleAdjustPixelChanged({
generatedRaw,
index: rawIndex,
sourceRaw,
})
? 1
: 0;
}
const protectedBoxes = getScaleAdjustProtectedRegions(verification).map((region) =>
rectToPixelBox({
height: sourceHeight,
region,
width: sourceWidth,
})
);
const lineStep = Math.max(
1,
Math.floor(Math.min(sourceWidth, sourceHeight) / 1400)
);
let bestVertical: ReturnType<typeof summarizeScaleAdjustSeamLine> | null = null;
let bestHorizontal: ReturnType<typeof summarizeScaleAdjustSeamLine> | null = null;
for (const [boxIndex, box] of protectedBoxes.entries()) {
const left = Math.max(2, box.left + 2);
const top = Math.max(2, box.top + 2);
const right = Math.min(sourceWidth - 2, box.left + box.width - 2);
const bottom = Math.min(sourceHeight - 2, box.top + box.height - 2);
if (right <= left || bottom <= top) continue;
for (let x = left; x <= right; x += lineStep) {
const stats = summarizeScaleAdjustSeamLine({
axis: "vertical",
boxIndex,
changeMask,
generatedRaw,
height: sourceHeight,
position: x,
rangeEnd: bottom,
rangeStart: top,
sourceRaw,
width: sourceWidth,
});
if (!bestVertical || stats.maxRunRatio > bestVertical.maxRunRatio) {
bestVertical = stats;
}
}
for (let y = top; y <= bottom; y += lineStep) {
const stats = summarizeScaleAdjustSeamLine({
axis: "horizontal",
boxIndex,
changeMask,
generatedRaw,
height: sourceHeight,
position: y,
rangeEnd: right,
rangeStart: left,
sourceRaw,
width: sourceWidth,
});
if (!bestHorizontal || stats.maxRunRatio > bestHorizontal.maxRunRatio) {
bestHorizontal = stats;
}
}
}
const verticalPair =
bestVertical &&
bestVertical.maxRunRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO &&
bestVertical.transitionRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO;
const horizontalPair =
bestHorizontal &&
bestHorizontal.maxRunRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO &&
bestHorizontal.transitionRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO;
const verticalStrong =
bestVertical &&
bestVertical.maxRunRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO &&
bestVertical.transitionRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO;
const horizontalStrong =
bestHorizontal &&
bestHorizontal.maxRunRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO &&
bestHorizontal.transitionRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO;
const verticalVeryStrong =
bestVertical &&
bestVertical.maxRunRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO &&
bestVertical.transitionRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO;
const horizontalVeryStrong =
bestHorizontal &&
bestHorizontal.maxRunRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_RUN_RATIO &&
bestHorizontal.transitionRatio >=
SCALE_ADJUST_RECTANGULAR_SEAM_VERY_STRONG_TRANSITION_RATIO;
return {
detected: Boolean(
(verticalPair && horizontalPair) ||
(verticalStrong && horizontalStrong) ||
verticalVeryStrong ||
horizontalVeryStrong
),
horizontal: bestHorizontal,
lineStep,
protectedBoxCount: protectedBoxes.length,
vertical: bestVertical,
};
}
async function getScaleAdjustFootprintChangeStats(params: {
generatedBuffer: Buffer;
sourceBuffer: Buffer;
@ -1232,6 +1539,10 @@ async function applyScaleAdjustVerificationToImages(params: {
outsidePixelStats: Awaited<
ReturnType<typeof getScaleAdjustOutsidePixelChangeStats>
>;
rectangularSeamDetected: boolean;
rectangularSeamStats: Awaited<
ReturnType<typeof getScaleAdjustRectangularSeamStats>
>;
}> = [];
for (const [index, image] of images.entries()) {
const generatedUrl = image.originalUrl || image.thumbnailUrl;
@ -1243,6 +1554,7 @@ async function applyScaleAdjustVerificationToImages(params: {
outsideDiffScores,
outsidePixelStats,
footprintStats,
rectangularSeamStats,
] = await Promise.all([
getScaleAdjustDiffScores({
generatedBuffer,
@ -1279,18 +1591,35 @@ async function applyScaleAdjustVerificationToImages(params: {
sourceWidth: sourceImage.sourceWidth,
verification,
}),
getScaleAdjustRectangularSeamStats({
generatedBuffer,
sourceBuffer: sourceImage.sourceBuffer,
sourceHeight: sourceImage.sourceHeight,
sourceWidth: sourceImage.sourceWidth,
verification,
}),
]);
const visuallyChangedInside = changeDiffScores.some(
(score) => score >= SCALE_ADJUST_VISUAL_DIFF_THRESHOLD
);
const changedInside = visuallyChangedInside && footprintStats.passed;
const footprintFallbackAccepted =
visuallyChangedInside &&
!footprintStats.passed &&
changeDiffScores.some(
(score) => score >= SCALE_ADJUST_FOOTPRINT_FALLBACK_DIFF_THRESHOLD
);
const changedInside =
(visuallyChangedInside && footprintStats.passed) ||
footprintFallbackAccepted;
const outsideChanged =
isScaleAdjustOutsideChanged(outsideDiffScores) ||
isScaleAdjustOutsidePixelChanged(outsidePixelStats);
const rectangularSeamDetected = rectangularSeamStats.detected;
console.info("[Canvas Generate Image] Scale adjust result guard:", {
debugLabel,
changedInside,
footprintFallbackAccepted,
footprintChanged: footprintStats.passed,
footprintStats: {
failedRegionCount: footprintStats.failedRegionCount,
@ -1320,6 +1649,38 @@ async function applyScaleAdjustVerificationToImages(params: {
Number(score.toFixed(2))
),
protectedRegionCount: verification.protectedRegions?.length || 0,
rectangularSeamDetected,
rectangularSeamStats: {
horizontal: rectangularSeamStats.horizontal
? {
edgeDiffMean: Number(
rectangularSeamStats.horizontal.edgeDiffMean.toFixed(2)
),
maxRunRatio: Number(
rectangularSeamStats.horizontal.maxRunRatio.toFixed(3)
),
position: rectangularSeamStats.horizontal.position,
transitionRatio: Number(
rectangularSeamStats.horizontal.transitionRatio.toFixed(3)
),
}
: null,
lineStep: rectangularSeamStats.lineStep,
vertical: rectangularSeamStats.vertical
? {
edgeDiffMean: Number(
rectangularSeamStats.vertical.edgeDiffMean.toFixed(2)
),
maxRunRatio: Number(
rectangularSeamStats.vertical.maxRunRatio.toFixed(3)
),
position: rectangularSeamStats.vertical.position,
transitionRatio: Number(
rectangularSeamStats.vertical.transitionRatio.toFixed(3)
),
}
: null,
},
regionCount: verification.regions.length,
regionDiffScores: diffScores.map((score) => Number(score.toFixed(2))),
scalePercent: verification.scalePercent ?? null,
@ -1329,6 +1690,7 @@ async function applyScaleAdjustVerificationToImages(params: {
const guardWarnings = [
...(outsideChanged ? ["outside-frame-changed"] : []),
...(!changedInside ? ["no-effective-scale-change"] : []),
...(rectangularSeamDetected ? ["rectangular-seam-detected"] : []),
];
if (guardWarnings.length > 0) {
console.warn("[Canvas Generate Image] Scale adjust result guard warning; keeping result for preview:", {
@ -1336,6 +1698,7 @@ async function applyScaleAdjustVerificationToImages(params: {
imageIndex: index,
reasons: guardWarnings,
changedInside,
footprintFallbackAccepted,
footprintChanged: footprintStats.passed,
footprintStats: {
failedRegionCount: footprintStats.failedRegionCount,
@ -1360,6 +1723,37 @@ async function applyScaleAdjustVerificationToImages(params: {
sampledPixelCount: outsidePixelStats.sampledPixelCount,
step: outsidePixelStats.step,
},
rectangularSeamDetected,
rectangularSeamStats: {
horizontal: rectangularSeamStats.horizontal
? {
edgeDiffMean: Number(
rectangularSeamStats.horizontal.edgeDiffMean.toFixed(2)
),
maxRunRatio: Number(
rectangularSeamStats.horizontal.maxRunRatio.toFixed(3)
),
position: rectangularSeamStats.horizontal.position,
transitionRatio: Number(
rectangularSeamStats.horizontal.transitionRatio.toFixed(3)
),
}
: null,
vertical: rectangularSeamStats.vertical
? {
edgeDiffMean: Number(
rectangularSeamStats.vertical.edgeDiffMean.toFixed(2)
),
maxRunRatio: Number(
rectangularSeamStats.vertical.maxRunRatio.toFixed(3)
),
position: rectangularSeamStats.vertical.position,
transitionRatio: Number(
rectangularSeamStats.vertical.transitionRatio.toFixed(3)
),
}
: null,
},
changeRegionDiffScores: changeDiffScores.map((score) =>
Number(score.toFixed(2))
),
@ -1392,6 +1786,8 @@ async function applyScaleAdjustVerificationToImages(params: {
outsideChanged,
outsideDiffScores,
outsidePixelStats,
rectangularSeamDetected,
rectangularSeamStats,
});
continue;
}
@ -1414,6 +1810,8 @@ async function applyScaleAdjustVerificationToImages(params: {
outsideChanged,
outsideDiffScores,
outsidePixelStats,
rectangularSeamDetected,
rectangularSeamStats,
});
}
@ -1424,11 +1822,20 @@ async function applyScaleAdjustVerificationToImages(params: {
const outsideMean = item.outsidePixelStats.meanDiff;
const outsidePenalty = item.outsideChanged ? 100 : 0;
const noChangePenalty = item.changedInside ? 0 : 60;
return insideScore - outsideMean * 0.35 - outsidePenalty - noChangePenalty;
const seamPenalty = item.rectangularSeamDetected ? 140 : 0;
return (
insideScore -
outsideMean * 0.35 -
outsidePenalty -
noChangePenalty -
seamPenalty
);
};
const rankedImages = [...evaluatedImages].sort((left, right) => {
const leftPassed = left.changedInside && !left.outsideChanged;
const rightPassed = right.changedInside && !right.outsideChanged;
const leftPassed =
left.changedInside && !left.outsideChanged && !left.rectangularSeamDetected;
const rightPassed =
right.changedInside && !right.outsideChanged && !right.rectangularSeamDetected;
if (leftPassed !== rightPassed) return leftPassed ? -1 : 1;
return scoreEvaluation(right) - scoreEvaluation(left);
});
@ -1438,9 +1845,13 @@ async function applyScaleAdjustVerificationToImages(params: {
debugLabel,
candidateCount: evaluatedImages.length,
imageIndex: selected.index,
passed: selected.changedInside && !selected.outsideChanged,
passed:
selected.changedInside &&
!selected.outsideChanged &&
!selected.rectangularSeamDetected,
changedInside: selected.changedInside,
outsideChanged: selected.outsideChanged,
rectangularSeamDetected: selected.rectangularSeamDetected,
changeRegionDiffScores: selected.changeDiffScores.map((score) =>
Number(score.toFixed(2))
),
@ -1473,12 +1884,16 @@ async function applyScaleAdjustVerificationToImages(params: {
if (
verification.rejectFailedCandidates &&
(!selected.changedInside || (selected.outsideChanged && !outsideChangeAccepted))
(!selected.changedInside ||
selected.rectangularSeamDetected ||
(selected.outsideChanged && !outsideChangeAccepted))
) {
throw new ScaleAdjustOutputRejectedError({
reason: selected.outsideChanged
? "outside-frame-changed"
: "no-effective-scale-change",
reason: selected.rectangularSeamDetected
? "rectangular-seam-detected"
: selected.outsideChanged
? "outside-frame-changed"
: "no-effective-scale-change",
diffScores: selected.changeDiffScores,
images: [selected.cleanedImage || selected.image],
outsideDiffScores: selected.outsideDiffScores,
@ -1976,6 +2391,7 @@ function buildScaleAdjustPatchCompositeMaskSvg(params: {
width: number;
height: number;
featherPx: number;
regions?: Array<{ left: number; top: number; width: number; height: number }>;
touchesLeft: boolean;
touchesTop: boolean;
touchesRight: boolean;
@ -1983,6 +2399,46 @@ function buildScaleAdjustPatchCompositeMaskSvg(params: {
}) {
const { height, width } = params;
const featherPx = Math.max(0, Math.round(params.featherPx));
const regions = params.regions?.filter(
(region) => region.width > 0 && region.height > 0
);
if (regions?.length) {
const blur = Math.max(1, featherPx);
const softPad = Math.max(2, Math.round(featherPx * 1.35));
const corePad = Math.max(0, Math.round(featherPx * 0.22));
const softRects = regions
.map((region) => {
const left = Math.max(0, region.left - softPad);
const top = Math.max(0, region.top - softPad);
const right = Math.min(width, region.left + region.width + softPad);
const bottom = Math.min(height, region.top + region.height + softPad);
return `<rect x="${left}" y="${top}" width="${Math.max(1, right - left)}" height="${Math.max(1, bottom - top)}" fill="#ffffff" filter="url(#scale-adjust-patch-soft-mask)" />`;
})
.join("");
const coreRects = regions
.map((region) => {
const left = Math.max(0, region.left - corePad);
const top = Math.max(0, region.top - corePad);
const right = Math.min(width, region.left + region.width + corePad);
const bottom = Math.min(height, region.top + region.height + corePad);
return `<rect x="${left}" y="${top}" width="${Math.max(1, right - left)}" height="${Math.max(1, bottom - top)}" fill="#ffffff" />`;
})
.join("");
return Buffer.from(`
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
<defs>
<filter id="scale-adjust-patch-soft-mask" x="-50%" y="-50%" width="200%" height="200%" color-interpolation-filters="sRGB">
<feGaussianBlur stdDeviation="${blur}" />
</filter>
</defs>
<rect x="0" y="0" width="${width}" height="${height}" fill="rgba(0,0,0,0)" />
${softRects}
${coreRects}
</svg>
`);
}
if (featherPx <= 0) {
return Buffer.from(`
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
@ -2148,6 +2604,75 @@ async function buildLocalEditDiffCompositeMask(params: {
.toBuffer();
}
async function normalizeAlphaMaskToRaw(params: {
maskBuffer: Buffer;
width: number;
height: number;
}) {
const { height, maskBuffer, width } = params;
return sharp(maskBuffer, { failOn: "none" })
.resize(width, height, { fit: "fill" })
.removeAlpha()
.extractChannel(0)
.raw()
.toBuffer();
}
async function normalizeOpacityMaskToRaw(params: {
maskBuffer: Buffer;
width: number;
height: number;
}) {
const { height, maskBuffer, width } = params;
return sharp(maskBuffer, { failOn: "none" })
.resize(width, height, { fit: "fill" })
.ensureAlpha()
.extractChannel(3)
.raw()
.toBuffer();
}
function multiplyAlphaMasks(primaryMask: Buffer, clipMask: Buffer) {
if (primaryMask.length !== clipMask.length) return primaryMask;
const multiplied = Buffer.allocUnsafe(primaryMask.length);
for (let index = 0; index < primaryMask.length; index += 1) {
multiplied[index] = Math.round((primaryMask[index] * clipMask[index]) / 255);
}
return multiplied;
}
function boostAlphaMask(mask: Buffer, cutoff: number, multiplier: number) {
const boosted = Buffer.allocUnsafe(mask.length);
for (let index = 0; index < mask.length; index += 1) {
const value = mask[index];
boosted[index] =
value < cutoff ? 0 : Math.min(255, Math.round(value * multiplier));
}
return boosted;
}
async function applyRawAlphaToImage(params: {
imageBuffer: Buffer;
alphaMaskRaw: Buffer;
width: number;
height: number;
}) {
const { alphaMaskRaw, height, imageBuffer, width } = params;
const rgbRaw = await sharp(imageBuffer, { failOn: "none" })
.resize(width, height, { fit: "fill" })
.removeAlpha()
.toColorspace("srgb")
.raw()
.toBuffer();
return sharp(rgbRaw, { raw: { channels: 3, height, width } })
.joinChannel(alphaMaskRaw, {
raw: { channels: 1, height, width },
})
.png({ compressionLevel: 6 })
.toBuffer();
}
async function applyLocalEditCompositeToImages(params: {
images: ParsedImage[];
composite: CanvasLocalEditCompositeConfig | null;
@ -2187,19 +2712,22 @@ async function applyLocalEditCompositeToImages(params: {
.toBuffer();
const overlayBuffer =
composite.maskMode === "diff"
? await sharp(resizedGeneratedBuffer, { failOn: "none" })
.removeAlpha()
.joinChannel(
await buildLocalEditDiffCompositeMask({
? await applyRawAlphaToImage({
alphaMaskRaw: await normalizeAlphaMaskToRaw({
height: sourceHeight,
maskBuffer: await buildLocalEditDiffCompositeMask({
composite,
generatedBuffer: resizedGeneratedBuffer,
height: sourceHeight,
sourceBuffer,
width: sourceWidth,
})
)
.png({ compressionLevel: 6 })
.toBuffer()
}),
width: sourceWidth,
}),
height: sourceHeight,
imageBuffer: resizedGeneratedBuffer,
width: sourceWidth,
})
: await sharp(resizedGeneratedBuffer, { failOn: "none" })
.ensureAlpha()
.composite([
@ -2312,30 +2840,139 @@ async function applyScaleAdjustPatchCompositeToImages(params: {
const touchesTop = workspaceBox.top <= 0;
const touchesRight = workspaceBox.left + workspaceBox.width >= sourceWidth;
const touchesBottom = workspaceBox.top + workspaceBox.height >= sourceHeight;
const patchMask = buildScaleAdjustPatchCompositeMaskSvg({
featherPx,
height: workspaceBox.height,
touchesBottom,
touchesLeft,
touchesRight,
touchesTop,
width: workspaceBox.width,
});
const maskRegionBoxes = composite.regions
?.map((region) =>
rectToPixelBox({
height: sourceHeight,
region,
width: sourceWidth,
})
)
.map((box) => {
const left = Math.max(0, box.left - workspaceBox.left);
const top = Math.max(0, box.top - workspaceBox.top);
const right = Math.min(
workspaceBox.width,
box.left + box.width - workspaceBox.left
);
const bottom = Math.min(
workspaceBox.height,
box.top + box.height - workspaceBox.top
);
return {
left,
top,
width: Math.max(0, right - left),
height: Math.max(0, bottom - top),
};
})
.filter((box) => box.width > 0 && box.height > 0);
const staticPatchMask =
composite.maskMode === "diff"
? null
: buildScaleAdjustPatchCompositeMaskSvg({
featherPx,
height: workspaceBox.height,
regions: maskRegionBoxes,
touchesBottom,
touchesLeft,
touchesRight,
touchesTop,
width: workspaceBox.width,
});
const regionClipPatchMask = maskRegionBoxes?.length
? buildScaleAdjustPatchCompositeMaskSvg({
featherPx,
height: workspaceBox.height,
regions: maskRegionBoxes,
touchesBottom,
touchesLeft,
touchesRight,
touchesTop,
width: workspaceBox.width,
})
: null;
const regionClipPatchMaskRaw = regionClipPatchMask
? await normalizeOpacityMaskToRaw({
height: workspaceBox.height,
maskBuffer: regionClipPatchMask,
width: workspaceBox.width,
})
: null;
const sourcePatchBuffer = await sharp(sourceBuffer, { failOn: "none" })
.extract(workspaceBox)
.png({ compressionLevel: 6 })
.toBuffer();
const diffRegions = (maskRegionBoxes?.length
? maskRegionBoxes
: [{ left: 0, top: 0, width: workspaceBox.width, height: workspaceBox.height }]
).map((box) => ({
x: box.left / Math.max(1, workspaceBox.width),
y: box.top / Math.max(1, workspaceBox.height),
width: box.width / Math.max(1, workspaceBox.width),
height: box.height / Math.max(1, workspaceBox.height),
}));
const mergedImages: ParsedImage[] = [];
for (const [index, image] of images.entries()) {
const generatedUrl = image.originalUrl || image.thumbnailUrl;
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
const patchOverlayBuffer = await sharp(generatedBuffer, { failOn: "none" })
const resizedGeneratedBuffer = await sharp(generatedBuffer, { failOn: "none" })
.rotate()
.resize(workspaceBox.width, workspaceBox.height, {
fit: "cover",
position: "center",
})
.ensureAlpha()
.composite([{ input: patchMask, blend: "dest-in" }])
.png({ compressionLevel: 6 })
.toBuffer();
const diffPatchMask =
composite.maskMode === "diff"
? await buildLocalEditDiffCompositeMask({
composite: {
diffCompareBlurPx: composite.diffCompareBlurPx,
diffHighThreshold: composite.diffHighThreshold,
diffLowThreshold: composite.diffLowThreshold,
featherMaxPx: composite.featherMaxPx,
featherMinPx: composite.featherMinPx,
featherRatio: composite.featherRatio,
maskMode: "diff",
regions: diffRegions,
sourceImageUrl: composite.sourceImageUrl,
},
generatedBuffer: resizedGeneratedBuffer,
height: workspaceBox.height,
sourceBuffer: sourcePatchBuffer,
width: workspaceBox.width,
})
: null;
const patchMask = diffPatchMask || staticPatchMask;
if (!patchMask) {
throw new Error("比例调整局部结果合成失败：无法生成合成蒙版");
}
const patchOverlayBuffer = diffPatchMask
? await applyRawAlphaToImage({
alphaMaskRaw: boostAlphaMask(
multiplyAlphaMasks(
await normalizeAlphaMaskToRaw({
height: workspaceBox.height,
maskBuffer: diffPatchMask,
width: workspaceBox.width,
}),
regionClipPatchMaskRaw ||
Buffer.alloc(workspaceBox.width * workspaceBox.height, 255)
),
8,
3
),
height: workspaceBox.height,
imageBuffer: resizedGeneratedBuffer,
width: workspaceBox.width,
})
: await sharp(resizedGeneratedBuffer, { failOn: "none" })
.ensureAlpha()
.composite([{ input: patchMask, blend: "dest-in" }])
.png({ compressionLevel: 6 })
.toBuffer();
const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" })
.ensureAlpha()
.composite([
@ -2374,6 +3011,9 @@ async function applyScaleAdjustPatchCompositeToImages(params: {
debugLabel,
featherPx,
imageCount: mergedImages.length,
diffRegionClipApplied: Boolean(regionClipPatchMask),
maskMode: composite.maskMode || "workspace",
maskRegionCount: maskRegionBoxes?.length || 0,
sourceHeight,
sourceWidth,
workspaceBox,