fix: use cutout drafts for scale adjustment
This commit is contained in:
134
parent
ce4e056f20
commit
ea52d65971
@ -2,7 +2,10 @@ import { NextRequest, NextResponse } from "next/server";
|
||||
import sharp from "sharp";
|
||||
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||
import {
|
||||
executeRmbgJob,
|
||||
fetchImageBufferFromUrl,
|
||||
} from "@/lib/ai/rmbg/service";
|
||||
import {
|
||||
createCanvasImageTask,
|
||||
enqueueCanvasImageTask,
|
||||
@ -106,6 +109,8 @@ const SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN = 0.18;
|
||||
const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb";
|
||||
const SCALE_ADJUST_STRUCTURE_LOCK_MIN_MASK_RATIO = 0.006;
|
||||
const SCALE_ADJUST_STRUCTURE_LOCK_COMPONENT_MIN_RATIO = 0.00018;
|
||||
const SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO = 0.012;
|
||||
const SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO = 0.92;
|
||||
const MIN_REGION_EDGE_PX = 24;
|
||||
const SCALE_ADJUST_OPERATION = "scale-adjust";
|
||||
type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION;
|
||||
@ -128,6 +133,11 @@ type ScaleAdjustPatchReference = {
|
||||
height: number;
|
||||
};
|
||||
};
|
||||
type ScaleAdjustStructureLockedComposite = {
|
||||
method?: "rmbg-cutout" | "pixel-mask";
|
||||
overlay: { input: Buffer; left: number; top: number };
|
||||
wash: { input: Buffer; left: number; top: number };
|
||||
};
|
||||
type AiModelWithProviderForSelection = Awaited<
|
||||
ReturnType<typeof getEnabledAiModelsWithProvider>
|
||||
>[number];
|
||||
@ -1663,20 +1673,310 @@ async function buildScaleAdjustMaskWashPatch(params: {
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
function getScaleAdjustAlphaBoundingBox(params: {
|
||||
height: number;
|
||||
rawRgba: Buffer;
|
||||
threshold?: number;
|
||||
width: number;
|
||||
}) {
|
||||
const { height, rawRgba, threshold = 18, width } = params;
|
||||
let left = width;
|
||||
let top = height;
|
||||
let right = -1;
|
||||
let bottom = -1;
|
||||
let count = 0;
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
for (let x = 0; x < width; x += 1) {
|
||||
const alpha = rawRgba[(y * width + x) * 4 + 3] || 0;
|
||||
if (alpha < threshold) continue;
|
||||
left = Math.min(left, x);
|
||||
top = Math.min(top, y);
|
||||
right = Math.max(right, x);
|
||||
bottom = Math.max(bottom, y);
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (right < left || bottom < top) return null;
|
||||
return {
|
||||
count,
|
||||
left,
|
||||
top,
|
||||
width: right - left + 1,
|
||||
height: bottom - top + 1,
|
||||
};
|
||||
}
|
||||
|
||||
function extractScaleAdjustAlphaChannel(params: {
|
||||
height: number;
|
||||
rawRgba: Buffer;
|
||||
width: number;
|
||||
}) {
|
||||
const { height, rawRgba, width } = params;
|
||||
const alpha = Buffer.alloc(width * height);
|
||||
for (let index = 0; index < width * height; index += 1) {
|
||||
alpha[index] = rawRgba[index * 4 + 3] || 0;
|
||||
}
|
||||
return alpha;
|
||||
}
|
||||
|
||||
async function resizeScaleAdjustAlphaMask(params: {
|
||||
height: number;
|
||||
maskBuffer: Buffer;
|
||||
targetHeight: number;
|
||||
targetWidth: number;
|
||||
width: number;
|
||||
}) {
|
||||
const { height, maskBuffer, targetHeight, targetWidth, width } = params;
|
||||
const { data, info } = await sharp(maskBuffer, {
|
||||
raw: { channels: 1, height, width },
|
||||
})
|
||||
.resize(targetWidth, targetHeight, {
|
||||
background: { alpha: 1, b: 0, g: 0, r: 0 },
|
||||
fit: "contain",
|
||||
kernel: sharp.kernel.lanczos3,
|
||||
})
|
||||
.greyscale()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
if (info.channels === 1) return Buffer.from(data);
|
||||
|
||||
const output = Buffer.alloc(targetWidth * targetHeight);
|
||||
for (let index = 0; index < targetWidth * targetHeight; index += 1) {
|
||||
output[index] = data[index * info.channels] || 0;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
async function buildScaleAdjustRmbgStructureLockedComposite(params: {
|
||||
placement: ReturnType<typeof getScaleAdjustVisualGuidePlacement>;
|
||||
raw: Buffer;
|
||||
sourceBuffer: Buffer;
|
||||
sourceImageUrl?: string;
|
||||
sourceWidth: number;
|
||||
userId?: string;
|
||||
}): Promise<ScaleAdjustStructureLockedComposite | null> {
|
||||
const {
|
||||
placement,
|
||||
raw,
|
||||
sourceBuffer,
|
||||
sourceImageUrl,
|
||||
sourceWidth,
|
||||
userId,
|
||||
} = params;
|
||||
if (!userId) return null;
|
||||
|
||||
try {
|
||||
const sourceCropBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.extract(placement.sourceBox)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const cutoutResult = await executeRmbgJob({
|
||||
userId,
|
||||
imageBase64: sourceCropBuffer.toString("base64"),
|
||||
imageBuffer: sourceCropBuffer,
|
||||
prompt: "比例调整主体抠图：只保留框选的目标人物/家具/商品主体，输出透明背景 PNG",
|
||||
skipGallerySave: true,
|
||||
sourceImageUrl,
|
||||
});
|
||||
const cutoutBuffer = await fetchImageBufferFromUrl(cutoutResult.original_url);
|
||||
const normalizedCutout = await sharp(cutoutBuffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.ensureAlpha()
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const cutoutMetadata = await sharp(normalizedCutout, { failOn: "none" }).metadata();
|
||||
const cutoutWidth = Math.round(Number(cutoutMetadata.width || 0));
|
||||
const cutoutHeight = Math.round(Number(cutoutMetadata.height || 0));
|
||||
if (cutoutWidth <= 0 || cutoutHeight <= 0) return null;
|
||||
|
||||
const cutoutRaw = await sharp(normalizedCutout, { failOn: "none" })
|
||||
.ensureAlpha()
|
||||
.raw()
|
||||
.toBuffer();
|
||||
const alphaBbox = getScaleAdjustAlphaBoundingBox({
|
||||
height: cutoutHeight,
|
||||
rawRgba: cutoutRaw,
|
||||
width: cutoutWidth,
|
||||
});
|
||||
if (!alphaBbox) return null;
|
||||
|
||||
const alphaRatio = alphaBbox.count / Math.max(1, cutoutWidth * cutoutHeight);
|
||||
const bboxCoversCanvas =
|
||||
alphaBbox.width >= cutoutWidth * 0.97 &&
|
||||
alphaBbox.height >= cutoutHeight * 0.97;
|
||||
if (
|
||||
alphaRatio < SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO ||
|
||||
(alphaRatio > SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO && bboxCoversCanvas)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sameSizeCutout =
|
||||
Math.abs(cutoutWidth - placement.sourceBox.width) <= 2 &&
|
||||
Math.abs(cutoutHeight - placement.sourceBox.height) <= 2;
|
||||
const fullAlphaMask = extractScaleAdjustAlphaChannel({
|
||||
height: cutoutHeight,
|
||||
rawRgba: cutoutRaw,
|
||||
width: cutoutWidth,
|
||||
});
|
||||
const sourceMaskBuffer = sameSizeCutout
|
||||
? fullAlphaMask
|
||||
: await resizeScaleAdjustAlphaMask({
|
||||
height: cutoutHeight,
|
||||
maskBuffer: fullAlphaMask,
|
||||
targetHeight: placement.sourceBox.height,
|
||||
targetWidth: placement.sourceBox.width,
|
||||
width: cutoutWidth,
|
||||
});
|
||||
const background = estimateScaleAdjustBoxBackground({
|
||||
box: placement.sourceBox,
|
||||
raw,
|
||||
width: sourceWidth,
|
||||
});
|
||||
const sourceWashMask = dilateScaleAdjustStructureMask({
|
||||
height: placement.sourceBox.height,
|
||||
mask: new Uint8Array(sourceMaskBuffer),
|
||||
radius: Math.max(
|
||||
2,
|
||||
Math.round(
|
||||
Math.min(placement.sourceBox.width, placement.sourceBox.height) * 0.014
|
||||
)
|
||||
),
|
||||
width: placement.sourceBox.width,
|
||||
});
|
||||
const sourceWashPatch = await buildScaleAdjustMaskWashPatch({
|
||||
background,
|
||||
box: placement.sourceBox,
|
||||
maskBuffer: Buffer.from(sourceWashMask),
|
||||
});
|
||||
|
||||
const subjectCutout = await sharp(normalizedCutout, { failOn: "none" })
|
||||
.extract(alphaBbox)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const widthFactor =
|
||||
placement.targetBox.width / Math.max(1, placement.sourceFootprintBox.width);
|
||||
const heightFactor =
|
||||
placement.targetBox.height / Math.max(1, placement.sourceFootprintBox.height);
|
||||
const structureScale = Math.max(
|
||||
0.05,
|
||||
Math.min(
|
||||
3.5,
|
||||
Math.sqrt(
|
||||
Math.max(0.0025, widthFactor) * Math.max(0.0025, heightFactor)
|
||||
)
|
||||
)
|
||||
);
|
||||
let placedWidth = Math.max(1, Math.round(alphaBbox.width * structureScale));
|
||||
let placedHeight = Math.max(1, Math.round(alphaBbox.height * structureScale));
|
||||
if (!sameSizeCutout) {
|
||||
const maxPlacedWidth = Math.max(1, Math.round(placement.targetBox.width * 1.06));
|
||||
const maxPlacedHeight = Math.max(1, Math.round(placement.targetBox.height * 1.06));
|
||||
const fitRatio = Math.min(
|
||||
1,
|
||||
maxPlacedWidth / Math.max(1, placedWidth),
|
||||
maxPlacedHeight / Math.max(1, placedHeight)
|
||||
);
|
||||
placedWidth = Math.max(1, Math.round(placedWidth * fitRatio));
|
||||
placedHeight = Math.max(1, Math.round(placedHeight * fitRatio));
|
||||
}
|
||||
|
||||
const sourceSubjectBox = sameSizeCutout
|
||||
? {
|
||||
left: placement.sourceBox.left + alphaBbox.left,
|
||||
top: placement.sourceBox.top + alphaBbox.top,
|
||||
width: alphaBbox.width,
|
||||
height: alphaBbox.height,
|
||||
}
|
||||
: null;
|
||||
const sourceCenterX = sourceSubjectBox
|
||||
? sourceSubjectBox.left + sourceSubjectBox.width / 2
|
||||
: placement.sourceBox.left + placement.sourceBox.width / 2;
|
||||
const sourceCenterY = sourceSubjectBox
|
||||
? sourceSubjectBox.top + sourceSubjectBox.height / 2
|
||||
: placement.sourceBox.top + placement.sourceBox.height / 2;
|
||||
const targetCenterX = sourceSubjectBox
|
||||
? placement.targetBox.left +
|
||||
((sourceCenterX - placement.sourceFootprintBox.left) /
|
||||
Math.max(1, placement.sourceFootprintBox.width)) *
|
||||
placement.targetBox.width
|
||||
: placement.targetBox.left + placement.targetBox.width / 2;
|
||||
const targetCenterY = sourceSubjectBox
|
||||
? placement.targetBox.top +
|
||||
((sourceCenterY - placement.sourceFootprintBox.top) /
|
||||
Math.max(1, placement.sourceFootprintBox.height)) *
|
||||
placement.targetBox.height
|
||||
: placement.targetBox.top + placement.targetBox.height / 2;
|
||||
const structureOverlay = await sharp(subjectCutout, { failOn: "none" })
|
||||
.resize(placedWidth, placedHeight, {
|
||||
fit: "fill",
|
||||
kernel: sharp.kernel.lanczos3,
|
||||
})
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
|
||||
console.info("[Canvas Local Edit] Scale adjust RMBG structure draft applied:", {
|
||||
alphaRatio: Number(alphaRatio.toFixed(3)),
|
||||
cutoutHeight,
|
||||
cutoutWidth,
|
||||
placedHeight,
|
||||
placedWidth,
|
||||
sameSizeCutout,
|
||||
sourceHeight: placement.sourceBox.height,
|
||||
sourceWidth: placement.sourceBox.width,
|
||||
});
|
||||
|
||||
return {
|
||||
method: "rmbg-cutout",
|
||||
overlay: {
|
||||
input: structureOverlay,
|
||||
left: Math.round(targetCenterX - placedWidth / 2),
|
||||
top: Math.round(targetCenterY - placedHeight / 2),
|
||||
},
|
||||
wash: {
|
||||
input: sourceWashPatch,
|
||||
left: placement.sourceBox.left,
|
||||
top: placement.sourceBox.top,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn("[Canvas Local Edit] Scale adjust RMBG structure draft failed; fallback to pixel mask:", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function buildScaleAdjustStructureLockedComposite(params: {
|
||||
includePersonPixels?: boolean;
|
||||
placement: ReturnType<typeof getScaleAdjustVisualGuidePlacement>;
|
||||
raw: Buffer;
|
||||
sourceBuffer: Buffer;
|
||||
sourceImageUrl?: string;
|
||||
sourceWidth: number;
|
||||
}) {
|
||||
userId?: string;
|
||||
}): Promise<ScaleAdjustStructureLockedComposite | null> {
|
||||
const {
|
||||
includePersonPixels = false,
|
||||
placement,
|
||||
raw,
|
||||
sourceBuffer,
|
||||
sourceImageUrl,
|
||||
sourceWidth,
|
||||
userId,
|
||||
} = params;
|
||||
const rmbgComposite = await buildScaleAdjustRmbgStructureLockedComposite({
|
||||
placement,
|
||||
raw,
|
||||
sourceBuffer,
|
||||
sourceImageUrl,
|
||||
sourceWidth,
|
||||
userId,
|
||||
});
|
||||
if (rmbgComposite) return rmbgComposite;
|
||||
|
||||
const initialMask = buildScaleAdjustStructureInitialMask({
|
||||
box: placement.sourceBox,
|
||||
includePersonPixels,
|
||||
@ -1784,6 +2084,7 @@ async function buildScaleAdjustStructureLockedComposite(params: {
|
||||
.toBuffer();
|
||||
|
||||
return {
|
||||
method: "pixel-mask",
|
||||
overlay: {
|
||||
input: structureOverlay,
|
||||
left: Math.round(targetCenterX - placedWidth / 2),
|
||||
@ -1885,16 +2186,20 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
|
||||
scalePercent?: ScaleAdjustPercent | null;
|
||||
sourceBuffer: Buffer;
|
||||
sourceHeight: number;
|
||||
sourceImageUrl?: string;
|
||||
sourceWidth: number;
|
||||
structureLockedDraft?: boolean;
|
||||
userId?: string;
|
||||
}) {
|
||||
const {
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceImageUrl,
|
||||
sourceWidth,
|
||||
structureLockedDraft = false,
|
||||
userId,
|
||||
} = params;
|
||||
const sourceRaw = structureLockedDraft
|
||||
? await sharp(sourceBuffer, { failOn: "none" })
|
||||
@ -1946,7 +2251,9 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
|
||||
placement,
|
||||
raw: sourceRaw,
|
||||
sourceBuffer,
|
||||
sourceImageUrl,
|
||||
sourceWidth,
|
||||
userId,
|
||||
})
|
||||
: null;
|
||||
|
||||
@ -2030,10 +2337,19 @@ async function createScaleAdjustPatchReference(params: {
|
||||
scalePercent?: ScaleAdjustPercent | null;
|
||||
sourceBuffer: Buffer;
|
||||
sourceHeight: number;
|
||||
sourceImageUrl?: string;
|
||||
sourceWidth: number;
|
||||
userId?: string;
|
||||
}): Promise<ScaleAdjustPatchReference> {
|
||||
const { regions, scalePercent, sourceBuffer, sourceHeight, sourceWidth } =
|
||||
params;
|
||||
const {
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceImageUrl,
|
||||
sourceWidth,
|
||||
userId,
|
||||
} = params;
|
||||
const workspaceRect = getScaleAdjustPatchWorkspaceRect(regions, scalePercent);
|
||||
const workspacePixelRect = rectToPixels(workspaceRect, sourceWidth, sourceHeight);
|
||||
const structureLockedDraft = hasScaleAdjustEnlargeTarget(regions, scalePercent);
|
||||
@ -2042,8 +2358,10 @@ async function createScaleAdjustPatchReference(params: {
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceImageUrl,
|
||||
sourceWidth,
|
||||
structureLockedDraft,
|
||||
userId,
|
||||
});
|
||||
const [sourceCropBuffer, targetDraftCropBuffer] = await Promise.all([
|
||||
sharp(sourceBuffer, { failOn: "none" })
|
||||
@ -3216,7 +3534,9 @@ export async function POST(request: NextRequest) {
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceImageUrl,
|
||||
sourceWidth,
|
||||
userId: session.user.id,
|
||||
})
|
||||
: null;
|
||||
const shouldCreateScaleAdjustCropReferences =
|
||||
|
||||
@ -153,6 +153,8 @@ class ScaleAdjustOutputRejectedError extends Error {
|
||||
const message =
|
||||
params.reason === "outside-frame-changed"
|
||||
? "比例调整结果改变了框选区域外的画面，已拒绝该模型结果"
|
||||
: params.reason === "rectangular-seam-detected"
|
||||
? "比例调整结果出现明显矩形拼接硬边，已拒绝该模型结果"
|
||||
: "比例调整结果没有产生有效变化，已拒绝该模型结果";
|
||||
super(message);
|
||||
this.name = "ScaleAdjustOutputRejectedError";
|
||||
@ -190,6 +192,14 @@ const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO = 0.84;
|
||||
const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO = 0.14;
|
||||
const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60;
|
||||
const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018;
|
||||
const SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD = 18;
|
||||
const SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF = 5;
|
||||
const SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF = 54;
|
||||
const SCALE_ADJUST_RECTANGULAR_SEAM_EDGE_INCREASE = 2.5;
|
||||
const SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO = 0.11;
|
||||
const SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO = 0.18;
|
||||
const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO = 0.1;
|
||||
const SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO = 0.26;
|
||||
|
||||
const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
|
||||
"gemini",
|
||||
@ -869,6 +879,257 @@ function isScaleAdjustOutsidePixelChanged(params: {
|
||||
);
|
||||
}
|
||||
|
||||
function getScaleAdjustRawRgbDiff(raw: Buffer, leftIndex: number, rightIndex: number) {
|
||||
return (
|
||||
Math.abs(raw[leftIndex] - raw[rightIndex]) * 0.299 +
|
||||
Math.abs(raw[leftIndex + 1] - raw[rightIndex + 1]) * 0.587 +
|
||||
Math.abs(raw[leftIndex + 2] - raw[rightIndex + 2]) * 0.114
|
||||
);
|
||||
}
|
||||
|
||||
function getScaleAdjustPixelChanged(params: {
|
||||
generatedRaw: Buffer;
|
||||
index: number;
|
||||
sourceRaw: Buffer;
|
||||
}) {
|
||||
return (
|
||||
Math.abs(params.generatedRaw[params.index] - params.sourceRaw[params.index]) *
|
||||
0.299 +
|
||||
Math.abs(
|
||||
params.generatedRaw[params.index + 1] -
|
||||
params.sourceRaw[params.index + 1]
|
||||
) *
|
||||
0.587 +
|
||||
Math.abs(
|
||||
params.generatedRaw[params.index + 2] -
|
||||
params.sourceRaw[params.index + 2]
|
||||
) *
|
||||
0.114
|
||||
) >= SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD;
|
||||
}
|
||||
|
||||
function summarizeScaleAdjustSeamLine(params: {
|
||||
axis: "horizontal" | "vertical";
|
||||
boxIndex: number;
|
||||
changeMask: Uint8Array;
|
||||
generatedRaw: Buffer;
|
||||
height: number;
|
||||
position: number;
|
||||
rangeEnd: number;
|
||||
rangeStart: number;
|
||||
sourceRaw: Buffer;
|
||||
width: number;
|
||||
}) {
|
||||
const {
|
||||
axis,
|
||||
boxIndex,
|
||||
changeMask,
|
||||
generatedRaw,
|
||||
height,
|
||||
position,
|
||||
rangeEnd,
|
||||
rangeStart,
|
||||
sourceRaw,
|
||||
width,
|
||||
} = params;
|
||||
let transitionCount = 0;
|
||||
let currentRun = 0;
|
||||
let maxRun = 0;
|
||||
let edgeDiffSum = 0;
|
||||
const sampleCount = Math.max(1, rangeEnd - rangeStart);
|
||||
|
||||
for (let offset = rangeStart; offset < rangeEnd; offset += 1) {
|
||||
const leftX = axis === "vertical" ? position - 1 : offset;
|
||||
const leftY = axis === "vertical" ? offset : position - 1;
|
||||
const rightX = axis === "vertical" ? position : offset;
|
||||
const rightY = axis === "vertical" ? offset : position;
|
||||
if (
|
||||
leftX < 0 ||
|
||||
rightX < 0 ||
|
||||
leftY < 0 ||
|
||||
rightY < 0 ||
|
||||
leftX >= width ||
|
||||
rightX >= width ||
|
||||
leftY >= height ||
|
||||
rightY >= height
|
||||
) {
|
||||
currentRun = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
const leftPixel = leftY * width + leftX;
|
||||
const rightPixel = rightY * width + rightX;
|
||||
const changedBoundary =
|
||||
changeMask[leftPixel] !== changeMask[rightPixel];
|
||||
const leftRawIndex = leftPixel * 3;
|
||||
const rightRawIndex = rightPixel * 3;
|
||||
const generatedEdgeDiff = getScaleAdjustRawRgbDiff(
|
||||
generatedRaw,
|
||||
leftRawIndex,
|
||||
rightRawIndex
|
||||
);
|
||||
const sourceEdgeDiff = getScaleAdjustRawRgbDiff(
|
||||
sourceRaw,
|
||||
leftRawIndex,
|
||||
rightRawIndex
|
||||
);
|
||||
const likelyBackgroundSeam =
|
||||
generatedEdgeDiff >= SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF &&
|
||||
generatedEdgeDiff <= SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF &&
|
||||
generatedEdgeDiff - sourceEdgeDiff >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_EDGE_INCREASE;
|
||||
|
||||
if (changedBoundary && likelyBackgroundSeam) {
|
||||
transitionCount += 1;
|
||||
currentRun += 1;
|
||||
maxRun = Math.max(maxRun, currentRun);
|
||||
edgeDiffSum += generatedEdgeDiff;
|
||||
} else {
|
||||
currentRun = 0;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
axis,
|
||||
boxIndex,
|
||||
edgeDiffMean: transitionCount > 0 ? edgeDiffSum / transitionCount : 0,
|
||||
maxRun,
|
||||
maxRunRatio: maxRun / sampleCount,
|
||||
position,
|
||||
sampleCount,
|
||||
transitionCount,
|
||||
transitionRatio: transitionCount / sampleCount,
|
||||
};
|
||||
}
|
||||
|
||||
async function getScaleAdjustRectangularSeamStats(params: {
|
||||
generatedBuffer: Buffer;
|
||||
sourceBuffer: Buffer;
|
||||
sourceWidth: number;
|
||||
sourceHeight: number;
|
||||
verification: CanvasScaleAdjustVerificationConfig;
|
||||
}) {
|
||||
const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } =
|
||||
params;
|
||||
const normalizeToRaw = (buffer: Buffer) =>
|
||||
sharp(buffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.resize(sourceWidth, sourceHeight, {
|
||||
fit: "cover",
|
||||
position: "center",
|
||||
})
|
||||
.removeAlpha()
|
||||
.toColorspace("srgb")
|
||||
.raw()
|
||||
.toBuffer();
|
||||
const [sourceRaw, generatedRaw] = await Promise.all([
|
||||
normalizeToRaw(sourceBuffer),
|
||||
normalizeToRaw(generatedBuffer),
|
||||
]);
|
||||
const changeMask = new Uint8Array(sourceWidth * sourceHeight);
|
||||
for (let pixel = 0; pixel < sourceWidth * sourceHeight; pixel += 1) {
|
||||
const rawIndex = pixel * 3;
|
||||
changeMask[pixel] = getScaleAdjustPixelChanged({
|
||||
generatedRaw,
|
||||
index: rawIndex,
|
||||
sourceRaw,
|
||||
})
|
||||
? 1
|
||||
: 0;
|
||||
}
|
||||
|
||||
const protectedBoxes = getScaleAdjustProtectedRegions(verification).map((region) =>
|
||||
rectToPixelBox({
|
||||
height: sourceHeight,
|
||||
region,
|
||||
width: sourceWidth,
|
||||
})
|
||||
);
|
||||
const lineStep = Math.max(
|
||||
1,
|
||||
Math.floor(Math.min(sourceWidth, sourceHeight) / 1400)
|
||||
);
|
||||
let bestVertical: ReturnType<typeof summarizeScaleAdjustSeamLine> | null = null;
|
||||
let bestHorizontal: ReturnType<typeof summarizeScaleAdjustSeamLine> | null = null;
|
||||
|
||||
for (const [boxIndex, box] of protectedBoxes.entries()) {
|
||||
const left = Math.max(2, box.left + 2);
|
||||
const top = Math.max(2, box.top + 2);
|
||||
const right = Math.min(sourceWidth - 2, box.left + box.width - 2);
|
||||
const bottom = Math.min(sourceHeight - 2, box.top + box.height - 2);
|
||||
if (right <= left || bottom <= top) continue;
|
||||
|
||||
for (let x = left; x <= right; x += lineStep) {
|
||||
const stats = summarizeScaleAdjustSeamLine({
|
||||
axis: "vertical",
|
||||
boxIndex,
|
||||
changeMask,
|
||||
generatedRaw,
|
||||
height: sourceHeight,
|
||||
position: x,
|
||||
rangeEnd: bottom,
|
||||
rangeStart: top,
|
||||
sourceRaw,
|
||||
width: sourceWidth,
|
||||
});
|
||||
if (!bestVertical || stats.maxRunRatio > bestVertical.maxRunRatio) {
|
||||
bestVertical = stats;
|
||||
}
|
||||
}
|
||||
|
||||
for (let y = top; y <= bottom; y += lineStep) {
|
||||
const stats = summarizeScaleAdjustSeamLine({
|
||||
axis: "horizontal",
|
||||
boxIndex,
|
||||
changeMask,
|
||||
generatedRaw,
|
||||
height: sourceHeight,
|
||||
position: y,
|
||||
rangeEnd: right,
|
||||
rangeStart: left,
|
||||
sourceRaw,
|
||||
width: sourceWidth,
|
||||
});
|
||||
if (!bestHorizontal || stats.maxRunRatio > bestHorizontal.maxRunRatio) {
|
||||
bestHorizontal = stats;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const verticalPair =
|
||||
bestVertical &&
|
||||
bestVertical.maxRunRatio >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO &&
|
||||
bestVertical.transitionRatio >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO;
|
||||
const horizontalPair =
|
||||
bestHorizontal &&
|
||||
bestHorizontal.maxRunRatio >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_RUN_RATIO &&
|
||||
bestHorizontal.transitionRatio >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_PAIR_TRANSITION_RATIO;
|
||||
const verticalStrong =
|
||||
bestVertical &&
|
||||
bestVertical.maxRunRatio >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO &&
|
||||
bestVertical.transitionRatio >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO;
|
||||
const horizontalStrong =
|
||||
bestHorizontal &&
|
||||
bestHorizontal.maxRunRatio >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_RUN_RATIO &&
|
||||
bestHorizontal.transitionRatio >=
|
||||
SCALE_ADJUST_RECTANGULAR_SEAM_STRONG_TRANSITION_RATIO;
|
||||
|
||||
return {
|
||||
detected: Boolean((verticalPair && horizontalPair) || verticalStrong || horizontalStrong),
|
||||
horizontal: bestHorizontal,
|
||||
lineStep,
|
||||
protectedBoxCount: protectedBoxes.length,
|
||||
vertical: bestVertical,
|
||||
};
|
||||
}
|
||||
|
||||
async function getScaleAdjustFootprintChangeStats(params: {
|
||||
generatedBuffer: Buffer;
|
||||
sourceBuffer: Buffer;
|
||||
@ -1232,6 +1493,10 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
outsidePixelStats: Awaited<
|
||||
ReturnType<typeof getScaleAdjustOutsidePixelChangeStats>
|
||||
>;
|
||||
rectangularSeamDetected: boolean;
|
||||
rectangularSeamStats: Awaited<
|
||||
ReturnType<typeof getScaleAdjustRectangularSeamStats>
|
||||
>;
|
||||
}> = [];
|
||||
for (const [index, image] of images.entries()) {
|
||||
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||||
@ -1243,6 +1508,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
outsideDiffScores,
|
||||
outsidePixelStats,
|
||||
footprintStats,
|
||||
rectangularSeamStats,
|
||||
] = await Promise.all([
|
||||
getScaleAdjustDiffScores({
|
||||
generatedBuffer,
|
||||
@ -1279,6 +1545,13 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
sourceWidth: sourceImage.sourceWidth,
|
||||
verification,
|
||||
}),
|
||||
getScaleAdjustRectangularSeamStats({
|
||||
generatedBuffer,
|
||||
sourceBuffer: sourceImage.sourceBuffer,
|
||||
sourceHeight: sourceImage.sourceHeight,
|
||||
sourceWidth: sourceImage.sourceWidth,
|
||||
verification,
|
||||
}),
|
||||
]);
|
||||
const visuallyChangedInside = changeDiffScores.some(
|
||||
(score) => score >= SCALE_ADJUST_VISUAL_DIFF_THRESHOLD
|
||||
@ -1287,6 +1560,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
const outsideChanged =
|
||||
isScaleAdjustOutsideChanged(outsideDiffScores) ||
|
||||
isScaleAdjustOutsidePixelChanged(outsidePixelStats);
|
||||
const rectangularSeamDetected = rectangularSeamStats.detected;
|
||||
|
||||
console.info("[Canvas Generate Image] Scale adjust result guard:", {
|
||||
debugLabel,
|
||||
@ -1320,6 +1594,38 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
Number(score.toFixed(2))
|
||||
),
|
||||
protectedRegionCount: verification.protectedRegions?.length || 0,
|
||||
rectangularSeamDetected,
|
||||
rectangularSeamStats: {
|
||||
horizontal: rectangularSeamStats.horizontal
|
||||
? {
|
||||
edgeDiffMean: Number(
|
||||
rectangularSeamStats.horizontal.edgeDiffMean.toFixed(2)
|
||||
),
|
||||
maxRunRatio: Number(
|
||||
rectangularSeamStats.horizontal.maxRunRatio.toFixed(3)
|
||||
),
|
||||
position: rectangularSeamStats.horizontal.position,
|
||||
transitionRatio: Number(
|
||||
rectangularSeamStats.horizontal.transitionRatio.toFixed(3)
|
||||
),
|
||||
}
|
||||
: null,
|
||||
lineStep: rectangularSeamStats.lineStep,
|
||||
vertical: rectangularSeamStats.vertical
|
||||
? {
|
||||
edgeDiffMean: Number(
|
||||
rectangularSeamStats.vertical.edgeDiffMean.toFixed(2)
|
||||
),
|
||||
maxRunRatio: Number(
|
||||
rectangularSeamStats.vertical.maxRunRatio.toFixed(3)
|
||||
),
|
||||
position: rectangularSeamStats.vertical.position,
|
||||
transitionRatio: Number(
|
||||
rectangularSeamStats.vertical.transitionRatio.toFixed(3)
|
||||
),
|
||||
}
|
||||
: null,
|
||||
},
|
||||
regionCount: verification.regions.length,
|
||||
regionDiffScores: diffScores.map((score) => Number(score.toFixed(2))),
|
||||
scalePercent: verification.scalePercent ?? null,
|
||||
@ -1329,6 +1635,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
const guardWarnings = [
|
||||
...(outsideChanged ? ["outside-frame-changed"] : []),
|
||||
...(!changedInside ? ["no-effective-scale-change"] : []),
|
||||
...(rectangularSeamDetected ? ["rectangular-seam-detected"] : []),
|
||||
];
|
||||
if (guardWarnings.length > 0) {
|
||||
console.warn("[Canvas Generate Image] Scale adjust result guard warning; keeping result for preview:", {
|
||||
@ -1360,6 +1667,37 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
sampledPixelCount: outsidePixelStats.sampledPixelCount,
|
||||
step: outsidePixelStats.step,
|
||||
},
|
||||
rectangularSeamDetected,
|
||||
rectangularSeamStats: {
|
||||
horizontal: rectangularSeamStats.horizontal
|
||||
? {
|
||||
edgeDiffMean: Number(
|
||||
rectangularSeamStats.horizontal.edgeDiffMean.toFixed(2)
|
||||
),
|
||||
maxRunRatio: Number(
|
||||
rectangularSeamStats.horizontal.maxRunRatio.toFixed(3)
|
||||
),
|
||||
position: rectangularSeamStats.horizontal.position,
|
||||
transitionRatio: Number(
|
||||
rectangularSeamStats.horizontal.transitionRatio.toFixed(3)
|
||||
),
|
||||
}
|
||||
: null,
|
||||
vertical: rectangularSeamStats.vertical
|
||||
? {
|
||||
edgeDiffMean: Number(
|
||||
rectangularSeamStats.vertical.edgeDiffMean.toFixed(2)
|
||||
),
|
||||
maxRunRatio: Number(
|
||||
rectangularSeamStats.vertical.maxRunRatio.toFixed(3)
|
||||
),
|
||||
position: rectangularSeamStats.vertical.position,
|
||||
transitionRatio: Number(
|
||||
rectangularSeamStats.vertical.transitionRatio.toFixed(3)
|
||||
),
|
||||
}
|
||||
: null,
|
||||
},
|
||||
changeRegionDiffScores: changeDiffScores.map((score) =>
|
||||
Number(score.toFixed(2))
|
||||
),
|
||||
@ -1392,6 +1730,8 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
outsideChanged,
|
||||
outsideDiffScores,
|
||||
outsidePixelStats,
|
||||
rectangularSeamDetected,
|
||||
rectangularSeamStats,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
@ -1414,6 +1754,8 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
outsideChanged,
|
||||
outsideDiffScores,
|
||||
outsidePixelStats,
|
||||
rectangularSeamDetected,
|
||||
rectangularSeamStats,
|
||||
});
|
||||
}
|
||||
|
||||
@ -1424,11 +1766,20 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
const outsideMean = item.outsidePixelStats.meanDiff;
|
||||
const outsidePenalty = item.outsideChanged ? 100 : 0;
|
||||
const noChangePenalty = item.changedInside ? 0 : 60;
|
||||
return insideScore - outsideMean * 0.35 - outsidePenalty - noChangePenalty;
|
||||
const seamPenalty = item.rectangularSeamDetected ? 140 : 0;
|
||||
return (
|
||||
insideScore -
|
||||
outsideMean * 0.35 -
|
||||
outsidePenalty -
|
||||
noChangePenalty -
|
||||
seamPenalty
|
||||
);
|
||||
};
|
||||
const rankedImages = [...evaluatedImages].sort((left, right) => {
|
||||
const leftPassed = left.changedInside && !left.outsideChanged;
|
||||
const rightPassed = right.changedInside && !right.outsideChanged;
|
||||
const leftPassed =
|
||||
left.changedInside && !left.outsideChanged && !left.rectangularSeamDetected;
|
||||
const rightPassed =
|
||||
right.changedInside && !right.outsideChanged && !right.rectangularSeamDetected;
|
||||
if (leftPassed !== rightPassed) return leftPassed ? -1 : 1;
|
||||
return scoreEvaluation(right) - scoreEvaluation(left);
|
||||
});
|
||||
@ -1438,9 +1789,13 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
debugLabel,
|
||||
candidateCount: evaluatedImages.length,
|
||||
imageIndex: selected.index,
|
||||
passed: selected.changedInside && !selected.outsideChanged,
|
||||
passed:
|
||||
selected.changedInside &&
|
||||
!selected.outsideChanged &&
|
||||
!selected.rectangularSeamDetected,
|
||||
changedInside: selected.changedInside,
|
||||
outsideChanged: selected.outsideChanged,
|
||||
rectangularSeamDetected: selected.rectangularSeamDetected,
|
||||
changeRegionDiffScores: selected.changeDiffScores.map((score) =>
|
||||
Number(score.toFixed(2))
|
||||
),
|
||||
@ -1473,12 +1828,16 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
|
||||
if (
|
||||
verification.rejectFailedCandidates &&
|
||||
(!selected.changedInside || (selected.outsideChanged && !outsideChangeAccepted))
|
||||
(!selected.changedInside ||
|
||||
selected.rectangularSeamDetected ||
|
||||
(selected.outsideChanged && !outsideChangeAccepted))
|
||||
) {
|
||||
throw new ScaleAdjustOutputRejectedError({
|
||||
reason: selected.outsideChanged
|
||||
? "outside-frame-changed"
|
||||
: "no-effective-scale-change",
|
||||
reason: selected.rectangularSeamDetected
|
||||
? "rectangular-seam-detected"
|
||||
: selected.outsideChanged
|
||||
? "outside-frame-changed"
|
||||
: "no-effective-scale-change",
|
||||
diffScores: selected.changeDiffScores,
|
||||
images: [selected.cleanedImage || selected.image],
|
||||
outsideDiffScores: selected.outsideDiffScores,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user