lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: refine scale adjust product lock repair mask

Browse Source
This commit is contained in:
134 2026-07-08 01:11:14 +08:00
parent 0add599427
commit 8fad43c597
2 changed files with 217 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
app/api/canvas/local-edit/route.ts
View File

@ -126,6 +126,7 @@ type ScaleAdjustCropReference = {
type ScaleAdjustPatchReference = {
backgroundPlateFullUrl?: string | null;
compositeClipMaskUrl?: string | null;
repairBlendMaskUrl?: string | null;
productLockMethod?: string | null;
productLockEditMaskUrl?: string | null;
productLockFalEditMaskUrl?: string | null;
@ -509,14 +510,6 @@ function resolveScaleAdjustPreferredModelId(params: {
params.requestedModelId || params.configuredModelId
)
);
if (params.preferMaskCapableGpt) {
const maskCapableGptImage2 = gptImage2Candidates
.filter(isScaleAdjustMaskCapableModel)
.filter((entry) => entry.model.healthStatus !== "auto_disabled");
if (maskCapableGptImage2[0]) {
return maskCapableGptImage2[0].model.id;
}
}
for (const providerType of params.preferProviderTypes || []) {
const providerCandidate = gptImage2Candidates.find(
(entry) =>
@ -526,6 +519,14 @@ function resolveScaleAdjustPreferredModelId(params: {
return providerCandidate.model.id;
}
}
if (params.preferMaskCapableGpt) {
const maskCapableGptImage2 = gptImage2Candidates
.filter(isScaleAdjustMaskCapableModel)
.filter((entry) => entry.model.healthStatus !== "auto_disabled");
if (maskCapableGptImage2[0]) {
return maskCapableGptImage2[0].model.id;
}
}
if (params.preferFalNanoBananaEdit) {
const falEditCandidate = params.models
.filter((entry) => isScaleAdjustUsableImageEditModel(entry))
@ -2278,6 +2279,81 @@ async function buildScaleAdjustCompositeClipMaskBuffer(params: {
.toBuffer();
}
async function buildScaleAdjustProductLockRepairBlendMask(params: {
height: number;
productLockOverlayBuffer: Buffer | null;
productLockProtectMaskBuffer: Buffer | null;
sourceRepairMaskBuffer: Buffer | null;
width: number;
}) {
const {
height,
productLockOverlayBuffer,
productLockProtectMaskBuffer,
sourceRepairMaskBuffer,
width,
} = params;
if (!sourceRepairMaskBuffer) return null;
const repairAlpha = await sharp(sourceRepairMaskBuffer, { failOn: "none" })
.rotate()
.resize(width, height, { fit: "fill" })
.ensureAlpha()
.extractChannel(3)
.greyscale()
.raw()
.toBuffer();
const overlayAlpha = productLockOverlayBuffer
? await sharp(productLockOverlayBuffer, { failOn: "none" })
.rotate()
.resize(width, height, { fit: "fill" })
.ensureAlpha()
.extractChannel(3)
.greyscale()
.threshold(8)
.blur(Math.max(6, Math.round(Math.min(width, height) * 0.012)))
.greyscale()
.raw()
.toBuffer()
: null;
const protectAlpha = productLockProtectMaskBuffer
? await sharp(productLockProtectMaskBuffer, { failOn: "none" })
.rotate()
.resize(width, height, { fit: "fill" })
.ensureAlpha()
.extractChannel(3)
.greyscale()
.threshold(8)
.blur(Math.max(4, Math.round(Math.min(width, height) * 0.008)))
.greyscale()
.raw()
.toBuffer()
: null;
const rgba = Buffer.alloc(width * height * 4);
let markedPixels = 0;
for (let index = 0; index < width * height; index += 1) {
const offset = index * 4;
const protectedValue = Math.max(
overlayAlpha?.[index] || 0,
protectAlpha?.[index] || 0
);
const alpha = clamp((repairAlpha[index] || 0) - protectedValue, 0, 255);
if (alpha > 8) markedPixels += 1;
rgba[offset] = 255;
rgba[offset + 1] = 255;
rgba[offset + 2] = 255;
rgba[offset + 3] = alpha;
}
if (markedPixels === 0) return sourceRepairMaskBuffer;
return sharp(rgba, {
raw: { channels: 4, height, width },
})
.png({ compressionLevel: 6 })
.toBuffer();
}
async function buildScaleAdjustTransparentEditMaskFromOpacityMask(params: {
height: number;
maskBuffer: Buffer;
@ -3805,6 +3881,15 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
width: sourceWidth,
})
: null;
const repairBlendMaskBuffer = productLockComposites.length
? await buildScaleAdjustProductLockRepairBlendMask({
height: sourceHeight,
productLockOverlayBuffer,
productLockProtectMaskBuffer,
sourceRepairMaskBuffer: compositeClipMaskBuffer,
width: sourceWidth,
})
: compositeClipMaskBuffer;
const productLockEditMaskBuffer =
productLockComposites.length && compositeClipMaskBuffer
? await buildScaleAdjustTransparentEditMaskFromOpacityMask({
@ -3826,6 +3911,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
backgroundPlateBuffer,
buffer,
compositeClipMaskBuffer,
repairBlendMaskBuffer,
productLockEditMaskBuffer,
productLockFalEditMaskBuffer,
productLockOverlayBuffer,
@ -3912,7 +3998,12 @@ async function createScaleAdjustPatchReference(params: {
targetDraftReference.productLockFalEditMaskBuffer
),
]);
const [backgroundPlateFullUpload, productLockProtectMaskUpload, compositeClipMaskUpload] =
const [
backgroundPlateFullUpload,
productLockProtectMaskUpload,
compositeClipMaskUpload,
repairBlendMaskUpload,
] =
await Promise.all([
uploadOptionalScaleAdjustAsset(
`canvas-scale-adjust-background-plate-${suffix}.jpg`,
@ -3926,11 +4017,16 @@ async function createScaleAdjustPatchReference(params: {
`canvas-scale-adjust-composite-clip-mask-${suffix}.png`,
targetDraftReference.compositeClipMaskBuffer
),
uploadOptionalScaleAdjustAsset(
`canvas-scale-adjust-repair-blend-mask-${suffix}.png`,
targetDraftReference.repairBlendMaskBuffer
),
]);
return {
backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null,
compositeClipMaskUrl: compositeClipMaskUpload?.url || null,
repairBlendMaskUrl: repairBlendMaskUpload?.url || null,
productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null,
productLockEditMaskUrl: productLockEditMaskUpload?.url || null,
productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null,
@ -3975,6 +4071,7 @@ async function createScaleAdjustPatchReference(params: {
productLockOverlayUpload,
productLockProtectMaskUpload,
compositeClipMaskUpload,
repairBlendMaskUpload,
productLockEditMaskUpload,
productLockFalEditMaskUpload,
] = await Promise.all([
@ -3994,6 +4091,10 @@ async function createScaleAdjustPatchReference(params: {
`canvas-scale-adjust-composite-clip-mask-${suffix}.png`,
targetDraftReference.compositeClipMaskBuffer
),
uploadOptionalScaleAdjustAsset(
`canvas-scale-adjust-repair-blend-mask-${suffix}.png`,
targetDraftReference.repairBlendMaskBuffer
),
uploadOptionalScaleAdjustAsset(
`canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`,
targetDraftReference.productLockEditMaskBuffer
@ -4013,6 +4114,7 @@ async function createScaleAdjustPatchReference(params: {
return {
backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null,
compositeClipMaskUrl: compositeClipMaskUpload?.url || null,
repairBlendMaskUrl: repairBlendMaskUpload?.url || null,
productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null,
productLockEditMaskUrl: productLockEditMaskUpload?.url || null,
productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null,
@ -5178,7 +5280,9 @@ export async function POST(request: NextRequest) {
const preferScaleAdjustFalNanoBananaEdit =
hasScaleAdjustShrink && !hasScaleAdjustEnlarge;
const preferScaleAdjustProviderTypes =
hasScaleAdjustShrink && !hasScaleAdjustEnlarge ? ["grsai-gpt-image"] : [];
hasScaleAdjustShrink && !hasScaleAdjustEnlarge
? ["snapgen-image", "grsai-gpt-image"]
: [];
const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent });
const resolvedAspectRatio = resolveLocalEditAspectRatio(
body.aspectRatio,
@ -5687,6 +5791,8 @@ export async function POST(request: NextRequest) {
scaleAdjustPatchReference?.productLockProtectMaskUrl || null,
scaleAdjustPatchCompositeClipMaskUrl:
scaleAdjustPatchReference?.compositeClipMaskUrl || null,
scaleAdjustRepairBlendMaskUrl:
scaleAdjustPatchReference?.repairBlendMaskUrl || null,
scaleAdjustProviderMaskImageUrl: useScaleAdjustProductLockMaskEdit
? scaleAdjustPatchReference?.productLockEditMaskUrl || null
: scaleAdjustEditMaskUpload?.url || null,
@ -5867,7 +5973,9 @@ export async function POST(request: NextRequest) {
protectMaskImageUrl:
scaleAdjustPatchReference?.productLockProtectMaskUrl || undefined,
repairMaskImageUrl:
scaleAdjustPatchReference?.compositeClipMaskUrl || undefined,
scaleAdjustPatchReference?.repairBlendMaskUrl ||
scaleAdjustPatchReference?.compositeClipMaskUrl ||
undefined,
sourceImageUrl,
sourceHeight,
sourceWidth,

107
lib/canvas/generate-image.ts
View File

@ -2833,6 +2833,81 @@ async function applyRawAlphaToImage(params: {
.toBuffer();
}
async function matchImageColorToSourceNearAlphaMask(params: {
alphaMaskRaw: Buffer;
height: number;
imageBuffer: Buffer;
sourceBuffer: Buffer;
width: number;
}) {
const { alphaMaskRaw, height, imageBuffer, sourceBuffer, width } = params;
if (alphaMaskRaw.length !== width * height) return imageBuffer;
const [imageRaw, sourceRaw] = await Promise.all([
sharp(imageBuffer, { failOn: "none" })
.resize(width, height, { fit: "fill" })
.removeAlpha()
.toColorspace("srgb")
.raw()
.toBuffer(),
sharp(sourceBuffer, { failOn: "none" })
.resize(width, height, { fit: "fill" })
.removeAlpha()
.toColorspace("srgb")
.raw()
.toBuffer(),
]);
const sourceSum = [0, 0, 0];
const imageSum = [0, 0, 0];
let count = 0;
for (let index = 0; index < alphaMaskRaw.length; index += 1) {
const alpha = alphaMaskRaw[index] || 0;
if (alpha < 4 || alpha > 110) continue;
const offset = index * 3;
const sourceLuma =
(sourceRaw[offset] || 0) * 0.299 +
(sourceRaw[offset + 1] || 0) * 0.587 +
(sourceRaw[offset + 2] || 0) * 0.114;
const imageLuma =
(imageRaw[offset] || 0) * 0.299 +
(imageRaw[offset + 1] || 0) * 0.587 +
(imageRaw[offset + 2] || 0) * 0.114;
if (sourceLuma < 24 || sourceLuma > 248 || imageLuma < 24 || imageLuma > 248) {
continue;
}
for (let channel = 0; channel < 3; channel += 1) {
sourceSum[channel] += sourceRaw[offset + channel] || 0;
imageSum[channel] += imageRaw[offset + channel] || 0;
}
count += 1;
}
if (count < 500) return imageBuffer;
const offsets = sourceSum.map((sum, channel) =>
Math.max(-32, Math.min(32, sum / count - imageSum[channel] / count))
);
const correctedRaw = Buffer.from(imageRaw);
for (let index = 0; index < correctedRaw.length; index += 3) {
correctedRaw[index] = Math.max(
0,
Math.min(255, Math.round((correctedRaw[index] || 0) + offsets[0]))
);
correctedRaw[index + 1] = Math.max(
0,
Math.min(255, Math.round((correctedRaw[index + 1] || 0) + offsets[1]))
);
correctedRaw[index + 2] = Math.max(
0,
Math.min(255, Math.round((correctedRaw[index + 2] || 0) + offsets[2]))
);
}
return sharp(correctedRaw, { raw: { channels: 3, height, width } })
.png({ compressionLevel: 6 })
.toBuffer();
}
async function applyScaleAdjustProductProtectMaskToOverlay(params: {
height: number;
overlayBuffer: Buffer;
@ -3356,18 +3431,32 @@ async function applyScaleAdjustProductLockCompositeToImages(params: {
.png({ compressionLevel: 6 })
.toBuffer()
: normalizedGeneratedBuffer;
const generatedRepairOverlay =
const repairAlphaMaskRaw =
sourceBaseBuffer && repairMaskBuffer
? await applyRawAlphaToImage({
alphaMaskRaw: await sharp(repairMaskBuffer, { failOn: "none" })
.resize(generatedWidth, generatedHeight, { fit: "fill" })
.ensureAlpha()
.extractChannel(3)
.blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006)))
.raw()
.toBuffer(),
? await sharp(repairMaskBuffer, { failOn: "none" })
.resize(generatedWidth, generatedHeight, { fit: "fill" })
.ensureAlpha()
.extractChannel(3)
.blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006)))
.raw()
.toBuffer()
: null;
const colorMatchedGeneratedBuffer =
sourceBaseBuffer && repairAlphaMaskRaw
? await matchImageColorToSourceNearAlphaMask({
alphaMaskRaw: repairAlphaMaskRaw,
height: generatedHeight,
imageBuffer: normalizedGeneratedBuffer,
sourceBuffer: baseForProductLock,
width: generatedWidth,
})
: normalizedGeneratedBuffer;
const generatedRepairOverlay =
repairAlphaMaskRaw
? await applyRawAlphaToImage({
alphaMaskRaw: repairAlphaMaskRaw,
height: generatedHeight,
imageBuffer: colorMatchedGeneratedBuffer,
width: generatedWidth,
})
: null;