501 lines
15 KiB
TypeScript
501 lines
15 KiB
TypeScript
import "server-only";
|
|
|
|
import sharp from "sharp";
|
|
|
|
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
|
import type { CanvasLocalEditCompositeRegion } from "@/lib/canvas/generate-image";
|
|
|
|
type SemanticProductRegion = {
|
|
candidateImageUrls: string[];
|
|
sourceCropImageUrl: string;
|
|
target: CanvasLocalEditCompositeRegion;
|
|
};
|
|
|
|
const COLOR_BINS_PER_CHANNEL = 6;
|
|
const COLOR_PROFILE_SIZE = COLOR_BINS_PER_CHANNEL ** 3;
|
|
const MAGENTA_DESPILL_MIN_ALPHA = 12;
|
|
const MAGENTA_LOW_DISTANCE = 28;
|
|
const MAGENTA_HIGH_DISTANCE = 190;
|
|
|
|
function clamp(value: number, min: number, max: number) {
|
|
return Math.min(max, Math.max(min, value));
|
|
}
|
|
|
|
function smoothStep(value: number) {
|
|
const progress = clamp(value, 0, 1);
|
|
return progress * progress * (3 - 2 * progress);
|
|
}
|
|
|
|
type ColorProfile = {
|
|
bins: Float64Array;
|
|
total: number;
|
|
};
|
|
|
|
function getColorBin(red: number, green: number, blue: number) {
|
|
const toBin = (value: number) =>
|
|
Math.min(
|
|
COLOR_BINS_PER_CHANNEL - 1,
|
|
Math.floor((value * COLOR_BINS_PER_CHANNEL) / 256)
|
|
);
|
|
return (
|
|
toBin(red) * COLOR_BINS_PER_CHANNEL ** 2 +
|
|
toBin(green) * COLOR_BINS_PER_CHANNEL +
|
|
toBin(blue)
|
|
);
|
|
}
|
|
|
|
function isInformativeColor(red: number, green: number, blue: number) {
|
|
const range = Math.max(red, green, blue) - Math.min(red, green, blue);
|
|
const luminance = red * 0.299 + green * 0.587 + blue * 0.114;
|
|
return range >= 14 || luminance < 190;
|
|
}
|
|
|
|
function buildColorProfile(params: {
|
|
alpha?: Buffer;
|
|
channels: number;
|
|
data: Buffer;
|
|
}) {
|
|
const { alpha, channels, data } = params;
|
|
const bins = new Float64Array(COLOR_PROFILE_SIZE);
|
|
let total = 0;
|
|
const pixelCount = Math.floor(data.length / channels);
|
|
|
|
for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) {
|
|
const alphaWeight = alpha ? (alpha[pixelIndex] || 0) / 255 : 1;
|
|
if (alphaWeight < 0.12) continue;
|
|
const offset = pixelIndex * channels;
|
|
const red = data[offset] || 0;
|
|
const green = data[offset + 1] || 0;
|
|
const blue = data[offset + 2] || 0;
|
|
if (!isInformativeColor(red, green, blue)) continue;
|
|
bins[getColorBin(red, green, blue)] += alphaWeight;
|
|
total += alphaWeight;
|
|
}
|
|
|
|
return { bins, total };
|
|
}
|
|
|
|
function getNeighborBinWeight(profile: ColorProfile, binIndex: number) {
|
|
const redBin = Math.floor(binIndex / COLOR_BINS_PER_CHANNEL ** 2);
|
|
const remainder = binIndex % COLOR_BINS_PER_CHANNEL ** 2;
|
|
const greenBin = Math.floor(remainder / COLOR_BINS_PER_CHANNEL);
|
|
const blueBin = remainder % COLOR_BINS_PER_CHANNEL;
|
|
let weight = 0;
|
|
|
|
for (let redOffset = -1; redOffset <= 1; redOffset += 1) {
|
|
for (let greenOffset = -1; greenOffset <= 1; greenOffset += 1) {
|
|
for (let blueOffset = -1; blueOffset <= 1; blueOffset += 1) {
|
|
if (Math.abs(redOffset) + Math.abs(greenOffset) + Math.abs(blueOffset) > 1) {
|
|
continue;
|
|
}
|
|
const nextRed = redBin + redOffset;
|
|
const nextGreen = greenBin + greenOffset;
|
|
const nextBlue = blueBin + blueOffset;
|
|
if (
|
|
nextRed < 0 ||
|
|
nextRed >= COLOR_BINS_PER_CHANNEL ||
|
|
nextGreen < 0 ||
|
|
nextGreen >= COLOR_BINS_PER_CHANNEL ||
|
|
nextBlue < 0 ||
|
|
nextBlue >= COLOR_BINS_PER_CHANNEL
|
|
) {
|
|
continue;
|
|
}
|
|
const nextIndex =
|
|
nextRed * COLOR_BINS_PER_CHANNEL ** 2 +
|
|
nextGreen * COLOR_BINS_PER_CHANNEL +
|
|
nextBlue;
|
|
weight += profile.bins[nextIndex] || 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
return weight;
|
|
}
|
|
|
|
function compareColorProfiles(source: ColorProfile, candidate: ColorProfile) {
|
|
const sourceMinWeight = Math.max(2, source.total * 0.0015);
|
|
const candidateMinWeight = Math.max(2, candidate.total * 0.001);
|
|
let coveredSourceBins = 0;
|
|
let coveredSourceWeight = 0;
|
|
let matchedCandidateWeight = 0;
|
|
let salientSourceBins = 0;
|
|
let salientSourceWeight = 0;
|
|
|
|
for (let index = 0; index < COLOR_PROFILE_SIZE; index += 1) {
|
|
const sourceWeight = source.bins[index] || 0;
|
|
if (sourceWeight < sourceMinWeight) continue;
|
|
salientSourceBins += 1;
|
|
salientSourceWeight += sourceWeight;
|
|
if (getNeighborBinWeight(candidate, index) >= candidateMinWeight) {
|
|
coveredSourceBins += 1;
|
|
coveredSourceWeight += sourceWeight;
|
|
}
|
|
}
|
|
|
|
for (let index = 0; index < COLOR_PROFILE_SIZE; index += 1) {
|
|
const candidateWeight = candidate.bins[index] || 0;
|
|
if (candidateWeight < candidateMinWeight) continue;
|
|
if (getNeighborBinWeight(source, index) >= sourceMinWeight) {
|
|
matchedCandidateWeight += candidateWeight;
|
|
}
|
|
}
|
|
|
|
const distinctCoverage =
|
|
coveredSourceBins / Math.max(1, salientSourceBins);
|
|
const weightedCoverage =
|
|
coveredSourceWeight / Math.max(1, salientSourceWeight);
|
|
return {
|
|
colorCoverage: distinctCoverage * 0.6 + weightedCoverage * 0.4,
|
|
colorFidelity: matchedCandidateWeight / Math.max(1, candidate.total),
|
|
};
|
|
}
|
|
|
|
async function buildSourceColorProfile(sourceCropImageUrl: string) {
|
|
const { data, info } = await sharp(
|
|
await fetchImageBufferFromUrl(sourceCropImageUrl),
|
|
{ failOn: "none" }
|
|
)
|
|
.rotate()
|
|
.resize({
|
|
fit: "inside",
|
|
height: 256,
|
|
width: 256,
|
|
withoutEnlargement: true,
|
|
})
|
|
.removeAlpha()
|
|
.toColorspace("srgb")
|
|
.raw()
|
|
.toBuffer({ resolveWithObject: true });
|
|
return buildColorProfile({ channels: info.channels, data });
|
|
}
|
|
|
|
function rectToPixelBox(params: {
|
|
height: number;
|
|
region: CanvasLocalEditCompositeRegion;
|
|
width: number;
|
|
}) {
|
|
const { height, region, width } = params;
|
|
const left = clamp(Math.floor(region.x * width), 0, width - 1);
|
|
const top = clamp(Math.floor(region.y * height), 0, height - 1);
|
|
const right = clamp(
|
|
Math.ceil((region.x + region.width) * width),
|
|
left + 1,
|
|
width
|
|
);
|
|
const bottom = clamp(
|
|
Math.ceil((region.y + region.height) * height),
|
|
top + 1,
|
|
height
|
|
);
|
|
return { height: bottom - top, left, top, width: right - left };
|
|
}
|
|
|
|
async function extractMagentaProduct(candidateImageUrl: string) {
|
|
const candidateBuffer = await sharp(
|
|
await fetchImageBufferFromUrl(candidateImageUrl),
|
|
{ failOn: "none" }
|
|
)
|
|
.rotate()
|
|
.removeAlpha()
|
|
.toColorspace("srgb")
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
const { data, info } = await sharp(candidateBuffer)
|
|
.raw()
|
|
.toBuffer({ resolveWithObject: true });
|
|
const { height, width } = info;
|
|
const alpha = Buffer.alloc(width * height);
|
|
let backgroundPixels = 0;
|
|
let visiblePixels = 0;
|
|
let minX = width;
|
|
let minY = height;
|
|
let maxX = -1;
|
|
let maxY = -1;
|
|
|
|
for (let pixelIndex = 0; pixelIndex < alpha.length; pixelIndex += 1) {
|
|
const offset = pixelIndex * 3;
|
|
const redDistance = 255 - (data[offset] || 0);
|
|
const greenDistance = data[offset + 1] || 0;
|
|
const blueDistance = 255 - (data[offset + 2] || 0);
|
|
const distance = Math.sqrt(
|
|
redDistance * redDistance +
|
|
greenDistance * greenDistance +
|
|
blueDistance * blueDistance
|
|
);
|
|
let nextAlpha = 255;
|
|
if (distance <= MAGENTA_LOW_DISTANCE) {
|
|
nextAlpha = 0;
|
|
} else if (distance < MAGENTA_HIGH_DISTANCE) {
|
|
nextAlpha = Math.round(
|
|
255 *
|
|
smoothStep(
|
|
(distance - MAGENTA_LOW_DISTANCE) /
|
|
(MAGENTA_HIGH_DISTANCE - MAGENTA_LOW_DISTANCE)
|
|
)
|
|
);
|
|
}
|
|
alpha[pixelIndex] = nextAlpha;
|
|
const opacity = nextAlpha / 255;
|
|
if (nextAlpha < MAGENTA_DESPILL_MIN_ALPHA) {
|
|
data[offset] = 0;
|
|
data[offset + 1] = 0;
|
|
data[offset + 2] = 0;
|
|
} else if (nextAlpha < 255) {
|
|
data[offset] = clamp(
|
|
Math.round(((data[offset] || 0) - (1 - opacity) * 255) / opacity),
|
|
0,
|
|
255
|
|
);
|
|
data[offset + 1] = clamp(
|
|
Math.round((data[offset + 1] || 0) / opacity),
|
|
0,
|
|
255
|
|
);
|
|
data[offset + 2] = clamp(
|
|
Math.round(((data[offset + 2] || 0) - (1 - opacity) * 255) / opacity),
|
|
0,
|
|
255
|
|
);
|
|
}
|
|
if (nextAlpha < 24) backgroundPixels += 1;
|
|
if (nextAlpha < 18) continue;
|
|
visiblePixels += 1;
|
|
const x = pixelIndex % width;
|
|
const y = Math.floor(pixelIndex / width);
|
|
minX = Math.min(minX, x);
|
|
minY = Math.min(minY, y);
|
|
maxX = Math.max(maxX, x);
|
|
maxY = Math.max(maxY, y);
|
|
}
|
|
|
|
const pixelCount = width * height;
|
|
const backgroundRatio = backgroundPixels / Math.max(1, pixelCount);
|
|
const visibleRatio = visiblePixels / Math.max(1, pixelCount);
|
|
if (
|
|
backgroundRatio < 0.28 ||
|
|
visibleRatio < 0.015 ||
|
|
visibleRatio > 0.68 ||
|
|
maxX < minX ||
|
|
maxY < minY
|
|
) {
|
|
throw new Error("完整产品提取结果未生成有效纯色背景");
|
|
}
|
|
|
|
const padding = Math.max(2, Math.round(Math.min(width, height) * 0.008));
|
|
const left = Math.max(0, minX - padding);
|
|
const top = Math.max(0, minY - padding);
|
|
const right = Math.min(width, maxX + padding + 1);
|
|
const bottom = Math.min(height, maxY + padding + 1);
|
|
const softenedAlpha = await sharp(alpha, {
|
|
raw: { channels: 1, height, width },
|
|
})
|
|
.blur(0.35)
|
|
.linear(1.18, -24)
|
|
.extractChannel(0)
|
|
.raw()
|
|
.toBuffer();
|
|
const rgbaBuffer = await sharp(data, {
|
|
raw: { channels: 3, height, width },
|
|
})
|
|
.joinChannel(softenedAlpha, {
|
|
raw: { channels: 1, height, width },
|
|
})
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
const productBuffer = await sharp(rgbaBuffer)
|
|
.extract({
|
|
height: bottom - top,
|
|
left,
|
|
top,
|
|
width: right - left,
|
|
})
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
const touchesEdge =
|
|
minX <= 2 || minY <= 2 || maxX >= width - 3 || maxY >= height - 3;
|
|
const colorProfile = buildColorProfile({ alpha, channels: 3, data });
|
|
const heightSpan = (maxY - minY + 1) / height;
|
|
const widthSpan = (maxX - minX + 1) / width;
|
|
const geometryScore =
|
|
heightSpan * 180 +
|
|
widthSpan * 80 +
|
|
Math.min(1, visibleRatio / 0.3) * 50 -
|
|
(touchesEdge ? 120 : 0);
|
|
|
|
return { colorProfile, geometryScore, productBuffer, visiblePixels };
|
|
}
|
|
|
|
async function chooseProductCandidate(params: {
|
|
candidateImageUrls: string[];
|
|
sourceCropImageUrl: string;
|
|
}) {
|
|
const { candidateImageUrls, sourceCropImageUrl } = params;
|
|
const sourceColorProfile = await buildSourceColorProfile(sourceCropImageUrl);
|
|
const settled = await Promise.allSettled(
|
|
candidateImageUrls.map((candidateImageUrl) =>
|
|
extractMagentaProduct(candidateImageUrl)
|
|
)
|
|
);
|
|
const candidates = settled.flatMap((item, candidateIndex) =>
|
|
item.status === "fulfilled"
|
|
? [{ ...item.value, candidateIndex }]
|
|
: []
|
|
);
|
|
if (candidates.length === 0) {
|
|
throw new Error("比例调整无法提取完整产品组");
|
|
}
|
|
const scoredCandidates = candidates.map((candidate) => {
|
|
const { colorCoverage, colorFidelity } = compareColorProfiles(
|
|
sourceColorProfile,
|
|
candidate.colorProfile
|
|
);
|
|
return {
|
|
...candidate,
|
|
colorCoverage,
|
|
colorFidelity,
|
|
score:
|
|
candidate.geometryScore +
|
|
colorCoverage * 520 +
|
|
colorFidelity * 160,
|
|
};
|
|
});
|
|
scoredCandidates.sort((left, right) => right.score - left.score);
|
|
const selectedCandidate = scoredCandidates[0];
|
|
return {
|
|
candidate: selectedCandidate,
|
|
stats: scoredCandidates.map((candidate) => ({
|
|
candidateIndex: candidate.candidateIndex,
|
|
colorCoverage: Number(candidate.colorCoverage.toFixed(4)),
|
|
colorFidelity: Number(candidate.colorFidelity.toFixed(4)),
|
|
score: Math.round(candidate.score),
|
|
selected: candidate === selectedCandidate,
|
|
visiblePixels: candidate.visiblePixels,
|
|
})),
|
|
};
|
|
}
|
|
|
|
async function buildFallbackShadow(params: {
|
|
height: number;
|
|
overlayBuffer: Buffer;
|
|
width: number;
|
|
}) {
|
|
const { height, overlayBuffer, width } = params;
|
|
const alpha = await sharp(overlayBuffer, { failOn: "none" })
|
|
.resize(width, height, { fit: "fill" })
|
|
.ensureAlpha()
|
|
.extractChannel(3)
|
|
.blur(clamp(Math.min(width, height) * 0.0048, 3, 16))
|
|
.extractChannel(0)
|
|
.raw()
|
|
.toBuffer();
|
|
const shadowRgba = Buffer.alloc(width * height * 4);
|
|
const offsetX = Math.max(2, Math.round(width * 0.006));
|
|
const offsetY = Math.max(2, Math.round(height * 0.004));
|
|
|
|
for (let y = 0; y < height - offsetY; y += 1) {
|
|
for (let x = 0; x < width - offsetX; x += 1) {
|
|
const sourceIndex = y * width + x;
|
|
const targetIndex = (y + offsetY) * width + x + offsetX;
|
|
const rgbaIndex = targetIndex * 4;
|
|
shadowRgba[rgbaIndex] = 45;
|
|
shadowRgba[rgbaIndex + 1] = 39;
|
|
shadowRgba[rgbaIndex + 2] = 35;
|
|
shadowRgba[rgbaIndex + 3] = Math.min(
|
|
44,
|
|
Math.round((alpha[sourceIndex] || 0) * 0.17)
|
|
);
|
|
}
|
|
}
|
|
|
|
return sharp(shadowRgba, {
|
|
raw: { channels: 4, height, width },
|
|
})
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
}
|
|
|
|
export async function buildSemanticScaleAdjustProductComposite(params: {
|
|
backgroundImageUrl: string;
|
|
regions: SemanticProductRegion[];
|
|
sourceHeight: number;
|
|
sourceWidth: number;
|
|
}) {
|
|
const { backgroundImageUrl, regions, sourceHeight, sourceWidth } = params;
|
|
const overlayInputs: sharp.OverlayOptions[] = [];
|
|
const candidateStats: Array<{
|
|
candidateIndex: number;
|
|
colorCoverage: number;
|
|
colorFidelity: number;
|
|
regionIndex: number;
|
|
score: number;
|
|
selected: boolean;
|
|
visiblePixels: number;
|
|
}> = [];
|
|
|
|
for (const [regionIndex, region] of regions.entries()) {
|
|
const choice = await chooseProductCandidate({
|
|
candidateImageUrls: region.candidateImageUrls,
|
|
sourceCropImageUrl: region.sourceCropImageUrl,
|
|
});
|
|
const { candidate } = choice;
|
|
const targetBox = rectToPixelBox({
|
|
height: sourceHeight,
|
|
region: region.target,
|
|
width: sourceWidth,
|
|
});
|
|
const resizedProduct = await sharp(candidate.productBuffer, {
|
|
failOn: "none",
|
|
})
|
|
.resize(targetBox.width, targetBox.height, {
|
|
fit: "fill",
|
|
kernel: sharp.kernel.lanczos3,
|
|
})
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
overlayInputs.push({
|
|
input: resizedProduct,
|
|
left: targetBox.left,
|
|
top: targetBox.top,
|
|
});
|
|
candidateStats.push(
|
|
...choice.stats.map((stat) => ({ ...stat, regionIndex }))
|
|
);
|
|
}
|
|
|
|
const overlayBuffer = await sharp({
|
|
create: {
|
|
background: { alpha: 0, b: 0, g: 0, r: 0 },
|
|
channels: 4,
|
|
height: sourceHeight,
|
|
width: sourceWidth,
|
|
},
|
|
})
|
|
.composite(overlayInputs)
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
const fallbackShadowBuffer = await buildFallbackShadow({
|
|
height: sourceHeight,
|
|
overlayBuffer,
|
|
width: sourceWidth,
|
|
});
|
|
const backgroundBuffer = await sharp(
|
|
await fetchImageBufferFromUrl(backgroundImageUrl),
|
|
{ failOn: "none" }
|
|
)
|
|
.rotate()
|
|
.resize(sourceWidth, sourceHeight, { fit: "fill" })
|
|
.ensureAlpha()
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
const mergedBuffer = await sharp(backgroundBuffer)
|
|
.composite([
|
|
{ input: fallbackShadowBuffer, left: 0, top: 0 },
|
|
{ input: overlayBuffer, left: 0, top: 0 },
|
|
])
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
|
|
return { candidateStats, mergedBuffer, overlayBuffer };
|
|
}
|