379 lines
12 KiB
TypeScript
379 lines
12 KiB
TypeScript
import "server-only";
|
|
|
|
import sharp from "sharp";
|
|
|
|
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
|
import type {
|
|
CanvasGenerateImageRequest,
|
|
CanvasGenerateImageResult,
|
|
CanvasLocalEditCompositeRegion,
|
|
CanvasScaleAdjustModelRelightConfig,
|
|
} from "@/lib/canvas/generate-image";
|
|
import { uploadFile } from "@/lib/storage/service";
|
|
|
|
type GenerateCanvasImage = (
|
|
request: CanvasGenerateImageRequest
|
|
) => Promise<CanvasGenerateImageResult>;
|
|
|
|
function clamp(value: number, min: number, max: number) {
|
|
return Math.min(max, Math.max(min, value));
|
|
}
|
|
|
|
function bufferToArrayBuffer(buffer: Buffer) {
|
|
return buffer.buffer.slice(
|
|
buffer.byteOffset,
|
|
buffer.byteOffset + buffer.byteLength
|
|
) as ArrayBuffer;
|
|
}
|
|
|
|
function rectToPixelBox(params: {
|
|
height: number;
|
|
region: CanvasLocalEditCompositeRegion;
|
|
width: number;
|
|
}) {
|
|
const { height, region, width } = params;
|
|
const left = clamp(Math.floor(region.x * width), 0, Math.max(0, width - 1));
|
|
const top = clamp(Math.floor(region.y * height), 0, Math.max(0, height - 1));
|
|
const right = clamp(
|
|
Math.ceil((region.x + region.width) * width),
|
|
left + 1,
|
|
width
|
|
);
|
|
const bottom = clamp(
|
|
Math.ceil((region.y + region.height) * height),
|
|
top + 1,
|
|
height
|
|
);
|
|
return { bottom, left, right, top };
|
|
}
|
|
|
|
async function buildModelRelightMask(params: {
|
|
config: CanvasScaleAdjustModelRelightConfig;
|
|
height: number;
|
|
width: number;
|
|
}) {
|
|
const { config, height, width } = params;
|
|
const productAlpha = await sharp(
|
|
await fetchImageBufferFromUrl(config.overlayImageUrl),
|
|
{ failOn: "none" }
|
|
)
|
|
.rotate()
|
|
.resize(width, height, { fit: "fill", kernel: sharp.kernel.lanczos3 })
|
|
.ensureAlpha()
|
|
.extractChannel(3)
|
|
.raw()
|
|
.toBuffer();
|
|
const editableMask = Buffer.alloc(width * height);
|
|
|
|
for (const item of config.regions) {
|
|
const target = item.target;
|
|
const expandedFloorRegion =
|
|
config.surfaceMode === "surround"
|
|
? {
|
|
x: target.x - target.width * 0.22,
|
|
y: target.y - target.height * 0.08,
|
|
width: target.width * 1.52,
|
|
height: target.height * 1.22,
|
|
}
|
|
: {
|
|
x: target.x - target.width * 0.16,
|
|
y: target.y + target.height * 0.68,
|
|
width: target.width * 1.32,
|
|
height: target.height * 0.5,
|
|
};
|
|
const box = rectToPixelBox({
|
|
height,
|
|
region: expandedFloorRegion,
|
|
width,
|
|
});
|
|
for (let y = box.top; y < box.bottom; y += 1) {
|
|
editableMask.fill(255, y * width + box.left, y * width + box.right);
|
|
}
|
|
}
|
|
|
|
for (let index = 0; index < editableMask.length; index += 1) {
|
|
if (productAlpha[index] >= 8) editableMask[index] = 0;
|
|
}
|
|
|
|
const maskBuffer = await sharp(editableMask, {
|
|
raw: { channels: 1, height, width },
|
|
})
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
return { editableMask, maskBuffer, productAlpha };
|
|
}
|
|
|
|
function findMedianLumaDelta(params: {
|
|
editableMask: Buffer;
|
|
exactRaw: Buffer;
|
|
generatedRaw: Buffer;
|
|
}) {
|
|
const { editableMask, exactRaw, generatedRaw } = params;
|
|
const histogram = new Uint32Array(511);
|
|
let sampleCount = 0;
|
|
for (
|
|
let maskIndex = 0, rgbIndex = 0;
|
|
maskIndex < editableMask.length;
|
|
maskIndex += 1, rgbIndex += 3
|
|
) {
|
|
if (editableMask[maskIndex] < 240) continue;
|
|
const exactLuma =
|
|
(77 * exactRaw[rgbIndex] +
|
|
150 * exactRaw[rgbIndex + 1] +
|
|
29 * exactRaw[rgbIndex + 2]) >>
|
|
8;
|
|
const generatedLuma =
|
|
(77 * generatedRaw[rgbIndex] +
|
|
150 * generatedRaw[rgbIndex + 1] +
|
|
29 * generatedRaw[rgbIndex + 2]) >>
|
|
8;
|
|
const bucket = clamp(exactLuma - generatedLuma + 255, 0, 510);
|
|
histogram[bucket] += 1;
|
|
sampleCount += 1;
|
|
}
|
|
|
|
if (sampleCount === 0) return { median: 0, sampleCount };
|
|
let seen = 0;
|
|
for (let index = 0; index < histogram.length; index += 1) {
|
|
seen += histogram[index];
|
|
if (seen >= sampleCount / 2) {
|
|
return { median: index - 255, sampleCount };
|
|
}
|
|
}
|
|
return { median: 0, sampleCount };
|
|
}
|
|
|
|
async function mergeModelShadowField(params: {
|
|
editableMask: Buffer;
|
|
exactBuffer: Buffer;
|
|
generatedBuffer: Buffer;
|
|
height: number;
|
|
productAlpha: Buffer;
|
|
width: number;
|
|
}) {
|
|
const {
|
|
editableMask,
|
|
exactBuffer,
|
|
generatedBuffer,
|
|
height,
|
|
productAlpha,
|
|
width,
|
|
} = params;
|
|
const [exactRaw, generatedRaw] = await Promise.all([
|
|
sharp(exactBuffer, { failOn: "none" })
|
|
.resize(width, height, { fit: "fill" })
|
|
.removeAlpha()
|
|
.toColorspace("srgb")
|
|
.raw()
|
|
.toBuffer(),
|
|
sharp(generatedBuffer, { failOn: "none" })
|
|
.rotate()
|
|
.resize(width, height, { fit: "fill", kernel: sharp.kernel.lanczos3 })
|
|
.removeAlpha()
|
|
.toColorspace("srgb")
|
|
.raw()
|
|
.toBuffer(),
|
|
]);
|
|
const { median, sampleCount } = findMedianLumaDelta({
|
|
editableMask,
|
|
exactRaw,
|
|
generatedRaw,
|
|
});
|
|
if (sampleCount < 128) return null;
|
|
|
|
const shadowAlpha = Buffer.alloc(width * height);
|
|
let rawShadowStrength = 0;
|
|
for (
|
|
let maskIndex = 0, rgbIndex = 0;
|
|
maskIndex < editableMask.length;
|
|
maskIndex += 1, rgbIndex += 3
|
|
) {
|
|
if (editableMask[maskIndex] < 240) continue;
|
|
const exactLuma =
|
|
(77 * exactRaw[rgbIndex] +
|
|
150 * exactRaw[rgbIndex + 1] +
|
|
29 * exactRaw[rgbIndex + 2]) >>
|
|
8;
|
|
const generatedLuma =
|
|
(77 * generatedRaw[rgbIndex] +
|
|
150 * generatedRaw[rgbIndex + 1] +
|
|
29 * generatedRaw[rgbIndex + 2]) >>
|
|
8;
|
|
const modelDarkness = Math.max(
|
|
0,
|
|
exactLuma - generatedLuma - median - 12
|
|
);
|
|
const alpha = Math.min(58, Math.round(modelDarkness * 0.78));
|
|
shadowAlpha[maskIndex] = alpha;
|
|
rawShadowStrength += alpha;
|
|
}
|
|
if (rawShadowStrength < 96) return null;
|
|
|
|
const blurSigma = clamp(Math.min(width, height) * 0.0063, 6, 24);
|
|
const softenedAlpha = await sharp(shadowAlpha, {
|
|
raw: { channels: 1, height, width },
|
|
})
|
|
.blur(blurSigma)
|
|
.extractChannel(0)
|
|
.raw()
|
|
.toBuffer();
|
|
const shadowLayer = Buffer.alloc(width * height * 4);
|
|
let appliedShadowStrength = 0;
|
|
for (
|
|
let pixelIndex = 0, rgbaIndex = 0;
|
|
pixelIndex < softenedAlpha.length;
|
|
pixelIndex += 1, rgbaIndex += 4
|
|
) {
|
|
const alpha = productAlpha[pixelIndex] >= 8 ? 0 : softenedAlpha[pixelIndex];
|
|
shadowLayer[rgbaIndex] = 32;
|
|
shadowLayer[rgbaIndex + 1] = 32;
|
|
shadowLayer[rgbaIndex + 2] = 32;
|
|
shadowLayer[rgbaIndex + 3] = alpha;
|
|
appliedShadowStrength += alpha;
|
|
}
|
|
if (appliedShadowStrength < 96) return null;
|
|
|
|
const mergedBuffer = await sharp(exactBuffer, { failOn: "none" })
|
|
.resize(width, height, { fit: "fill" })
|
|
.ensureAlpha()
|
|
.composite([
|
|
{
|
|
input: shadowLayer,
|
|
raw: { channels: 4, height, width },
|
|
blend: "over",
|
|
},
|
|
])
|
|
.png({ compressionLevel: 6 })
|
|
.toBuffer();
|
|
return { appliedShadowStrength, median, mergedBuffer, sampleCount };
|
|
}
|
|
|
|
export async function applyScaleAdjustModelRelight(params: {
|
|
generateImage: GenerateCanvasImage;
|
|
initialResult: CanvasGenerateImageResult;
|
|
ownerKey?: string;
|
|
request: CanvasGenerateImageRequest;
|
|
}) {
|
|
const { generateImage, initialResult, ownerKey, request } = params;
|
|
const config = request.scaleAdjustModelRelight;
|
|
if (!config || initialResult.images.length === 0) return initialResult;
|
|
|
|
const relitImages = [] as CanvasGenerateImageResult["images"];
|
|
for (const [index, image] of initialResult.images.entries()) {
|
|
const exactUrl = image.originalUrl || image.thumbnailUrl;
|
|
try {
|
|
const exactBuffer = await fetchImageBufferFromUrl(exactUrl);
|
|
const metadata = await sharp(exactBuffer, { failOn: "none" }).metadata();
|
|
const width = Math.round(Number(metadata.width || 0));
|
|
const height = Math.round(Number(metadata.height || 0));
|
|
if (width <= 0 || height <= 0) throw new Error("精确合成结果尺寸无效");
|
|
|
|
const { editableMask, maskBuffer, productAlpha } =
|
|
await buildModelRelightMask({ config, height, width });
|
|
const suffix = `${Date.now()}-${index + 1}-${Math.random()
|
|
.toString(36)
|
|
.slice(2, 8)}`;
|
|
const maskUpload = await uploadFile(
|
|
`canvas-scale-adjust-model-relight-mask-${suffix}.png`,
|
|
bufferToArrayBuffer(maskBuffer)
|
|
);
|
|
const modelResult = await generateImage({
|
|
allowFailover: false,
|
|
aspectRatio: "auto",
|
|
debugLabel: "canvas-scale-adjust-model-relight",
|
|
debugMeta: {
|
|
operation: "scale-adjust-model-relight",
|
|
scaleAdjustModelRelightApplied: true,
|
|
scaleAdjustProviderMaskImageUrl: maskUpload.url,
|
|
sourceHeight: config.sourceHeight,
|
|
sourceWidth: config.sourceWidth,
|
|
},
|
|
displayPrompt: request.displayPrompt,
|
|
imageCount: 1,
|
|
imageSize: "auto",
|
|
model: config.model,
|
|
negativePrompt:
|
|
"复制产品、增加产品、移动产品、改变产品造型、改变构图、修改人物、文字、标注、边框",
|
|
ownerKey,
|
|
prompt:
|
|
config.surfaceMode === "surround"
|
|
? "仅编辑白色遮罩中的产品周围表面。根据画面现有主光方向、环境光颜色、产品与墙面或地面的接触关系，为当前位置的产品生成真实自然的接触阴影与投射阴影。阴影方向、软硬、颜色和强度必须与室内现有光照一致。黑色区域保持像素级不变；不要重绘、复制、移动或变形产品，不要修改人物、家具、墙面纹理和构图。"
|
|
: "仅编辑白色遮罩中的地面区域。根据画面现有主光方向、环境光颜色、产品落地点和地面材质，为当前位置的产品生成真实自然的接触阴影与投射阴影。阴影必须贴合产品落地边缘，方向、软硬、颜色和强度与室内现有光照一致。黑色区域保持像素级不变；不要重绘、复制、移动或变形产品，不要修改人物、家具、墙面和构图。",
|
|
promptExtend: false,
|
|
providerImageParams: {
|
|
maskImageUrl: maskUpload.url,
|
|
quality: config.quality || "medium",
|
|
},
|
|
referenceImages: [exactUrl],
|
|
});
|
|
const generatedUrl =
|
|
modelResult.images[0]?.originalUrl ||
|
|
modelResult.images[0]?.thumbnailUrl ||
|
|
"";
|
|
if (!generatedUrl) throw new Error("光影模型未返回图片");
|
|
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
|
const merged = await mergeModelShadowField({
|
|
editableMask,
|
|
exactBuffer,
|
|
generatedBuffer,
|
|
height,
|
|
productAlpha,
|
|
width,
|
|
});
|
|
if (!merged) throw new Error("光影模型未生成有效的地面明暗变化");
|
|
|
|
const finalJpegBuffer = await sharp(merged.mergedBuffer, {
|
|
failOn: "none",
|
|
})
|
|
.flatten({ background: "#ffffff" })
|
|
.jpeg({ quality: 94, mozjpeg: true })
|
|
.toBuffer();
|
|
const thumbnailBuffer = await sharp(merged.mergedBuffer, {
|
|
failOn: "none",
|
|
})
|
|
.flatten({ background: "#ffffff" })
|
|
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
|
|
.jpeg({ quality: 88, mozjpeg: true })
|
|
.toBuffer();
|
|
const [originalUpload, thumbnailUpload] = await Promise.all([
|
|
uploadFile(
|
|
`canvas-scale-adjust-model-relit-${suffix}.jpg`,
|
|
bufferToArrayBuffer(finalJpegBuffer)
|
|
),
|
|
uploadFile(
|
|
`canvas-scale-adjust-model-relit-thumb-${suffix}.jpg`,
|
|
bufferToArrayBuffer(thumbnailBuffer)
|
|
),
|
|
]);
|
|
console.info("[Canvas Scale Adjust] Model relight applied:", {
|
|
appliedShadowStrength: merged.appliedShadowStrength,
|
|
exactUrl,
|
|
generatedUrl,
|
|
height,
|
|
medianLumaDelta: merged.median,
|
|
sampleCount: merged.sampleCount,
|
|
width,
|
|
});
|
|
relitImages.push({
|
|
originalUrl: originalUpload.url,
|
|
thumbnailUrl: thumbnailUpload.url,
|
|
});
|
|
} catch (error) {
|
|
console.error(
|
|
"[Canvas Scale Adjust] Model relight failed; keeping verified local shadow fallback:",
|
|
{
|
|
error: error instanceof Error ? error.message : String(error),
|
|
exactUrl,
|
|
index,
|
|
}
|
|
);
|
|
relitImages.push(image);
|
|
}
|
|
}
|
|
|
|
return {
|
|
...initialResult,
|
|
images: relitImages,
|
|
};
|
|
}
|