import "server-only"; import sharp from "sharp"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import type { CanvasGenerateImageRequest, CanvasGenerateImageResult, CanvasLocalEditCompositeRegion, CanvasScaleAdjustModelRelightConfig, } from "@/lib/canvas/generate-image"; import { uploadFile } from "@/lib/storage/service"; type GenerateCanvasImage = ( request: CanvasGenerateImageRequest ) => Promise; function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } function bufferToArrayBuffer(buffer: Buffer) { return buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; } function rectToPixelBox(params: { height: number; region: CanvasLocalEditCompositeRegion; width: number; }) { const { height, region, width } = params; const left = clamp(Math.floor(region.x * width), 0, Math.max(0, width - 1)); const top = clamp(Math.floor(region.y * height), 0, Math.max(0, height - 1)); const right = clamp( Math.ceil((region.x + region.width) * width), left + 1, width ); const bottom = clamp( Math.ceil((region.y + region.height) * height), top + 1, height ); return { bottom, left, right, top }; } async function buildModelRelightMask(params: { config: CanvasScaleAdjustModelRelightConfig; height: number; width: number; }) { const { config, height, width } = params; const productAlpha = await sharp( await fetchImageBufferFromUrl(config.overlayImageUrl), { failOn: "none" } ) .rotate() .resize(width, height, { fit: "fill", kernel: sharp.kernel.lanczos3 }) .ensureAlpha() .extractChannel(3) .raw() .toBuffer(); const editableMask = Buffer.alloc(width * height); for (const item of config.regions) { const target = item.target; const expandedFloorRegion = config.surfaceMode === "surround" ? { x: target.x - target.width * 0.22, y: target.y - target.height * 0.08, width: target.width * 1.52, height: target.height * 1.22, } : { x: target.x - target.width * 0.16, y: target.y + target.height * 0.68, width: target.width * 1.32, height: target.height * 0.5, }; const box = rectToPixelBox({ height, region: expandedFloorRegion, width, }); for (let y = box.top; y < box.bottom; y += 1) { editableMask.fill(255, y * width + box.left, y * width + box.right); } } for (let index = 0; index < editableMask.length; index += 1) { if (productAlpha[index] >= 8) editableMask[index] = 0; } const maskBuffer = await sharp(editableMask, { raw: { channels: 1, height, width }, }) .png({ compressionLevel: 6 }) .toBuffer(); return { editableMask, maskBuffer, productAlpha }; } function findMedianLumaDelta(params: { editableMask: Buffer; exactRaw: Buffer; generatedRaw: Buffer; }) { const { editableMask, exactRaw, generatedRaw } = params; const histogram = new Uint32Array(511); let sampleCount = 0; for ( let maskIndex = 0, rgbIndex = 0; maskIndex < editableMask.length; maskIndex += 1, rgbIndex += 3 ) { if (editableMask[maskIndex] < 240) continue; const exactLuma = (77 * exactRaw[rgbIndex] + 150 * exactRaw[rgbIndex + 1] + 29 * exactRaw[rgbIndex + 2]) >> 8; const generatedLuma = (77 * generatedRaw[rgbIndex] + 150 * generatedRaw[rgbIndex + 1] + 29 * generatedRaw[rgbIndex + 2]) >> 8; const bucket = clamp(exactLuma - generatedLuma + 255, 0, 510); histogram[bucket] += 1; sampleCount += 1; } if (sampleCount === 0) return { median: 0, sampleCount }; let seen = 0; for (let index = 0; index < histogram.length; index += 1) { seen += histogram[index]; if (seen >= sampleCount / 2) { return { median: index - 255, sampleCount }; } } return { median: 0, sampleCount }; } async function mergeModelShadowField(params: { editableMask: Buffer; exactBuffer: Buffer; generatedBuffer: Buffer; height: number; productAlpha: Buffer; width: number; }) { const { editableMask, exactBuffer, generatedBuffer, height, productAlpha, width, } = params; const [exactRaw, generatedRaw] = await Promise.all([ sharp(exactBuffer, { failOn: "none" }) .resize(width, height, { fit: "fill" }) .removeAlpha() .toColorspace("srgb") .raw() .toBuffer(), sharp(generatedBuffer, { failOn: "none" }) .rotate() .resize(width, height, { fit: "fill", kernel: sharp.kernel.lanczos3 }) .removeAlpha() .toColorspace("srgb") .raw() .toBuffer(), ]); const { median, sampleCount } = findMedianLumaDelta({ editableMask, exactRaw, generatedRaw, }); if (sampleCount < 128) return null; const shadowAlpha = Buffer.alloc(width * height); let rawShadowStrength = 0; for ( let maskIndex = 0, rgbIndex = 0; maskIndex < editableMask.length; maskIndex += 1, rgbIndex += 3 ) { if (editableMask[maskIndex] < 240) continue; const exactLuma = (77 * exactRaw[rgbIndex] + 150 * exactRaw[rgbIndex + 1] + 29 * exactRaw[rgbIndex + 2]) >> 8; const generatedLuma = (77 * generatedRaw[rgbIndex] + 150 * generatedRaw[rgbIndex + 1] + 29 * generatedRaw[rgbIndex + 2]) >> 8; const modelDarkness = Math.max( 0, exactLuma - generatedLuma - median - 12 ); const alpha = Math.min(58, Math.round(modelDarkness * 0.78)); shadowAlpha[maskIndex] = alpha; rawShadowStrength += alpha; } if (rawShadowStrength < 96) return null; const blurSigma = clamp(Math.min(width, height) * 0.0063, 6, 24); const softenedAlpha = await sharp(shadowAlpha, { raw: { channels: 1, height, width }, }) .blur(blurSigma) .extractChannel(0) .raw() .toBuffer(); const shadowLayer = Buffer.alloc(width * height * 4); let appliedShadowStrength = 0; for ( let pixelIndex = 0, rgbaIndex = 0; pixelIndex < softenedAlpha.length; pixelIndex += 1, rgbaIndex += 4 ) { const alpha = productAlpha[pixelIndex] >= 8 ? 0 : softenedAlpha[pixelIndex]; shadowLayer[rgbaIndex] = 32; shadowLayer[rgbaIndex + 1] = 32; shadowLayer[rgbaIndex + 2] = 32; shadowLayer[rgbaIndex + 3] = alpha; appliedShadowStrength += alpha; } if (appliedShadowStrength < 96) return null; const mergedBuffer = await sharp(exactBuffer, { failOn: "none" }) .resize(width, height, { fit: "fill" }) .ensureAlpha() .composite([ { input: shadowLayer, raw: { channels: 4, height, width }, blend: "over", }, ]) .png({ compressionLevel: 6 }) .toBuffer(); return { appliedShadowStrength, median, mergedBuffer, sampleCount }; } export async function applyScaleAdjustModelRelight(params: { generateImage: GenerateCanvasImage; initialResult: CanvasGenerateImageResult; ownerKey?: string; request: CanvasGenerateImageRequest; }) { const { generateImage, initialResult, ownerKey, request } = params; const config = request.scaleAdjustModelRelight; if (!config || initialResult.images.length === 0) return initialResult; const relitImages = [] as CanvasGenerateImageResult["images"]; for (const [index, image] of initialResult.images.entries()) { const exactUrl = image.originalUrl || image.thumbnailUrl; try { const exactBuffer = await fetchImageBufferFromUrl(exactUrl); const metadata = await sharp(exactBuffer, { failOn: "none" }).metadata(); const width = Math.round(Number(metadata.width || 0)); const height = Math.round(Number(metadata.height || 0)); if (width <= 0 || height <= 0) throw new Error("精确合成结果尺寸无效"); const { editableMask, maskBuffer, productAlpha } = await buildModelRelightMask({ config, height, width }); const suffix = `${Date.now()}-${index + 1}-${Math.random() .toString(36) .slice(2, 8)}`; const maskUpload = await uploadFile( `canvas-scale-adjust-model-relight-mask-${suffix}.png`, bufferToArrayBuffer(maskBuffer) ); const modelResult = await generateImage({ allowFailover: false, aspectRatio: "auto", debugLabel: "canvas-scale-adjust-model-relight", debugMeta: { operation: "scale-adjust-model-relight", scaleAdjustModelRelightApplied: true, scaleAdjustProviderMaskImageUrl: maskUpload.url, sourceHeight: config.sourceHeight, sourceWidth: config.sourceWidth, }, displayPrompt: request.displayPrompt, imageCount: 1, imageSize: "auto", model: config.model, negativePrompt: "复制产品、增加产品、移动产品、改变产品造型、改变构图、修改人物、文字、标注、边框", ownerKey, prompt: config.surfaceMode === "surround" ? "仅编辑白色遮罩中的产品周围表面。根据画面现有主光方向、环境光颜色、产品与墙面或地面的接触关系,为当前位置的产品生成真实自然的接触阴影与投射阴影。阴影方向、软硬、颜色和强度必须与室内现有光照一致。黑色区域保持像素级不变;不要重绘、复制、移动或变形产品,不要修改人物、家具、墙面纹理和构图。" : "仅编辑白色遮罩中的地面区域。根据画面现有主光方向、环境光颜色、产品落地点和地面材质,为当前位置的产品生成真实自然的接触阴影与投射阴影。阴影必须贴合产品落地边缘,方向、软硬、颜色和强度与室内现有光照一致。黑色区域保持像素级不变;不要重绘、复制、移动或变形产品,不要修改人物、家具、墙面和构图。", promptExtend: false, providerImageParams: { maskImageUrl: maskUpload.url, quality: config.quality || "medium", }, referenceImages: [exactUrl], }); const generatedUrl = modelResult.images[0]?.originalUrl || modelResult.images[0]?.thumbnailUrl || ""; if (!generatedUrl) throw new Error("光影模型未返回图片"); const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); const merged = await mergeModelShadowField({ editableMask, exactBuffer, generatedBuffer, height, productAlpha, width, }); if (!merged) throw new Error("光影模型未生成有效的地面明暗变化"); const finalJpegBuffer = await sharp(merged.mergedBuffer, { failOn: "none", }) .flatten({ background: "#ffffff" }) .jpeg({ quality: 94, mozjpeg: true }) .toBuffer(); const thumbnailBuffer = await sharp(merged.mergedBuffer, { failOn: "none", }) .flatten({ background: "#ffffff" }) .resize(512, 512, { fit: "inside", withoutEnlargement: true }) .jpeg({ quality: 88, mozjpeg: true }) .toBuffer(); const [originalUpload, thumbnailUpload] = await Promise.all([ uploadFile( `canvas-scale-adjust-model-relit-${suffix}.jpg`, bufferToArrayBuffer(finalJpegBuffer) ), uploadFile( `canvas-scale-adjust-model-relit-thumb-${suffix}.jpg`, bufferToArrayBuffer(thumbnailBuffer) ), ]); console.info("[Canvas Scale Adjust] Model relight applied:", { appliedShadowStrength: merged.appliedShadowStrength, exactUrl, generatedUrl, height, medianLumaDelta: merged.median, sampleCount: merged.sampleCount, width, }); relitImages.push({ originalUrl: originalUpload.url, thumbnailUrl: thumbnailUpload.url, }); } catch (error) { console.error( "[Canvas Scale Adjust] Model relight failed; keeping verified local shadow fallback:", { error: error instanceof Error ? error.message : String(error), exactUrl, index, } ); relitImages.push(image); } } return { ...initialResult, images: relitImages, }; }