import "server-only"; import { analyzeReplacementExecutionPlan, buildAgentReplacementPrompt, createAgentImageThumbnailBuffer, normalizeReplacementResultImage, resolveDirectReplacementModelRef, resolveReplacementReferenceGeometry, resolveReplacementScenePlanningRuntime, type AgentDetailReplacementStrategy, } from "@/app/(chat)/api/chat/_lib/detail-replacement"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; import { generateCanvasImage, type CanvasGenerateImageResult, } from "@/lib/canvas/generate-image"; import { uploadFile } from "@/lib/storage/service"; export interface CanvasLayoutReplaceTaskRequest { taskKind: "canvas-layout-replace"; sourceImageUrl: string; referenceImageUrl: string; supplementalPrompt?: string; model?: string; imageSize?: string; sourceNodeId?: string; referenceLabel?: string; debugMeta?: Record; } function normalizeHttpUrl(value: unknown) { const url = String(value || "").trim(); return /^https?:\/\//i.test(url) ? url : ""; } export function isCanvasLayoutReplaceTaskRequest( request: unknown, ): request is CanvasLayoutReplaceTaskRequest { return Boolean( request && typeof request === "object" && !Array.isArray(request) && (request as { taskKind?: unknown }).taskKind === "canvas-layout-replace", ); } function normalizeLayoutReplacePromptSupplement(value: unknown) { return String(value || "") .trim() .replace(/\s+/g, " ") .slice(0, 600); } function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer { return buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength, ) as ArrayBuffer; } function selectLayoutPatchMask(params: { generated: Buffer; height: number; reference: Buffer; width: number; }) { const { generated, height, reference, width } = params; const pixelCount = width * height; const rawMask = new Uint8Array(pixelCount); const diffThreshold = 34; for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) { const offset = pixelIndex * 3; const diff = Math.max( Math.abs((generated[offset] || 0) - (reference[offset] || 0)), Math.abs((generated[offset + 1] || 0) - (reference[offset + 1] || 0)), Math.abs((generated[offset + 2] || 0) - (reference[offset + 2] || 0)), ); if (diff >= diffThreshold) { rawMask[pixelIndex] = 1; } } const visited = new Uint8Array(pixelCount); const queue = new Int32Array(pixelCount); const components: Array<{ area: number; maxX: number; maxY: number; minX: number; minY: number; }> = []; for (let start = 0; start < pixelCount; start += 1) { if (!rawMask[start] || visited[start]) continue; let head = 0; let tail = 0; let area = 0; let minX = width; let minY = height; let maxX = 0; let maxY = 0; queue[tail] = start; tail += 1; visited[start] = 1; while (head < tail) { const current = queue[head]; head += 1; area += 1; const x = current % width; const y = Math.floor(current / width); if (x < minX) minX = x; if (x > maxX) maxX = x; if (y < minY) minY = y; if (y > maxY) maxY = y; const left = current - 1; const right = current + 1; const up = current - width; const down = current + width; if (x > 0 && rawMask[left] && !visited[left]) { visited[left] = 1; queue[tail] = left; tail += 1; } if (x < width - 1 && rawMask[right] && !visited[right]) { visited[right] = 1; queue[tail] = right; tail += 1; } if (y > 0 && rawMask[up] && !visited[up]) { visited[up] = 1; queue[tail] = up; tail += 1; } if (y < height - 1 && rawMask[down] && !visited[down]) { visited[down] = 1; queue[tail] = down; tail += 1; } } components.push({ area, maxX, maxY, minX, minY }); } const minArea = Math.max(900, Math.round(pixelCount * 0.0015)); const minBoxArea = Math.max(3200, Math.round(pixelCount * 0.008)); const candidates = components .filter((component) => { const boxArea = (component.maxX - component.minX + 1) * (component.maxY - component.minY + 1); return component.area >= minArea && boxArea >= minBoxArea; }) .sort((left, right) => right.area - left.area); const primary = candidates[0]; if (!primary) { return null; } const selected = candidates.filter((component, index) => { if (index === 0) return true; if (component.area >= primary.area * 0.18) return true; const expandedPrimary = { maxX: Math.min(width - 1, primary.maxX + Math.round(width * 0.05)), maxY: Math.min(height - 1, primary.maxY + Math.round(height * 0.05)), minX: Math.max(0, primary.minX - Math.round(width * 0.05)), minY: Math.max(0, primary.minY - Math.round(height * 0.05)), }; return !( component.maxX < expandedPrimary.minX || component.minX > expandedPrimary.maxX || component.maxY < expandedPrimary.minY || component.minY > expandedPrimary.maxY ); }); const selectedMask = new Uint8Array(pixelCount); for (const component of selected) { for (let y = component.minY; y <= component.maxY; y += 1) { const rowOffset = y * width; for (let x = component.minX; x <= component.maxX; x += 1) { const index = rowOffset + x; if (rawMask[index]) selectedMask[index] = 255; } } } return { mask: selectedMask, maskSource: "generated-diff", selectedComponentCount: selected.length, selectedPixelCount: selected.reduce((sum, component) => sum + component.area, 0), }; } function selectReferenceProductMask(params: { height: number; reference: Buffer; width: number; }) { const { height, reference, width } = params; const pixelCount = width * height; const rawMask = new Uint8Array(pixelCount); for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) { const offset = pixelIndex * 3; const red = reference[offset] || 0; const green = reference[offset + 1] || 0; const blue = reference[offset + 2] || 0; const luminance = 0.2126 * red + 0.7152 * green + 0.0722 * blue; const channelSpan = Math.max(red, green, blue) - Math.min(red, green, blue); if (luminance <= 222 || channelSpan >= 38) { rawMask[pixelIndex] = 1; } } const visited = new Uint8Array(pixelCount); const queue = new Int32Array(pixelCount); const components: Array<{ area: number; boxArea: number; maxX: number; maxY: number; minX: number; minY: number; }> = []; for (let start = 0; start < pixelCount; start += 1) { if (!rawMask[start] || visited[start]) continue; let head = 0; let tail = 0; let area = 0; let minX = width; let minY = height; let maxX = 0; let maxY = 0; queue[tail] = start; tail += 1; visited[start] = 1; while (head < tail) { const current = queue[head]; head += 1; area += 1; const x = current % width; const y = Math.floor(current / width); if (x < minX) minX = x; if (x > maxX) maxX = x; if (y < minY) minY = y; if (y > maxY) maxY = y; const left = current - 1; const right = current + 1; const up = current - width; const down = current + width; if (x > 0 && rawMask[left] && !visited[left]) { visited[left] = 1; queue[tail] = left; tail += 1; } if (x < width - 1 && rawMask[right] && !visited[right]) { visited[right] = 1; queue[tail] = right; tail += 1; } if (y > 0 && rawMask[up] && !visited[up]) { visited[up] = 1; queue[tail] = up; tail += 1; } if (y < height - 1 && rawMask[down] && !visited[down]) { visited[down] = 1; queue[tail] = down; tail += 1; } } const boxArea = (maxX - minX + 1) * (maxY - minY + 1); components.push({ area, boxArea, maxX, maxY, minX, minY }); } const minArea = Math.max(1400, Math.round(pixelCount * 0.002)); const minBoxArea = Math.max(6000, Math.round(pixelCount * 0.012)); const candidates = components .filter((component) => { const boxWidth = component.maxX - component.minX + 1; const boxHeight = component.maxY - component.minY + 1; const aspect = boxWidth / Math.max(1, boxHeight); const centerY = (component.minY + component.maxY) / 2; const looksLikeCtaButton = aspect >= 6 && boxHeight <= height * 0.08; const looksLikeBottomUtility = centerY >= height * 0.78 && aspect >= 3.5; return ( component.area >= minArea && component.boxArea >= minBoxArea && !looksLikeCtaButton && !looksLikeBottomUtility ); }) .sort((left, right) => right.area - left.area); const primary = candidates[0]; if (!primary) { return null; } const selected = candidates.filter((component, index) => { if (index === 0) return true; if (component.area >= primary.area * 0.22) return true; const expandedPrimary = { maxX: Math.min(width - 1, primary.maxX + Math.round(width * 0.08)), maxY: Math.min(height - 1, primary.maxY + Math.round(height * 0.08)), minX: Math.max(0, primary.minX - Math.round(width * 0.08)), minY: Math.max(0, primary.minY - Math.round(height * 0.08)), }; return !( component.maxX < expandedPrimary.minX || component.minX > expandedPrimary.maxX || component.maxY < expandedPrimary.minY || component.minY > expandedPrimary.maxY ); }); const selectedMask = new Uint8Array(pixelCount); for (const component of selected) { for (let y = component.minY; y <= component.maxY; y += 1) { const rowOffset = y * width; for (let x = component.minX; x <= component.maxX; x += 1) { const index = rowOffset + x; if (rawMask[index]) selectedMask[index] = 255; } } } return { mask: selectedMask, maskSource: "reference-product", selectedComponentCount: selected.length, selectedPixelCount: selected.reduce((sum, component) => sum + component.area, 0), }; } async function buildLayoutLockedCompositeImage(params: { generatedImageUrl: string; referenceGeometry: { height: number; width: number }; referenceImageUrl: string; }) { const sharp = (await import("sharp")).default; const [referenceSourceBuffer, generatedSourceBuffer] = await Promise.all([ fetchImageBufferFromUrl(params.referenceImageUrl), fetchImageBufferFromUrl(params.generatedImageUrl), ]); const targetWidth = params.referenceGeometry.width; const targetHeight = params.referenceGeometry.height; const referenceBuffer = await sharp(referenceSourceBuffer, { failOn: "none" }) .rotate() .resize(targetWidth, targetHeight, { fit: "fill", withoutEnlargement: false, }) .png() .toBuffer(); const generatedBuffer = await sharp(generatedSourceBuffer, { failOn: "none" }) .rotate() .resize(targetWidth, targetHeight, { background: { r: 255, g: 255, b: 255, alpha: 1 }, fit: "contain", position: "centre", withoutEnlargement: false, }) .png() .toBuffer(); const [referenceRaw, generatedRaw] = await Promise.all([ sharp(referenceBuffer, { failOn: "none" }) .removeAlpha() .raw() .toBuffer({ resolveWithObject: true }), sharp(generatedBuffer, { failOn: "none" }) .removeAlpha() .raw() .toBuffer({ resolveWithObject: true }), ]); if ( referenceRaw.info.width !== generatedRaw.info.width || referenceRaw.info.height !== generatedRaw.info.height ) { return null; } const selectedMask = selectReferenceProductMask({ height: referenceRaw.info.height, reference: referenceRaw.data, width: referenceRaw.info.width, }) || selectLayoutPatchMask({ generated: generatedRaw.data, height: referenceRaw.info.height, reference: referenceRaw.data, width: referenceRaw.info.width, }); if (!selectedMask) { return null; } const usesReferenceProductMask = selectedMask.maskSource === "reference-product"; const firstBlurRadius = usesReferenceProductMask ? 14 : 10; const maskThreshold = usesReferenceProductMask ? 12 : 2; const pixelCount = targetWidth * targetHeight; const alphaMask = await sharp(selectedMask.mask, { raw: { channels: 1, height: targetHeight, width: targetWidth, }, }) .blur(firstBlurRadius) .threshold(maskThreshold) .blur(4) .extractChannel(0) .raw() .toBuffer(); if (alphaMask.length !== pixelCount) { throw new Error( `排版替换蒙版通道异常: expected ${pixelCount}, got ${alphaMask.length}`, ); } const generatedPatchRaw = Buffer.alloc(pixelCount * 4); const sourceChannels = generatedRaw.info.channels; for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) { const sourceOffset = pixelIndex * sourceChannels; const targetOffset = pixelIndex * 4; generatedPatchRaw[targetOffset] = generatedRaw.data[sourceOffset] || 0; generatedPatchRaw[targetOffset + 1] = generatedRaw.data[sourceOffset + 1] || 0; generatedPatchRaw[targetOffset + 2] = generatedRaw.data[sourceOffset + 2] || 0; generatedPatchRaw[targetOffset + 3] = alphaMask[pixelIndex] || 0; } const generatedPatch = await sharp(generatedPatchRaw, { raw: { channels: 4, height: targetHeight, width: targetWidth, }, }) .png() .toBuffer(); const compositeBuffer = await sharp(referenceBuffer, { failOn: "none" }) .composite([{ input: generatedPatch }]) .png() .toBuffer(); return { buffer: compositeBuffer, height: targetHeight, maskSource: selectedMask.maskSource, selectedComponentCount: selectedMask.selectedComponentCount, selectedPixelCount: selectedMask.selectedPixelCount, width: targetWidth, }; } export async function executeCanvasLayoutReplaceTask(params: { request: CanvasLayoutReplaceTaskRequest; userId: string; }): Promise { const sourceImageUrl = normalizeHttpUrl(params.request.sourceImageUrl); const referenceImageUrl = normalizeHttpUrl(params.request.referenceImageUrl); if (!sourceImageUrl) { throw new Error("缺少当前产品图片"); } if (!referenceImageUrl) { throw new Error("缺少排版参考图"); } const models = await getEnabledAiModelsWithProvider().catch((error) => { console.warn("[Canvas Layout Replace] Failed to load models:", error); return undefined; }); const resolvedModelRef = await resolveDirectReplacementModelRef({ selectedChatModel: params.request.model, finalModelId: params.request.model, models, }); if (!resolvedModelRef || resolvedModelRef === "auto") { throw new Error("当前风格化未配置可用的生图模型"); } const replacementStrategy: AgentDetailReplacementStrategy = "product_only"; const referenceLabel = String(params.request.referenceLabel || "").trim() || "画布排版参考图"; const referenceGeometry = await resolveReplacementReferenceGeometry(referenceImageUrl); const planningRuntime = await resolveReplacementScenePlanningRuntime({ models }); const replacementExecutionPlan = await analyzeReplacementExecutionPlan({ modeType: "detail_replacement", replacementStrategy, referenceImageUrl, referenceLabel, productImageUrls: [sourceImageUrl], modelImageUrl: null, workflowPrompt: null, planningRuntime, models, }); const basePrompt = "仅替换参考详情图中的产品主体"; const replacementPrompt = buildAgentReplacementPrompt({ basePrompt, modeType: "detail_replacement", replacementStrategy, marketplaceProductDetailReference: false, referenceLabel, referenceIndex: 0, totalReferences: 1, hasModelImage: false, productImageCount: 1, originalProductImageCount: 1, rerunReferenceCount: 0, templateReferenceCount: 0, scenePlan: replacementExecutionPlan.scenePlan, referenceGeometry, }); const supplement = normalizeLayoutReplacePromptSupplement( params.request.supplementalPrompt, ); const finalPrompt = [ replacementPrompt, supplement ? `【画布排版替换补充】${supplement}。这段只用于理解新产品材质、颜色、结构或替换细节;不得改写、移动、删除、重排参考图中的任何文字、图标、按钮、标签、分割线、装饰、留白或页面结构。` : "", "【最终验收标准】输出是一张与最后一张参考图相同页面比例的单页结果;文字、图标、按钮、标签、分割线、品牌区、留白和信息层级必须视觉上与参考图一致,只允许旧产品主体/产品展示区被当前产品替换。", ] .filter(Boolean) .join("\n\n"); const generated = await generateCanvasImage({ prompt: finalPrompt, displayPrompt: "排版替换结果", model: resolvedModelRef, ownerKey: params.userId, aspectRatio: referenceGeometry?.aspectRatio || "auto", imageSize: params.request.imageSize || "2K", imageCount: 1, promptExtend: false, referenceImages: [sourceImageUrl, referenceImageUrl], debugLabel: "canvas-layout-replace", debugMeta: { ...(params.request.debugMeta || {}), operation: "layout-replace", layoutReplaceServerAlgorithm: "agent-detail-replacement/product_only", sourceNodeId: params.request.sourceNodeId || null, referenceWidth: referenceGeometry?.width, referenceHeight: referenceGeometry?.height, referenceAspectRatio: referenceGeometry?.aspectRatio, pageKind: replacementExecutionPlan.scenePlan.pageKind, generationRoute: replacementExecutionPlan.scenePlan.generationRoute, promptPackPolicy: replacementExecutionPlan.scenePlan.promptPackPolicy, }, }); const firstImage = generated.images[0]; if (!firstImage?.originalUrl) { throw new Error("排版替换未返回结果图片"); } let normalizedImage = await normalizeReplacementResultImage({ generatedImageUrl: firstImage.originalUrl, generatedThumbnailUrl: firstImage.thumbnailUrl, referenceImageUrl, referenceGeometry, scenePlan: replacementExecutionPlan.scenePlan, uploadPrefix: "canvas-layout-replace", }); if (referenceGeometry) { const lockedComposite = await buildLayoutLockedCompositeImage({ generatedImageUrl: firstImage.originalUrl, referenceGeometry, referenceImageUrl, }).catch((error) => { console.warn("[Canvas Layout Replace] Layout locked composite failed:", { error: error instanceof Error ? error.message : String(error), referenceImageUrl, resultImageUrl: firstImage.originalUrl, }); return null; }); if (lockedComposite?.buffer) { const thumbnailBuffer = await createAgentImageThumbnailBuffer( lockedComposite.buffer, ); const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const [originalUpload, thumbnailUpload] = await Promise.all([ uploadFile( `canvas-layout-replace-locked-orig-${suffix}.png`, bufferToArrayBuffer(lockedComposite.buffer), ), uploadFile( `canvas-layout-replace-locked-thumb-${suffix}.jpg`, bufferToArrayBuffer(thumbnailBuffer), ), ]); normalizedImage = { originalUrl: originalUpload.url, thumbnailUrl: thumbnailUpload.url || originalUpload.url, imageSize: `${lockedComposite.width}x${lockedComposite.height}`, }; console.info("[Canvas Layout Replace] Applied layout locked composite:", { maskSource: lockedComposite.maskSource, referenceWidth: lockedComposite.width, referenceHeight: lockedComposite.height, selectedComponentCount: lockedComposite.selectedComponentCount, selectedPixelCount: lockedComposite.selectedPixelCount, }); } } return { images: [ { originalUrl: normalizedImage.originalUrl, thumbnailUrl: normalizedImage.thumbnailUrl || normalizedImage.originalUrl, }, ], model: generated.model, modelRef: generated.modelRef, prompt: generated.prompt, }; }