From b17010bb35ebae541736d469287ec97a0fe8a638 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 26 May 2026 14:48:55 +0800 Subject: [PATCH] fix: preserve unselected local edit areas --- app/api/canvas/local-edit/route.ts | 6 +- lib/canvas/generate-image.ts | 170 ++++++++++++++++++++++++++++- 2 files changed, 174 insertions(+), 2 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 03346e427..da2ab194a 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -408,7 +408,7 @@ function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { referenceLines ? `\n区域专属参考图说明：\n${referenceLines}` : "", "", "最终输出必须是干净照片：不得出现任何彩色框、蓝框、绿框、红框、矩形边框、编号、提示词气泡、标注文字或 UI 元素。", - "未被列为编辑区域的画面必须尽量保持原图不变。", + "未被列为编辑区域的画面必须逐像素保持原图不变，不允许重绘、改色、补光、换材质、移动物体、改变清晰度或改变背景细节。", "新增内容必须匹配原图的透视、光线、比例、清晰度、色温和真实摄影风格。", "如果某个区域提示词与原图空间冲突，以自然融入原图为最高优先级。", ].join("\n"); @@ -524,6 +524,10 @@ export async function POST(request: NextRequest) { sourceImageUrl, ...regionReferenceImageUrls, ], + localEditComposite: { + sourceImageUrl, + regions: regions.map((region) => region.rect), + }, ownerKey: session.user.id, }; diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 9ac6b7132..d013aeb67 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -1,6 +1,8 @@ import "server-only"; +import sharp from "sharp"; import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; +import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; import { createBflImage } from "@/lib/ai/providers/bfl-image"; @@ -25,6 +27,19 @@ import { recordAiModelHealthSuccess, shouldFailoverAiModelError, } from "@/lib/ai/runtime/model-health"; +import { uploadFile } from "@/lib/storage/service"; + +export interface CanvasLocalEditCompositeRegion { + x: number; + y: number; + width: number; + height: number; +} + +export interface CanvasLocalEditCompositeConfig { + sourceImageUrl: string; + regions: CanvasLocalEditCompositeRegion[]; +} export interface CanvasGenerateImageRequest { prompt: string; @@ -40,6 +55,7 @@ export interface CanvasGenerateImageRequest { ownerKey?: string; debugLabel?: string; debugMeta?: Record; + localEditComposite?: CanvasLocalEditCompositeConfig; allowFailover?: boolean; internalFailoverTriedModelRefs?: string[]; onQueueStatusChange?: ( @@ -123,6 +139,150 @@ function parseImagesFromMarkdown(markdown: string): ParsedImage[] { return parsed; } +function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer { + return buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; +} + +function clampRatio(value: number) { + if (!Number.isFinite(value)) return 0; + return Math.min(1, Math.max(0, value)); +} + +function normalizeLocalEditCompositeConfig( + value: CanvasGenerateImageRequest["localEditComposite"] +): CanvasLocalEditCompositeConfig | null { + if (!value || typeof value !== "object") return null; + const sourceImageUrl = String(value.sourceImageUrl || "").trim(); + if (!/^https?:\/\//i.test(sourceImageUrl)) return null; + if (!Array.isArray(value.regions) || value.regions.length === 0) return null; + + const regions = value.regions + .map((region) => { + const x = clampRatio(Number(region?.x)); + const y = clampRatio(Number(region?.y)); + const width = clampRatio(Number(region?.width)); + const height = clampRatio(Number(region?.height)); + return { + x, + y, + width: Math.min(width, Math.max(0, 1 - x)), + height: Math.min(height, Math.max(0, 1 - y)), + }; + }) + .filter((region) => region.width > 0 && region.height > 0); + + return regions.length > 0 ? { sourceImageUrl, regions } : null; +} + +function buildLocalEditCompositeMaskSvg(params: { + width: number; + height: number; + regions: CanvasLocalEditCompositeRegion[]; +}) { + const { width, height, regions } = params; + const rects = regions + .map((region) => { + const left = Math.max(0, Math.round(region.x * width)); + const top = Math.max(0, Math.round(region.y * height)); + const rectWidth = Math.min( + width - left, + Math.max(1, Math.round(region.width * width)) + ); + const rectHeight = Math.min( + height - top, + Math.max(1, Math.round(region.height * height)) + ); + return ``; + }) + .join(""); + + return Buffer.from(` + + + ${rects} + + `); +} + +async function applyLocalEditCompositeToImages(params: { + images: ParsedImage[]; + composite: CanvasLocalEditCompositeConfig | null; + debugLabel: string; +}) { + const { images, composite, debugLabel } = params; + if (!composite || images.length === 0) return images; + + const sourceBuffer = await sharp( + await fetchImageBufferFromUrl(composite.sourceImageUrl), + { failOn: "none" } + ) + .rotate() + .png({ compressionLevel: 6 }) + .toBuffer(); + const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata(); + const sourceWidth = Number(sourceMetadata.width || 0); + const sourceHeight = Number(sourceMetadata.height || 0); + if (sourceWidth <= 0 || sourceHeight <= 0) { + throw new Error("局部编辑结果合成失败：无法识别原图尺寸"); + } + + const maskSvg = buildLocalEditCompositeMaskSvg({ + height: sourceHeight, + regions: composite.regions, + width: sourceWidth, + }); + + const mergedImages: ParsedImage[] = []; + for (const [index, image] of images.entries()) { + const generatedUrl = image.originalUrl || image.thumbnailUrl; + const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); + const overlayBuffer = await sharp(generatedBuffer, { failOn: "none" }) + .rotate() + .resize(sourceWidth, sourceHeight, { fit: "cover", position: "center" }) + .ensureAlpha() + .composite([{ input: maskSvg, blend: "dest-in" }]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .ensureAlpha() + .composite([{ input: overlayBuffer, left: 0, top: 0 }]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + .resize(512, 512, { fit: "inside", withoutEnlargement: true }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const suffix = `${Date.now()}-${index + 1}-${Math.random().toString(36).slice(2, 8)}`; + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile( + `canvas-local-edit-composited-${suffix}.png`, + bufferToArrayBuffer(mergedBuffer) + ), + uploadFile( + `canvas-local-edit-composited-thumb-${suffix}.png`, + bufferToArrayBuffer(thumbnailBuffer) + ), + ]); + mergedImages.push({ + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + }); + } + + console.info("[Canvas Generate Image] Local edit composite applied:", { + debugLabel, + imageCount: mergedImages.length, + regionCount: composite.regions.length, + sourceHeight, + sourceWidth, + }); + + return mergedImages; +} + function getProviderBatchLimit(providerType: string) { switch (providerType) { case "ruxa-image": @@ -239,6 +399,9 @@ export async function generateCanvasImage( request.debugMeta && typeof request.debugMeta === "object" ? request.debugMeta : undefined; + const localEditComposite = normalizeLocalEditCompositeConfig( + request.localEditComposite + ); const allModels = await getEnabledAiModelsWithProvider(); const imageModels = allModels.filter((m) => m.model.supportsImageGen); @@ -335,6 +498,7 @@ export async function generateCanvasImage( selectedModelId: selectedModel.model.modelId, providerType: provider.type, ...debugMeta, + hasLocalEditComposite: Boolean(localEditComposite), }); const messages = [ @@ -622,7 +786,11 @@ export async function generateCanvasImage( throw new Error("图片生成失败"); } - const finalImages = collectedImages.slice(0, targetImageCount); + const finalImages = await applyLocalEditCompositeToImages({ + composite: localEditComposite, + debugLabel, + images: collectedImages.slice(0, targetImageCount), + }); console.info("[Canvas Generate Image] Completed:", { debugLabel, elapsedMs: Date.now() - startedAt,