fix: preserve unselected local edit areas
This commit is contained in:
parent
939138a02a
commit
b17010bb35
@ -408,7 +408,7 @@ function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) {
|
|||||||
referenceLines ? `\n区域专属参考图说明:\n${referenceLines}` : "",
|
referenceLines ? `\n区域专属参考图说明:\n${referenceLines}` : "",
|
||||||
"",
|
"",
|
||||||
"最终输出必须是干净照片:不得出现任何彩色框、蓝框、绿框、红框、矩形边框、编号、提示词气泡、标注文字或 UI 元素。",
|
"最终输出必须是干净照片:不得出现任何彩色框、蓝框、绿框、红框、矩形边框、编号、提示词气泡、标注文字或 UI 元素。",
|
||||||
"未被列为编辑区域的画面必须尽量保持原图不变。",
|
"未被列为编辑区域的画面必须逐像素保持原图不变,不允许重绘、改色、补光、换材质、移动物体、改变清晰度或改变背景细节。",
|
||||||
"新增内容必须匹配原图的透视、光线、比例、清晰度、色温和真实摄影风格。",
|
"新增内容必须匹配原图的透视、光线、比例、清晰度、色温和真实摄影风格。",
|
||||||
"如果某个区域提示词与原图空间冲突,以自然融入原图为最高优先级。",
|
"如果某个区域提示词与原图空间冲突,以自然融入原图为最高优先级。",
|
||||||
].join("\n");
|
].join("\n");
|
||||||
@ -524,6 +524,10 @@ export async function POST(request: NextRequest) {
|
|||||||
sourceImageUrl,
|
sourceImageUrl,
|
||||||
...regionReferenceImageUrls,
|
...regionReferenceImageUrls,
|
||||||
],
|
],
|
||||||
|
localEditComposite: {
|
||||||
|
sourceImageUrl,
|
||||||
|
regions: regions.map((region) => region.rect),
|
||||||
|
},
|
||||||
ownerKey: session.user.id,
|
ownerKey: session.user.id,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -1,6 +1,8 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
|
||||||
|
import sharp from "sharp";
|
||||||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||||
|
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||||
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||||||
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
||||||
import { createBflImage } from "@/lib/ai/providers/bfl-image";
|
import { createBflImage } from "@/lib/ai/providers/bfl-image";
|
||||||
@ -25,6 +27,19 @@ import {
|
|||||||
recordAiModelHealthSuccess,
|
recordAiModelHealthSuccess,
|
||||||
shouldFailoverAiModelError,
|
shouldFailoverAiModelError,
|
||||||
} from "@/lib/ai/runtime/model-health";
|
} from "@/lib/ai/runtime/model-health";
|
||||||
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
|
export interface CanvasLocalEditCompositeRegion {
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CanvasLocalEditCompositeConfig {
|
||||||
|
sourceImageUrl: string;
|
||||||
|
regions: CanvasLocalEditCompositeRegion[];
|
||||||
|
}
|
||||||
|
|
||||||
export interface CanvasGenerateImageRequest {
|
export interface CanvasGenerateImageRequest {
|
||||||
prompt: string;
|
prompt: string;
|
||||||
@ -40,6 +55,7 @@ export interface CanvasGenerateImageRequest {
|
|||||||
ownerKey?: string;
|
ownerKey?: string;
|
||||||
debugLabel?: string;
|
debugLabel?: string;
|
||||||
debugMeta?: Record<string, unknown>;
|
debugMeta?: Record<string, unknown>;
|
||||||
|
localEditComposite?: CanvasLocalEditCompositeConfig;
|
||||||
allowFailover?: boolean;
|
allowFailover?: boolean;
|
||||||
internalFailoverTriedModelRefs?: string[];
|
internalFailoverTriedModelRefs?: string[];
|
||||||
onQueueStatusChange?: (
|
onQueueStatusChange?: (
|
||||||
@ -123,6 +139,150 @@ function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
|||||||
return parsed;
|
return parsed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||||
|
return buffer.buffer.slice(
|
||||||
|
buffer.byteOffset,
|
||||||
|
buffer.byteOffset + buffer.byteLength
|
||||||
|
) as ArrayBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampRatio(value: number) {
|
||||||
|
if (!Number.isFinite(value)) return 0;
|
||||||
|
return Math.min(1, Math.max(0, value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeLocalEditCompositeConfig(
|
||||||
|
value: CanvasGenerateImageRequest["localEditComposite"]
|
||||||
|
): CanvasLocalEditCompositeConfig | null {
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
const sourceImageUrl = String(value.sourceImageUrl || "").trim();
|
||||||
|
if (!/^https?:\/\//i.test(sourceImageUrl)) return null;
|
||||||
|
if (!Array.isArray(value.regions) || value.regions.length === 0) return null;
|
||||||
|
|
||||||
|
const regions = value.regions
|
||||||
|
.map((region) => {
|
||||||
|
const x = clampRatio(Number(region?.x));
|
||||||
|
const y = clampRatio(Number(region?.y));
|
||||||
|
const width = clampRatio(Number(region?.width));
|
||||||
|
const height = clampRatio(Number(region?.height));
|
||||||
|
return {
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
width: Math.min(width, Math.max(0, 1 - x)),
|
||||||
|
height: Math.min(height, Math.max(0, 1 - y)),
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((region) => region.width > 0 && region.height > 0);
|
||||||
|
|
||||||
|
return regions.length > 0 ? { sourceImageUrl, regions } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildLocalEditCompositeMaskSvg(params: {
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
regions: CanvasLocalEditCompositeRegion[];
|
||||||
|
}) {
|
||||||
|
const { width, height, regions } = params;
|
||||||
|
const rects = regions
|
||||||
|
.map((region) => {
|
||||||
|
const left = Math.max(0, Math.round(region.x * width));
|
||||||
|
const top = Math.max(0, Math.round(region.y * height));
|
||||||
|
const rectWidth = Math.min(
|
||||||
|
width - left,
|
||||||
|
Math.max(1, Math.round(region.width * width))
|
||||||
|
);
|
||||||
|
const rectHeight = Math.min(
|
||||||
|
height - top,
|
||||||
|
Math.max(1, Math.round(region.height * height))
|
||||||
|
);
|
||||||
|
return `<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" fill="#ffffff" />`;
|
||||||
|
})
|
||||||
|
.join("");
|
||||||
|
|
||||||
|
return Buffer.from(`
|
||||||
|
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||||||
|
<rect x="0" y="0" width="${width}" height="${height}" fill="rgba(0,0,0,0)" />
|
||||||
|
${rects}
|
||||||
|
</svg>
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function applyLocalEditCompositeToImages(params: {
|
||||||
|
images: ParsedImage[];
|
||||||
|
composite: CanvasLocalEditCompositeConfig | null;
|
||||||
|
debugLabel: string;
|
||||||
|
}) {
|
||||||
|
const { images, composite, debugLabel } = params;
|
||||||
|
if (!composite || images.length === 0) return images;
|
||||||
|
|
||||||
|
const sourceBuffer = await sharp(
|
||||||
|
await fetchImageBufferFromUrl(composite.sourceImageUrl),
|
||||||
|
{ failOn: "none" }
|
||||||
|
)
|
||||||
|
.rotate()
|
||||||
|
.png({ compressionLevel: 6 })
|
||||||
|
.toBuffer();
|
||||||
|
const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata();
|
||||||
|
const sourceWidth = Number(sourceMetadata.width || 0);
|
||||||
|
const sourceHeight = Number(sourceMetadata.height || 0);
|
||||||
|
if (sourceWidth <= 0 || sourceHeight <= 0) {
|
||||||
|
throw new Error("局部编辑结果合成失败:无法识别原图尺寸");
|
||||||
|
}
|
||||||
|
|
||||||
|
const maskSvg = buildLocalEditCompositeMaskSvg({
|
||||||
|
height: sourceHeight,
|
||||||
|
regions: composite.regions,
|
||||||
|
width: sourceWidth,
|
||||||
|
});
|
||||||
|
|
||||||
|
const mergedImages: ParsedImage[] = [];
|
||||||
|
for (const [index, image] of images.entries()) {
|
||||||
|
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||||||
|
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
||||||
|
const overlayBuffer = await sharp(generatedBuffer, { failOn: "none" })
|
||||||
|
.rotate()
|
||||||
|
.resize(sourceWidth, sourceHeight, { fit: "cover", position: "center" })
|
||||||
|
.ensureAlpha()
|
||||||
|
.composite([{ input: maskSvg, blend: "dest-in" }])
|
||||||
|
.png({ compressionLevel: 6 })
|
||||||
|
.toBuffer();
|
||||||
|
const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||||||
|
.ensureAlpha()
|
||||||
|
.composite([{ input: overlayBuffer, left: 0, top: 0 }])
|
||||||
|
.png({ compressionLevel: 6 })
|
||||||
|
.toBuffer();
|
||||||
|
const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" })
|
||||||
|
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
|
||||||
|
.png({ compressionLevel: 6 })
|
||||||
|
.toBuffer();
|
||||||
|
const suffix = `${Date.now()}-${index + 1}-${Math.random().toString(36).slice(2, 8)}`;
|
||||||
|
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||||||
|
uploadFile(
|
||||||
|
`canvas-local-edit-composited-${suffix}.png`,
|
||||||
|
bufferToArrayBuffer(mergedBuffer)
|
||||||
|
),
|
||||||
|
uploadFile(
|
||||||
|
`canvas-local-edit-composited-thumb-${suffix}.png`,
|
||||||
|
bufferToArrayBuffer(thumbnailBuffer)
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
mergedImages.push({
|
||||||
|
originalUrl: originalUpload.url,
|
||||||
|
thumbnailUrl: thumbnailUpload.url,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.info("[Canvas Generate Image] Local edit composite applied:", {
|
||||||
|
debugLabel,
|
||||||
|
imageCount: mergedImages.length,
|
||||||
|
regionCount: composite.regions.length,
|
||||||
|
sourceHeight,
|
||||||
|
sourceWidth,
|
||||||
|
});
|
||||||
|
|
||||||
|
return mergedImages;
|
||||||
|
}
|
||||||
|
|
||||||
function getProviderBatchLimit(providerType: string) {
|
function getProviderBatchLimit(providerType: string) {
|
||||||
switch (providerType) {
|
switch (providerType) {
|
||||||
case "ruxa-image":
|
case "ruxa-image":
|
||||||
@ -239,6 +399,9 @@ export async function generateCanvasImage(
|
|||||||
request.debugMeta && typeof request.debugMeta === "object"
|
request.debugMeta && typeof request.debugMeta === "object"
|
||||||
? request.debugMeta
|
? request.debugMeta
|
||||||
: undefined;
|
: undefined;
|
||||||
|
const localEditComposite = normalizeLocalEditCompositeConfig(
|
||||||
|
request.localEditComposite
|
||||||
|
);
|
||||||
|
|
||||||
const allModels = await getEnabledAiModelsWithProvider();
|
const allModels = await getEnabledAiModelsWithProvider();
|
||||||
const imageModels = allModels.filter((m) => m.model.supportsImageGen);
|
const imageModels = allModels.filter((m) => m.model.supportsImageGen);
|
||||||
@ -335,6 +498,7 @@ export async function generateCanvasImage(
|
|||||||
selectedModelId: selectedModel.model.modelId,
|
selectedModelId: selectedModel.model.modelId,
|
||||||
providerType: provider.type,
|
providerType: provider.type,
|
||||||
...debugMeta,
|
...debugMeta,
|
||||||
|
hasLocalEditComposite: Boolean(localEditComposite),
|
||||||
});
|
});
|
||||||
|
|
||||||
const messages = [
|
const messages = [
|
||||||
@ -622,7 +786,11 @@ export async function generateCanvasImage(
|
|||||||
throw new Error("图片生成失败");
|
throw new Error("图片生成失败");
|
||||||
}
|
}
|
||||||
|
|
||||||
const finalImages = collectedImages.slice(0, targetImageCount);
|
const finalImages = await applyLocalEditCompositeToImages({
|
||||||
|
composite: localEditComposite,
|
||||||
|
debugLabel,
|
||||||
|
images: collectedImages.slice(0, targetImageCount),
|
||||||
|
});
|
||||||
console.info("[Canvas Generate Image] Completed:", {
|
console.info("[Canvas Generate Image] Completed:", {
|
||||||
debugLabel,
|
debugLabel,
|
||||||
elapsedMs: Date.now() - startedAt,
|
elapsedMs: Date.now() - startedAt,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user