fix: guide FAL scale adjust with target mockup
This commit is contained in:
134
parent
186e2cefc2
commit
e02dff0b06
@ -97,6 +97,8 @@ const MAX_REFERENCE_IMAGES_PER_REGION = 3;
|
||||
const MAX_SCALE_ADJUST_CROP_REFERENCES = 4;
|
||||
const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18;
|
||||
const SCALE_ADJUST_CROP_MAX_EDGE = 1536;
|
||||
const SCALE_ADJUST_FAL_MOCKUP_PADDING_RATIO = 0.04;
|
||||
const SCALE_ADJUST_FAL_MOCKUP_BLUR_PX = 38;
|
||||
const MIN_REGION_EDGE_PX = 24;
|
||||
const SCALE_ADJUST_OPERATION = "scale-adjust";
|
||||
type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION;
|
||||
@ -226,6 +228,12 @@ function isScaleAdjustDedicatedDetailModel(
|
||||
return label.includes("详情专线");
|
||||
}
|
||||
|
||||
function isFalImageModelEntry(
|
||||
entry: AiModelWithProviderForSelection | null | undefined
|
||||
) {
|
||||
return entry?.provider?.type === "fal-image";
|
||||
}
|
||||
|
||||
function resolveScaleAdjustPreferredModelId(params: {
|
||||
configuredModelId: string;
|
||||
requestedModelId: string;
|
||||
@ -536,6 +544,162 @@ function buildScaleAdjustCompositeRegions(
|
||||
});
|
||||
}
|
||||
|
||||
function expandPixelBox(params: {
|
||||
amount: number;
|
||||
box: NormalizedLocalEditRegion["pixelRect"];
|
||||
height: number;
|
||||
width: number;
|
||||
}) {
|
||||
const { amount, box, height, width } = params;
|
||||
const left = clamp(box.left - amount, 0, Math.max(0, width - 1));
|
||||
const top = clamp(box.top - amount, 0, Math.max(0, height - 1));
|
||||
const right = clamp(box.left + box.width + amount, left + 1, width);
|
||||
const bottom = clamp(box.top + box.height + amount, top + 1, height);
|
||||
|
||||
return {
|
||||
left,
|
||||
top,
|
||||
width: Math.max(1, right - left),
|
||||
height: Math.max(1, bottom - top),
|
||||
};
|
||||
}
|
||||
|
||||
async function createFeatherAlphaMask(params: {
|
||||
width: number;
|
||||
height: number;
|
||||
feather: number;
|
||||
inset: number;
|
||||
}) {
|
||||
const { width, height } = params;
|
||||
const feather = Math.max(1, Math.round(params.feather));
|
||||
const maxInset = Math.max(1, Math.floor(Math.min(width, height) * 0.42));
|
||||
const inset = clamp(Math.round(params.inset), 1, maxInset);
|
||||
const innerWidth = Math.max(1, width - inset * 2);
|
||||
const innerHeight = Math.max(1, height - inset * 2);
|
||||
const svg = Buffer.from(`
|
||||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs>
|
||||
<filter id="soft"><feGaussianBlur stdDeviation="${feather}" /></filter>
|
||||
</defs>
|
||||
<rect x="${inset}" y="${inset}" width="${innerWidth}" height="${innerHeight}" fill="#fff" filter="url(#soft)" />
|
||||
</svg>
|
||||
`);
|
||||
|
||||
return sharp(svg, { failOn: "none" })
|
||||
.resize(width, height, { fit: "fill" })
|
||||
.removeAlpha()
|
||||
.greyscale()
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function addFeatherAlpha(params: {
|
||||
buffer: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
feather: number;
|
||||
inset: number;
|
||||
}) {
|
||||
const mask = await createFeatherAlphaMask({
|
||||
feather: params.feather,
|
||||
height: params.height,
|
||||
inset: params.inset,
|
||||
width: params.width,
|
||||
});
|
||||
|
||||
return sharp(params.buffer, { failOn: "none" })
|
||||
.resize(params.width, params.height, { fit: "fill" })
|
||||
.removeAlpha()
|
||||
.joinChannel(mask)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function buildScaleAdjustFalTargetMockup(params: {
|
||||
sourceBuffer: Buffer;
|
||||
sourceHeight: number;
|
||||
sourceWidth: number;
|
||||
regions: NormalizedLocalEditRegion[];
|
||||
scalePercent?: ScaleAdjustPercent | null;
|
||||
}) {
|
||||
const { sourceBuffer, sourceHeight, sourceWidth, regions, scalePercent } = params;
|
||||
const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight));
|
||||
const composites: sharp.OverlayOptions[] = [];
|
||||
|
||||
for (const region of regions) {
|
||||
const targetRect = getScaleAdjustTargetRect(region, scalePercent);
|
||||
const targetPixelRect =
|
||||
region.targetPixelRect || rectToPixels(targetRect, sourceWidth, sourceHeight);
|
||||
const workspaceRect = getScaleAdjustWorkspaceRect(
|
||||
region.rect,
|
||||
targetRect,
|
||||
scalePercent
|
||||
);
|
||||
const workspacePixelRect = expandPixelBox({
|
||||
amount: Math.round(minSide * SCALE_ADJUST_FAL_MOCKUP_PADDING_RATIO),
|
||||
box: rectToPixels(workspaceRect, sourceWidth, sourceHeight),
|
||||
height: sourceHeight,
|
||||
width: sourceWidth,
|
||||
});
|
||||
|
||||
const blurredWorkspace = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.extract(workspacePixelRect)
|
||||
.blur(SCALE_ADJUST_FAL_MOCKUP_BLUR_PX)
|
||||
.modulate({ brightness: 1.05, saturation: 0.86 })
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const workspaceFeather = Math.max(
|
||||
18,
|
||||
Math.round(Math.min(workspacePixelRect.width, workspacePixelRect.height) * 0.1)
|
||||
);
|
||||
const featheredWorkspace = await addFeatherAlpha({
|
||||
buffer: blurredWorkspace,
|
||||
feather: workspaceFeather,
|
||||
height: workspacePixelRect.height,
|
||||
inset: Math.max(workspaceFeather * 2.6, workspaceFeather + 1),
|
||||
width: workspacePixelRect.width,
|
||||
});
|
||||
|
||||
const subjectCrop = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.extract(region.pixelRect)
|
||||
.resize(targetPixelRect.width, targetPixelRect.height, {
|
||||
fit: "cover",
|
||||
position: "center",
|
||||
})
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const subjectFeather = Math.max(
|
||||
8,
|
||||
Math.round(Math.min(targetPixelRect.width, targetPixelRect.height) * 0.08)
|
||||
);
|
||||
const featheredSubject = await addFeatherAlpha({
|
||||
buffer: subjectCrop,
|
||||
feather: subjectFeather,
|
||||
height: targetPixelRect.height,
|
||||
inset: Math.max(subjectFeather * 1.6, subjectFeather + 1),
|
||||
width: targetPixelRect.width,
|
||||
});
|
||||
|
||||
composites.push(
|
||||
{
|
||||
input: featheredWorkspace,
|
||||
left: workspacePixelRect.left,
|
||||
top: workspacePixelRect.top,
|
||||
},
|
||||
{
|
||||
input: featheredSubject,
|
||||
left: targetPixelRect.left,
|
||||
top: targetPixelRect.top,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return sharp(sourceBuffer, { failOn: "none" })
|
||||
.composite(composites)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
function expandPixelRectForCrop(params: {
|
||||
height: number;
|
||||
pixelRect: NormalizedLocalEditRegion["pixelRect"];
|
||||
@ -907,6 +1071,42 @@ function buildScaleAdjustPrompt(
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function buildFalScaleAdjustMockupPrompt(
|
||||
regions: NormalizedLocalEditRegion[],
|
||||
scalePercent: ScaleAdjustPercent | null | undefined,
|
||||
sourceSize: { width: number; height: number }
|
||||
) {
|
||||
const regionLines = regions
|
||||
.map((region) => {
|
||||
const targetRect = getScaleAdjustTargetRect(region, scalePercent);
|
||||
const targetPixelRect =
|
||||
region.targetPixelRect ||
|
||||
rectToPixels(targetRect, sourceSize.width, sourceSize.height);
|
||||
const widthFactor =
|
||||
region.rect.width > 0 ? targetRect.width / region.rect.width : 1;
|
||||
const heightFactor =
|
||||
region.rect.height > 0 ? targetRect.height / region.rect.height : 1;
|
||||
|
||||
return `区域 ${region.index}：原主体约在 x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}；目标主体约在 x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}；目标视觉占位约为原主体宽度 ${widthFactor.toFixed(2)} 倍、高度 ${heightFactor.toFixed(2)} 倍。必须保持主体自身宽高比和透视，不要拉伸或压扁。`;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
return [
|
||||
"这是一项室内家用场景里的家具/物体局部比例调整任务。输入图片1是一张粗略目标效果草图，不是最终结果，也不是标注图。",
|
||||
`最终输出必须是一张真实、干净、自然的单张照片，并保持原始画幅 ${sourceSize.width} x ${sourceSize.height} px。`,
|
||||
"输入图片1已经把目标主体粗略放到了用户希望的最终大小和位置；请把它作为最终布局意图来理解，但不要直接复制其中的模糊、拼接边缘、软边、矩形块或局部贴片痕迹。",
|
||||
"核心目标：只调整被选中的家具/物体主体比例和位置；不要恢复成原来的大尺寸，也不要返回几乎未修改的原图。",
|
||||
"请用 AI 自然重建原主体移除后露出的背景，让地毯、地板、窗户、墙面、桌面、阴影、遮挡和透视连续自然。",
|
||||
"请保留目标主体的真实材质、结构、轮廓、透视和光照；主体不能透明、幽灵化、变形、被裁切，不能变成说明图或标注元素。",
|
||||
"非目标区域必须保持原室内照片逻辑：画幅边界、窗框、沙发、植物、墙面、地毯整体纹理、光线、色温和镜头透视不要整体移动、裁切、放大、缩小或重排。",
|
||||
"",
|
||||
"区域要求：",
|
||||
regionLines,
|
||||
"",
|
||||
"输出前自检：如果目标主体仍接近原尺寸、输出像未修改原图、整张图被放大/裁切/重排，或者出现矩形框、箭头、文字、遮罩、拼图、硬边贴片、色差边缘，就是失败；必须重新生成一张无缝真实照片。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth();
|
||||
@ -1029,61 +1229,6 @@ export async function POST(request: NextRequest) {
|
||||
operation === SCALE_ADJUST_OPERATION &&
|
||||
(hasScaleAdjustTargetRects || scalePercent !== null);
|
||||
const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent });
|
||||
|
||||
const annotationSvg = isScaleAdjustOperation
|
||||
? buildScaleAdjustAnnotationSvg({
|
||||
height: sourceHeight,
|
||||
regions,
|
||||
scalePercent,
|
||||
width: sourceWidth,
|
||||
})
|
||||
: buildAnnotationSvg({
|
||||
height: sourceHeight,
|
||||
hidePromptText: false,
|
||||
regions,
|
||||
width: sourceWidth,
|
||||
});
|
||||
const annotatedBase = sharp(sourceBuffer, { failOn: "none" });
|
||||
const annotatedBuffer = await annotatedBase
|
||||
.composite([{ input: annotationSvg, left: 0, top: 0 }])
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const annotatedFilename = `${isScaleAdjustOperation ? "canvas-scale-adjust-guide" : "canvas-local-edit"}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`;
|
||||
const annotatedUpload = await uploadFile(
|
||||
annotatedFilename,
|
||||
toArrayBuffer(annotatedBuffer)
|
||||
);
|
||||
const scaleAdjustCropReferences = isScaleAdjustOperation
|
||||
? await createScaleAdjustCropReferences({
|
||||
regions,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
: [];
|
||||
const prompt =
|
||||
isScaleAdjustOperation
|
||||
? buildScaleAdjustPrompt(
|
||||
regions,
|
||||
scalePercent,
|
||||
scaleAdjustCropReferences,
|
||||
{ width: sourceWidth, height: sourceHeight }
|
||||
)
|
||||
: buildLocalEditPrompt(regions);
|
||||
const manualRegionReferenceImageUrls = regions.flatMap((region) =>
|
||||
region.referenceImages.map((reference) => reference.url)
|
||||
);
|
||||
const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map(
|
||||
(reference) => reference.url
|
||||
);
|
||||
const referenceImages = isScaleAdjustOperation
|
||||
? [
|
||||
sourceImageUrl,
|
||||
annotatedUpload.url,
|
||||
...scaleAdjustCropReferenceImageUrls,
|
||||
...manualRegionReferenceImageUrls,
|
||||
]
|
||||
: [sourceImageUrl, ...manualRegionReferenceImageUrls];
|
||||
const resolvedAspectRatio = resolveLocalEditAspectRatio(
|
||||
body.aspectRatio,
|
||||
sourceWidth,
|
||||
@ -1116,6 +1261,87 @@ export async function POST(request: NextRequest) {
|
||||
});
|
||||
}
|
||||
const resolvedModel = detailFixedModel || requestedModel;
|
||||
const resolvedScaleAdjustModelEntry = isScaleAdjustOperation
|
||||
? findModelSelectionEntry(scaleAdjustModelCandidates, resolvedModel || "")
|
||||
: null;
|
||||
const useFalScaleAdjustMockup =
|
||||
isScaleAdjustOperation && isFalImageModelEntry(resolvedScaleAdjustModelEntry);
|
||||
|
||||
const annotatedBuffer =
|
||||
useFalScaleAdjustMockup
|
||||
? await buildScaleAdjustFalTargetMockup({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
: await sharp(sourceBuffer, { failOn: "none" })
|
||||
.composite([
|
||||
{
|
||||
input: isScaleAdjustOperation
|
||||
? buildScaleAdjustAnnotationSvg({
|
||||
height: sourceHeight,
|
||||
regions,
|
||||
scalePercent,
|
||||
width: sourceWidth,
|
||||
})
|
||||
: buildAnnotationSvg({
|
||||
height: sourceHeight,
|
||||
hidePromptText: false,
|
||||
regions,
|
||||
width: sourceWidth,
|
||||
}),
|
||||
left: 0,
|
||||
top: 0,
|
||||
},
|
||||
])
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const annotatedFilename = `${useFalScaleAdjustMockup ? "canvas-scale-adjust-fal-target-mockup" : isScaleAdjustOperation ? "canvas-scale-adjust-guide" : "canvas-local-edit"}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`;
|
||||
const annotatedUpload = await uploadFile(
|
||||
annotatedFilename,
|
||||
toArrayBuffer(annotatedBuffer)
|
||||
);
|
||||
const scaleAdjustCropReferences = isScaleAdjustOperation
|
||||
? useFalScaleAdjustMockup
|
||||
? []
|
||||
: await createScaleAdjustCropReferences({
|
||||
regions,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
: [];
|
||||
const prompt = isScaleAdjustOperation
|
||||
? useFalScaleAdjustMockup
|
||||
? buildFalScaleAdjustMockupPrompt(regions, scalePercent, {
|
||||
width: sourceWidth,
|
||||
height: sourceHeight,
|
||||
})
|
||||
: buildScaleAdjustPrompt(
|
||||
regions,
|
||||
scalePercent,
|
||||
scaleAdjustCropReferences,
|
||||
{ width: sourceWidth, height: sourceHeight }
|
||||
)
|
||||
: buildLocalEditPrompt(regions);
|
||||
const manualRegionReferenceImageUrls = regions.flatMap((region) =>
|
||||
region.referenceImages.map((reference) => reference.url)
|
||||
);
|
||||
const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map(
|
||||
(reference) => reference.url
|
||||
);
|
||||
const referenceImages = isScaleAdjustOperation
|
||||
? useFalScaleAdjustMockup
|
||||
? [annotatedUpload.url]
|
||||
: [
|
||||
sourceImageUrl,
|
||||
annotatedUpload.url,
|
||||
...scaleAdjustCropReferenceImageUrls,
|
||||
...manualRegionReferenceImageUrls,
|
||||
]
|
||||
: [sourceImageUrl, ...manualRegionReferenceImageUrls];
|
||||
const scaleAdjustCompositeRegions = isScaleAdjustOperation
|
||||
? buildScaleAdjustCompositeRegions(regions, scalePercent)
|
||||
: [];
|
||||
@ -1129,21 +1355,26 @@ export async function POST(request: NextRequest) {
|
||||
annotatedReferenceUrl: annotatedUpload.url,
|
||||
annotatedReferenceSentToModel: isScaleAdjustOperation,
|
||||
annotatedReferenceType: isScaleAdjustOperation
|
||||
? "scale-adjust-neutral-coordinate-guide-map"
|
||||
? useFalScaleAdjustMockup
|
||||
? "scale-adjust-fal-rough-target-mockup"
|
||||
: "scale-adjust-neutral-coordinate-guide-map"
|
||||
: "source-overlay",
|
||||
cleanSourceReferenceIndex: 1,
|
||||
cleanSourceReferenceIndex:
|
||||
isScaleAdjustOperation && useFalScaleAdjustMockup ? null : 1,
|
||||
configuredDetailModelId: configuredDetailModel || null,
|
||||
detailFixedModelApplied: Boolean(detailFixedModel),
|
||||
detailFixedModelId: detailFixedModel || null,
|
||||
displayPrompt,
|
||||
falScaleAdjustMockupApplied: useFalScaleAdjustMockup,
|
||||
operation,
|
||||
requestedModel: requestedModel || null,
|
||||
resolvedAspectRatio,
|
||||
resolvedModel: resolvedModel || null,
|
||||
resolvedScaleAdjustProviderType:
|
||||
resolvedScaleAdjustModelEntry?.provider?.type || null,
|
||||
manualRegionReferenceImageCount: manualRegionReferenceImageUrls.length,
|
||||
regionReferenceImageCount:
|
||||
manualRegionReferenceImageUrls.length +
|
||||
scaleAdjustCropReferenceImageUrls.length,
|
||||
referenceImages.length,
|
||||
regionCount: regions.length,
|
||||
scalePercent,
|
||||
scaleAdjustCropReferenceCount: scaleAdjustCropReferenceImageUrls.length,
|
||||
@ -1153,7 +1384,9 @@ export async function POST(request: NextRequest) {
|
||||
scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect)
|
||||
.length,
|
||||
scaleAdjustPromptStrategy: isScaleAdjustOperation
|
||||
? "locked-plate-neutral-coordinate-guide-map-pixel-anchors-v8"
|
||||
? useFalScaleAdjustMockup
|
||||
? "fal-rough-target-mockup-only-v17"
|
||||
: "locked-plate-neutral-coordinate-guide-map-pixel-anchors-v8"
|
||||
: undefined,
|
||||
scaleAdjustPostCompositeApplied: false,
|
||||
scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user