fix: support mask guided scale adjust
This commit is contained in:
134
parent
7152435a3d
commit
2f312a01df
@ -17,6 +17,7 @@ import {
|
||||
import {
|
||||
modelLacksReferenceImageInputSupport,
|
||||
} from "@/lib/ai/reference-image-capability";
|
||||
import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image";
|
||||
import { getImageToolCreditCost } from "@/lib/points/config";
|
||||
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
@ -278,6 +279,19 @@ function getScaleAdjustModelFamily(
|
||||
);
|
||||
}
|
||||
|
||||
function isScaleAdjustMaskCapableModel(
|
||||
entry: AiModelWithProviderForSelection | null | undefined
|
||||
) {
|
||||
if (!entry || getScaleAdjustModelFamily(entry) !== "gpt-image-2") {
|
||||
return false;
|
||||
}
|
||||
if (entry.provider.type === "openai-image") return true;
|
||||
if (entry.provider.type === "fal-image") {
|
||||
return isFalGptImage2EditModelId(entry.model.modelId || "");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isScaleAdjustCoordinateGuideModel(
|
||||
entry: AiModelWithProviderForSelection | null | undefined
|
||||
) {
|
||||
@ -1004,6 +1018,89 @@ function buildScaleAdjustTargetFootprintShadowSvg(params: {
|
||||
);
|
||||
}
|
||||
|
||||
function getScaleAdjustEditMaskBoxes(params: {
|
||||
regions: NormalizedLocalEditRegion[];
|
||||
scalePercent?: ScaleAdjustPercent | null;
|
||||
sourceHeight: number;
|
||||
sourceWidth: number;
|
||||
}) {
|
||||
const { regions, scalePercent, sourceHeight, sourceWidth } = params;
|
||||
return regions.map((region) => {
|
||||
const targetRect = getScaleAdjustTargetRect(region, scalePercent);
|
||||
const workspaceRect = getScaleAdjustWorkspaceRect(
|
||||
region.rect,
|
||||
targetRect,
|
||||
scalePercent
|
||||
);
|
||||
return rectToPixels(workspaceRect, sourceWidth, sourceHeight);
|
||||
});
|
||||
}
|
||||
|
||||
async function buildScaleAdjustEditMaskBuffer(params: {
|
||||
regions: NormalizedLocalEditRegion[];
|
||||
scalePercent?: ScaleAdjustPercent | null;
|
||||
sourceHeight: number;
|
||||
sourceWidth: number;
|
||||
}) {
|
||||
const { regions, scalePercent, sourceHeight, sourceWidth } = params;
|
||||
const boxes = getScaleAdjustEditMaskBoxes({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
});
|
||||
const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight));
|
||||
const blur = Math.max(5, Math.min(28, Math.round(minSide * 0.018)));
|
||||
const pad = Math.max(2, Math.round(blur * 0.55));
|
||||
const rects = boxes
|
||||
.map((box) => {
|
||||
const left = clamp(box.left - pad, 0, Math.max(0, sourceWidth - 1));
|
||||
const top = clamp(box.top - pad, 0, Math.max(0, sourceHeight - 1));
|
||||
const right = clamp(
|
||||
box.left + box.width + pad,
|
||||
left + 1,
|
||||
sourceWidth
|
||||
);
|
||||
const bottom = clamp(
|
||||
box.top + box.height + pad,
|
||||
top + 1,
|
||||
sourceHeight
|
||||
);
|
||||
return `<rect x="${left}" y="${top}" width="${Math.max(1, right - left)}" height="${Math.max(1, bottom - top)}" rx="${Math.max(4, Math.round(blur * 0.8))}" ry="${Math.max(4, Math.round(blur * 0.8))}" fill="#ffffff" filter="url(#soft-edit-mask)" />`;
|
||||
})
|
||||
.join("");
|
||||
const editMaskSvg = Buffer.from(
|
||||
`<svg xmlns="http://www.w3.org/2000/svg" width="${sourceWidth}" height="${sourceHeight}" viewBox="0 0 ${sourceWidth} ${sourceHeight}">
|
||||
<defs>
|
||||
<filter id="soft-edit-mask" x="-20%" y="-20%" width="140%" height="140%">
|
||||
<feGaussianBlur stdDeviation="${blur}" />
|
||||
</filter>
|
||||
</defs>
|
||||
<rect x="0" y="0" width="${sourceWidth}" height="${sourceHeight}" fill="#000000" />
|
||||
${rects}
|
||||
</svg>`
|
||||
);
|
||||
const editAlpha = await sharp(editMaskSvg, { failOn: "none" })
|
||||
.resize(sourceWidth, sourceHeight, { fit: "fill" })
|
||||
.greyscale()
|
||||
.raw()
|
||||
.toBuffer();
|
||||
const rgba = Buffer.alloc(sourceWidth * sourceHeight * 4);
|
||||
for (let index = 0; index < sourceWidth * sourceHeight; index += 1) {
|
||||
const offset = index * 4;
|
||||
const editable = editAlpha[index] || 0;
|
||||
rgba[offset] = 0;
|
||||
rgba[offset + 1] = 0;
|
||||
rgba[offset + 2] = 0;
|
||||
rgba[offset + 3] = clamp(255 - editable, 0, 255);
|
||||
}
|
||||
return sharp(rgba, {
|
||||
raw: { channels: 4, height: sourceHeight, width: sourceWidth },
|
||||
})
|
||||
.png({ compressionLevel: 9 })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
|
||||
regions: NormalizedLocalEditRegion[];
|
||||
scalePercent?: ScaleAdjustPercent | null;
|
||||
@ -1999,8 +2096,29 @@ export async function POST(request: NextRequest) {
|
||||
annotatedFilename,
|
||||
toArrayBuffer(annotatedBuffer)
|
||||
);
|
||||
const useScaleAdjustEditMask =
|
||||
isScaleAdjustOperation &&
|
||||
isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry);
|
||||
const scaleAdjustEditMaskUpload = useScaleAdjustEditMask
|
||||
? await uploadFile(
|
||||
`canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2, 8)}.png`,
|
||||
toArrayBuffer(
|
||||
await buildScaleAdjustEditMaskBuffer({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
)
|
||||
)
|
||||
: null;
|
||||
const shouldCreateScaleAdjustCropReferences =
|
||||
isScaleAdjustOperation &&
|
||||
(useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinateGuide);
|
||||
const scaleAdjustCropReferences: ScaleAdjustCropReference[] =
|
||||
isScaleAdjustOperation && useScaleAdjustVisualLayoutGuide
|
||||
shouldCreateScaleAdjustCropReferences
|
||||
? await createScaleAdjustCropReferences({
|
||||
regions,
|
||||
sourceBuffer,
|
||||
@ -2054,11 +2172,9 @@ export async function POST(request: NextRequest) {
|
||||
const scaleAdjustVerificationConfig = isScaleAdjustOperation
|
||||
? {
|
||||
allowOutsideChangedWhenChangedInside: false,
|
||||
allowDeterministicFallback: false,
|
||||
protectedRegions: scaleAdjustCompositeRegions,
|
||||
rejectFailedCandidates:
|
||||
useScaleAdjustCoordinateGuide && Boolean(scaleAdjustModelFamily)
|
||||
? true
|
||||
: undefined,
|
||||
rejectFailedCandidates: true,
|
||||
sourceImageUrl,
|
||||
regions: regions.map((region) => region.rect),
|
||||
regionPrompts: regions.map((region) => region.prompt),
|
||||
@ -2097,6 +2213,12 @@ export async function POST(request: NextRequest) {
|
||||
: "scale-adjust-neutral-coordinate-map"
|
||||
: "scale-adjust-full-frame-semantic-edit-map"
|
||||
: "source-overlay",
|
||||
scaleAdjustEditMaskApplied: Boolean(scaleAdjustEditMaskUpload),
|
||||
scaleAdjustEditMaskUrl: scaleAdjustEditMaskUpload?.url || null,
|
||||
scaleAdjustEditMaskProviderType:
|
||||
useScaleAdjustEditMask
|
||||
? resolvedScaleAdjustModelEntry?.provider?.type || null
|
||||
: null,
|
||||
cleanSourceReferenceIndex: 1,
|
||||
configuredDetailModelId: configuredDetailModel || null,
|
||||
detailFixedModelApplied: Boolean(detailFixedModel),
|
||||
@ -2181,6 +2303,9 @@ export async function POST(request: NextRequest) {
|
||||
: "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位",
|
||||
prompt,
|
||||
promptExtend: false,
|
||||
providerImageParams: scaleAdjustEditMaskUpload
|
||||
? { maskImageUrl: scaleAdjustEditMaskUpload.url }
|
||||
: undefined,
|
||||
referenceImages,
|
||||
scaleAdjustVerification: scaleAdjustVerificationConfig,
|
||||
localEditComposite: isScaleAdjustOperation
|
||||
|
||||
@ -19,6 +19,7 @@ export interface OpenAIImageParams {
|
||||
quality?: string;
|
||||
background?: string;
|
||||
outputFormat?: string;
|
||||
maskImageUrl?: string;
|
||||
}
|
||||
|
||||
interface UploadedImage {
|
||||
@ -643,6 +644,7 @@ async function requestOpenAIImageGeneration(params: {
|
||||
aspectRatio?: string;
|
||||
imageSizeLabel?: "1K" | "2K" | "4K";
|
||||
imageUrls: string[];
|
||||
maskImageUrl?: string;
|
||||
quality?: string;
|
||||
background?: string;
|
||||
outputFormat?: string;
|
||||
@ -656,6 +658,7 @@ async function requestOpenAIImageGeneration(params: {
|
||||
aspectRatio,
|
||||
imageSizeLabel,
|
||||
imageUrls,
|
||||
maskImageUrl,
|
||||
quality,
|
||||
background,
|
||||
outputFormat,
|
||||
@ -723,6 +726,10 @@ async function requestOpenAIImageGeneration(params: {
|
||||
formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename);
|
||||
}
|
||||
}
|
||||
if (maskImageUrl) {
|
||||
const mask = await loadInputImage(maskImageUrl);
|
||||
formData.append("mask", new Blob([mask.buffer], { type: mask.mimeType }), mask.filename);
|
||||
}
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
@ -898,6 +905,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) {
|
||||
timeoutMs: resolveRequestTimeoutMs(baseURL, imageSize, activeModelId),
|
||||
useBltcyImageCompatParams: shouldUseBltcyImageCompatParams(baseURL),
|
||||
imageCount: imageUrls.length,
|
||||
hasMask: Boolean(options.imageParams?.maskImageUrl),
|
||||
baseURL: normalizeBaseURL(baseURL),
|
||||
});
|
||||
|
||||
@ -910,6 +918,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) {
|
||||
aspectRatio: aspectRatio || undefined,
|
||||
imageSizeLabel: imageSize,
|
||||
imageUrls,
|
||||
maskImageUrl: options.imageParams?.maskImageUrl,
|
||||
quality,
|
||||
background,
|
||||
outputFormat,
|
||||
|
||||
@ -53,6 +53,7 @@ export interface CanvasLocalEditCompositeConfig {
|
||||
}
|
||||
|
||||
export interface CanvasScaleAdjustVerificationConfig {
|
||||
allowDeterministicFallback?: boolean;
|
||||
allowOutsideChangedWhenChangedInside?: boolean;
|
||||
protectedRegions?: CanvasLocalEditCompositeRegion[];
|
||||
rejectFailedCandidates?: boolean;
|
||||
@ -75,6 +76,7 @@ export interface CanvasGenerateImageRetryVariant {
|
||||
negativePrompt?: string;
|
||||
prompt?: string;
|
||||
promptExtend?: boolean;
|
||||
providerImageParams?: Record<string, unknown>;
|
||||
referenceImage?: string;
|
||||
referenceImages?: string[];
|
||||
scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
|
||||
@ -92,6 +94,7 @@ export interface CanvasGenerateImageRequest {
|
||||
falSystemPrompt?: string;
|
||||
falThinkingLevel?: "minimal" | "high";
|
||||
promptExtend?: boolean;
|
||||
providerImageParams?: Record<string, unknown>;
|
||||
referenceImage?: string;
|
||||
referenceImages?: string[];
|
||||
billingFeature?: "multi-view" | "repaint";
|
||||
@ -395,15 +398,22 @@ function normalizeScaleAdjustVerificationConfig(
|
||||
Boolean(region)
|
||||
)
|
||||
: [];
|
||||
const regionPrompts = Array.isArray(value.regionPrompts)
|
||||
? value.regionPrompts
|
||||
.map((prompt) => String(prompt || "").trim())
|
||||
.slice(0, regions.length)
|
||||
: [];
|
||||
|
||||
return regions.length > 0 &&
|
||||
(targetRegions.length > 0 || typeof scalePercent === "number")
|
||||
? {
|
||||
sourceImageUrl,
|
||||
regions,
|
||||
allowDeterministicFallback: value.allowDeterministicFallback === true,
|
||||
allowOutsideChangedWhenChangedInside:
|
||||
value.allowOutsideChangedWhenChangedInside === true,
|
||||
rejectFailedCandidates: value.rejectFailedCandidates === true,
|
||||
...(regionPrompts.length > 0 ? { regionPrompts } : {}),
|
||||
...(targetRegions.length > 0 ? { targetRegions } : {}),
|
||||
...(typeof scalePercent === "number" ? { scalePercent } : {}),
|
||||
...(protectedRegions.length > 0 ? { protectedRegions } : {}),
|
||||
@ -2526,6 +2536,10 @@ export async function generateCanvasImage(
|
||||
: 1;
|
||||
const promptExtend =
|
||||
typeof request.promptExtend === "boolean" ? request.promptExtend : undefined;
|
||||
const providerImageParams =
|
||||
request.providerImageParams && typeof request.providerImageParams === "object"
|
||||
? request.providerImageParams
|
||||
: undefined;
|
||||
const ownerKey =
|
||||
typeof request.ownerKey === "string" && request.ownerKey.trim()
|
||||
? request.ownerKey.trim()
|
||||
@ -2742,6 +2756,7 @@ export async function generateCanvasImage(
|
||||
];
|
||||
|
||||
const imageParams = {
|
||||
...(providerImageParams || {}),
|
||||
aspectRatio,
|
||||
safetyTolerance: request.falSafetyTolerance,
|
||||
imageSize,
|
||||
@ -3242,7 +3257,7 @@ export async function generateCanvasImage(
|
||||
selectedModelId: selectedModel.model.modelId,
|
||||
...debugMeta,
|
||||
});
|
||||
if (scaleAdjustVerification) {
|
||||
if (scaleAdjustVerification?.allowDeterministicFallback) {
|
||||
try {
|
||||
const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({
|
||||
debugLabel,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user