diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts
index 919766b67..944a6c451 100644
--- a/app/api/canvas/local-edit/route.ts
+++ b/app/api/canvas/local-edit/route.ts
@@ -17,6 +17,7 @@ import {
import {
modelLacksReferenceImageInputSupport,
} from "@/lib/ai/reference-image-capability";
+import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image";
import { getImageToolCreditCost } from "@/lib/points/config";
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
import { uploadFile } from "@/lib/storage/service";
@@ -278,6 +279,19 @@ function getScaleAdjustModelFamily(
);
}
+function isScaleAdjustMaskCapableModel(
+ entry: AiModelWithProviderForSelection | null | undefined
+) {
+ if (!entry || getScaleAdjustModelFamily(entry) !== "gpt-image-2") {
+ return false;
+ }
+ if (entry.provider.type === "openai-image") return true;
+ if (entry.provider.type === "fal-image") {
+ return isFalGptImage2EditModelId(entry.model.modelId || "");
+ }
+ return false;
+}
+
function isScaleAdjustCoordinateGuideModel(
entry: AiModelWithProviderForSelection | null | undefined
) {
@@ -1004,6 +1018,89 @@ function buildScaleAdjustTargetFootprintShadowSvg(params: {
);
}
+function getScaleAdjustEditMaskBoxes(params: {
+ regions: NormalizedLocalEditRegion[];
+ scalePercent?: ScaleAdjustPercent | null;
+ sourceHeight: number;
+ sourceWidth: number;
+}) {
+ const { regions, scalePercent, sourceHeight, sourceWidth } = params;
+ return regions.map((region) => {
+ const targetRect = getScaleAdjustTargetRect(region, scalePercent);
+ const workspaceRect = getScaleAdjustWorkspaceRect(
+ region.rect,
+ targetRect,
+ scalePercent
+ );
+ return rectToPixels(workspaceRect, sourceWidth, sourceHeight);
+ });
+}
+
+async function buildScaleAdjustEditMaskBuffer(params: {
+ regions: NormalizedLocalEditRegion[];
+ scalePercent?: ScaleAdjustPercent | null;
+ sourceHeight: number;
+ sourceWidth: number;
+}) {
+ const { regions, scalePercent, sourceHeight, sourceWidth } = params;
+ const boxes = getScaleAdjustEditMaskBoxes({
+ regions,
+ scalePercent,
+ sourceHeight,
+ sourceWidth,
+ });
+ const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight));
+ const blur = Math.max(5, Math.min(28, Math.round(minSide * 0.018)));
+ const pad = Math.max(2, Math.round(blur * 0.55));
+ const rects = boxes
+ .map((box) => {
+ const left = clamp(box.left - pad, 0, Math.max(0, sourceWidth - 1));
+ const top = clamp(box.top - pad, 0, Math.max(0, sourceHeight - 1));
+ const right = clamp(
+ box.left + box.width + pad,
+ left + 1,
+ sourceWidth
+ );
+ const bottom = clamp(
+ box.top + box.height + pad,
+ top + 1,
+ sourceHeight
+ );
+ return ``;
+ })
+ .join("");
+ const editMaskSvg = Buffer.from(
+ ``
+ );
+ const editAlpha = await sharp(editMaskSvg, { failOn: "none" })
+ .resize(sourceWidth, sourceHeight, { fit: "fill" })
+ .greyscale()
+ .raw()
+ .toBuffer();
+ const rgba = Buffer.alloc(sourceWidth * sourceHeight * 4);
+ for (let index = 0; index < sourceWidth * sourceHeight; index += 1) {
+ const offset = index * 4;
+ const editable = editAlpha[index] || 0;
+ rgba[offset] = 0;
+ rgba[offset + 1] = 0;
+ rgba[offset + 2] = 0;
+ rgba[offset + 3] = clamp(255 - editable, 0, 255);
+ }
+ return sharp(rgba, {
+ raw: { channels: 4, height: sourceHeight, width: sourceWidth },
+ })
+ .png({ compressionLevel: 9 })
+ .toBuffer();
+}
+
async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
regions: NormalizedLocalEditRegion[];
scalePercent?: ScaleAdjustPercent | null;
@@ -1999,8 +2096,29 @@ export async function POST(request: NextRequest) {
annotatedFilename,
toArrayBuffer(annotatedBuffer)
);
+ const useScaleAdjustEditMask =
+ isScaleAdjustOperation &&
+ isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry);
+ const scaleAdjustEditMaskUpload = useScaleAdjustEditMask
+ ? await uploadFile(
+ `canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random()
+ .toString(36)
+ .slice(2, 8)}.png`,
+ toArrayBuffer(
+ await buildScaleAdjustEditMaskBuffer({
+ regions,
+ scalePercent,
+ sourceHeight,
+ sourceWidth,
+ })
+ )
+ )
+ : null;
+ const shouldCreateScaleAdjustCropReferences =
+ isScaleAdjustOperation &&
+ (useScaleAdjustVisualLayoutGuide || !useScaleAdjustCoordinateGuide);
const scaleAdjustCropReferences: ScaleAdjustCropReference[] =
- isScaleAdjustOperation && useScaleAdjustVisualLayoutGuide
+ shouldCreateScaleAdjustCropReferences
? await createScaleAdjustCropReferences({
regions,
sourceBuffer,
@@ -2054,11 +2172,9 @@ export async function POST(request: NextRequest) {
const scaleAdjustVerificationConfig = isScaleAdjustOperation
? {
allowOutsideChangedWhenChangedInside: false,
+ allowDeterministicFallback: false,
protectedRegions: scaleAdjustCompositeRegions,
- rejectFailedCandidates:
- useScaleAdjustCoordinateGuide && Boolean(scaleAdjustModelFamily)
- ? true
- : undefined,
+ rejectFailedCandidates: true,
sourceImageUrl,
regions: regions.map((region) => region.rect),
regionPrompts: regions.map((region) => region.prompt),
@@ -2097,6 +2213,12 @@ export async function POST(request: NextRequest) {
: "scale-adjust-neutral-coordinate-map"
: "scale-adjust-full-frame-semantic-edit-map"
: "source-overlay",
+ scaleAdjustEditMaskApplied: Boolean(scaleAdjustEditMaskUpload),
+ scaleAdjustEditMaskUrl: scaleAdjustEditMaskUpload?.url || null,
+ scaleAdjustEditMaskProviderType:
+ useScaleAdjustEditMask
+ ? resolvedScaleAdjustModelEntry?.provider?.type || null
+ : null,
cleanSourceReferenceIndex: 1,
configuredDetailModelId: configuredDetailModel || null,
detailFixedModelApplied: Boolean(detailFixedModel),
@@ -2181,6 +2303,9 @@ export async function POST(request: NextRequest) {
: "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位",
prompt,
promptExtend: false,
+ providerImageParams: scaleAdjustEditMaskUpload
+ ? { maskImageUrl: scaleAdjustEditMaskUpload.url }
+ : undefined,
referenceImages,
scaleAdjustVerification: scaleAdjustVerificationConfig,
localEditComposite: isScaleAdjustOperation
diff --git a/lib/ai/providers/openai-image.ts b/lib/ai/providers/openai-image.ts
index 28eef4685..1dd0ee129 100644
--- a/lib/ai/providers/openai-image.ts
+++ b/lib/ai/providers/openai-image.ts
@@ -19,6 +19,7 @@ export interface OpenAIImageParams {
quality?: string;
background?: string;
outputFormat?: string;
+ maskImageUrl?: string;
}
interface UploadedImage {
@@ -643,6 +644,7 @@ async function requestOpenAIImageGeneration(params: {
aspectRatio?: string;
imageSizeLabel?: "1K" | "2K" | "4K";
imageUrls: string[];
+ maskImageUrl?: string;
quality?: string;
background?: string;
outputFormat?: string;
@@ -656,6 +658,7 @@ async function requestOpenAIImageGeneration(params: {
aspectRatio,
imageSizeLabel,
imageUrls,
+ maskImageUrl,
quality,
background,
outputFormat,
@@ -723,6 +726,10 @@ async function requestOpenAIImageGeneration(params: {
formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename);
}
}
+ if (maskImageUrl) {
+ const mask = await loadInputImage(maskImageUrl);
+ formData.append("mask", new Blob([mask.buffer], { type: mask.mimeType }), mask.filename);
+ }
const response = await fetch(endpoint, {
method: "POST",
@@ -898,6 +905,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) {
timeoutMs: resolveRequestTimeoutMs(baseURL, imageSize, activeModelId),
useBltcyImageCompatParams: shouldUseBltcyImageCompatParams(baseURL),
imageCount: imageUrls.length,
+ hasMask: Boolean(options.imageParams?.maskImageUrl),
baseURL: normalizeBaseURL(baseURL),
});
@@ -910,6 +918,7 @@ export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) {
aspectRatio: aspectRatio || undefined,
imageSizeLabel: imageSize,
imageUrls,
+ maskImageUrl: options.imageParams?.maskImageUrl,
quality,
background,
outputFormat,
diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts
index bda4f43a6..151146cb2 100644
--- a/lib/canvas/generate-image.ts
+++ b/lib/canvas/generate-image.ts
@@ -53,6 +53,7 @@ export interface CanvasLocalEditCompositeConfig {
}
export interface CanvasScaleAdjustVerificationConfig {
+ allowDeterministicFallback?: boolean;
allowOutsideChangedWhenChangedInside?: boolean;
protectedRegions?: CanvasLocalEditCompositeRegion[];
rejectFailedCandidates?: boolean;
@@ -75,6 +76,7 @@ export interface CanvasGenerateImageRetryVariant {
negativePrompt?: string;
prompt?: string;
promptExtend?: boolean;
+ providerImageParams?: Record;
referenceImage?: string;
referenceImages?: string[];
scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
@@ -92,6 +94,7 @@ export interface CanvasGenerateImageRequest {
falSystemPrompt?: string;
falThinkingLevel?: "minimal" | "high";
promptExtend?: boolean;
+ providerImageParams?: Record;
referenceImage?: string;
referenceImages?: string[];
billingFeature?: "multi-view" | "repaint";
@@ -395,15 +398,22 @@ function normalizeScaleAdjustVerificationConfig(
Boolean(region)
)
: [];
+ const regionPrompts = Array.isArray(value.regionPrompts)
+ ? value.regionPrompts
+ .map((prompt) => String(prompt || "").trim())
+ .slice(0, regions.length)
+ : [];
return regions.length > 0 &&
(targetRegions.length > 0 || typeof scalePercent === "number")
? {
sourceImageUrl,
regions,
+ allowDeterministicFallback: value.allowDeterministicFallback === true,
allowOutsideChangedWhenChangedInside:
value.allowOutsideChangedWhenChangedInside === true,
rejectFailedCandidates: value.rejectFailedCandidates === true,
+ ...(regionPrompts.length > 0 ? { regionPrompts } : {}),
...(targetRegions.length > 0 ? { targetRegions } : {}),
...(typeof scalePercent === "number" ? { scalePercent } : {}),
...(protectedRegions.length > 0 ? { protectedRegions } : {}),
@@ -2526,6 +2536,10 @@ export async function generateCanvasImage(
: 1;
const promptExtend =
typeof request.promptExtend === "boolean" ? request.promptExtend : undefined;
+ const providerImageParams =
+ request.providerImageParams && typeof request.providerImageParams === "object"
+ ? request.providerImageParams
+ : undefined;
const ownerKey =
typeof request.ownerKey === "string" && request.ownerKey.trim()
? request.ownerKey.trim()
@@ -2742,6 +2756,7 @@ export async function generateCanvasImage(
];
const imageParams = {
+ ...(providerImageParams || {}),
aspectRatio,
safetyTolerance: request.falSafetyTolerance,
imageSize,
@@ -3242,7 +3257,7 @@ export async function generateCanvasImage(
selectedModelId: selectedModel.model.modelId,
...debugMeta,
});
- if (scaleAdjustVerification) {
+ if (scaleAdjustVerification?.allowDeterministicFallback) {
try {
const fallbackImages = await buildScaleAdjustDeterministicFallbackImages({
debugLabel,