fix: edit scale adjust through ai patch workspace
This commit is contained in:
134
parent
89fca5d654
commit
90a9edcd56
@ -101,6 +101,8 @@ const MAX_REFERENCE_IMAGES_PER_REGION = 3;
|
||||
const MAX_SCALE_ADJUST_CROP_REFERENCES = 4;
|
||||
const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18;
|
||||
const SCALE_ADJUST_CROP_MAX_EDGE = 1536;
|
||||
const SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN = 0.045;
|
||||
const SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN = 0.18;
|
||||
const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb";
|
||||
const MIN_REGION_EDGE_PX = 24;
|
||||
const SCALE_ADJUST_OPERATION = "scale-adjust";
|
||||
@ -112,6 +114,17 @@ type ScaleAdjustCropReference = {
|
||||
regionIndex: number;
|
||||
url: string;
|
||||
};
|
||||
type ScaleAdjustPatchReference = {
|
||||
sourceCropUrl: string;
|
||||
targetDraftUrl: string;
|
||||
workspaceRect: NormalizedLocalEditRegion["rect"];
|
||||
workspacePixelRect: {
|
||||
left: number;
|
||||
top: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
};
|
||||
type AiModelWithProviderForSelection = Awaited<
|
||||
ReturnType<typeof getEnabledAiModelsWithProvider>
|
||||
>[number];
|
||||
@ -711,6 +724,39 @@ function buildScaleAdjustCompositeRegions(
|
||||
});
|
||||
}
|
||||
|
||||
function getScaleAdjustPatchWorkspaceRect(
|
||||
regions: NormalizedLocalEditRegion[],
|
||||
scalePercent?: ScaleAdjustPercent | null
|
||||
) {
|
||||
const workspaceRects = regions.map((region) => {
|
||||
const targetRect = getScaleAdjustTargetRect(region, scalePercent);
|
||||
const baseWorkspace = getScaleAdjustWorkspaceRect(
|
||||
region.rect,
|
||||
targetRect,
|
||||
scalePercent
|
||||
);
|
||||
const marginX = Math.min(
|
||||
SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN,
|
||||
Math.max(
|
||||
SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN,
|
||||
baseWorkspace.width * 0.22,
|
||||
region.rect.width * 0.18
|
||||
)
|
||||
);
|
||||
const marginY = Math.min(
|
||||
SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN,
|
||||
Math.max(
|
||||
SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN,
|
||||
baseWorkspace.height * 0.22,
|
||||
region.rect.height * 0.18
|
||||
)
|
||||
);
|
||||
return expandRectWithMargins(baseWorkspace, marginX, marginY);
|
||||
});
|
||||
|
||||
return workspaceRects.reduce((combined, rect) => getRectUnion(combined, rect));
|
||||
}
|
||||
|
||||
function expandPixelRectForCrop(params: {
|
||||
height: number;
|
||||
pixelRect: NormalizedLocalEditRegion["pixelRect"];
|
||||
@ -1187,6 +1233,54 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: {
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
async function createScaleAdjustPatchReference(params: {
|
||||
regions: NormalizedLocalEditRegion[];
|
||||
scalePercent?: ScaleAdjustPercent | null;
|
||||
sourceBuffer: Buffer;
|
||||
sourceHeight: number;
|
||||
sourceWidth: number;
|
||||
}): Promise<ScaleAdjustPatchReference> {
|
||||
const { regions, scalePercent, sourceBuffer, sourceHeight, sourceWidth } =
|
||||
params;
|
||||
const workspaceRect = getScaleAdjustPatchWorkspaceRect(regions, scalePercent);
|
||||
const workspacePixelRect = rectToPixels(workspaceRect, sourceWidth, sourceHeight);
|
||||
const targetDraftBuffer = await buildScaleAdjustVisualLayoutGuideBuffer({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
});
|
||||
const [sourceCropBuffer, targetDraftCropBuffer] = await Promise.all([
|
||||
sharp(sourceBuffer, { failOn: "none" })
|
||||
.extract(workspacePixelRect)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer(),
|
||||
sharp(targetDraftBuffer, { failOn: "none" })
|
||||
.extract(workspacePixelRect)
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer(),
|
||||
]);
|
||||
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const [sourceCropUpload, targetDraftUpload] = await Promise.all([
|
||||
uploadFile(
|
||||
`canvas-scale-adjust-workspace-source-${suffix}.png`,
|
||||
toArrayBuffer(sourceCropBuffer)
|
||||
),
|
||||
uploadFile(
|
||||
`canvas-scale-adjust-workspace-target-draft-${suffix}.png`,
|
||||
toArrayBuffer(targetDraftCropBuffer)
|
||||
),
|
||||
]);
|
||||
|
||||
return {
|
||||
sourceCropUrl: sourceCropUpload.url,
|
||||
targetDraftUrl: targetDraftUpload.url,
|
||||
workspacePixelRect,
|
||||
workspaceRect,
|
||||
};
|
||||
}
|
||||
|
||||
function wrapText(text: string, maxChars: number, maxLines = 3) {
|
||||
const compact = text.replace(/\s+/g, " ").trim();
|
||||
if (!compact) return [];
|
||||
@ -1922,6 +2016,98 @@ function buildScaleAdjustTextOnlyPrompt(
|
||||
].filter(Boolean).join("\n");
|
||||
}
|
||||
|
||||
function toPatchRelativePixelBox(
|
||||
box: { left: number; top: number; width: number; height: number },
|
||||
workspace: ScaleAdjustPatchReference["workspacePixelRect"]
|
||||
) {
|
||||
const left = clamp(box.left - workspace.left, 0, Math.max(0, workspace.width - 1));
|
||||
const top = clamp(box.top - workspace.top, 0, Math.max(0, workspace.height - 1));
|
||||
const right = clamp(
|
||||
box.left + box.width - workspace.left,
|
||||
left + 1,
|
||||
workspace.width
|
||||
);
|
||||
const bottom = clamp(
|
||||
box.top + box.height - workspace.top,
|
||||
top + 1,
|
||||
workspace.height
|
||||
);
|
||||
|
||||
return {
|
||||
left,
|
||||
top,
|
||||
width: Math.max(1, right - left),
|
||||
height: Math.max(1, bottom - top),
|
||||
};
|
||||
}
|
||||
|
||||
function buildScaleAdjustPatchPrompt(
|
||||
regions: NormalizedLocalEditRegion[],
|
||||
scalePercent: ScaleAdjustPercent | null | undefined,
|
||||
patchReference: ScaleAdjustPatchReference,
|
||||
sourceSize: { width: number; height: number },
|
||||
cropReferences: ScaleAdjustCropReference[] = []
|
||||
) {
|
||||
const cropLines = cropReferences
|
||||
.map(
|
||||
(reference, index) =>
|
||||
`Input image ${index + 3}: clean identity crop for Region ${reference.regionIndex}; use it only to recognize the target furniture shape and materials.`
|
||||
)
|
||||
.join("\n");
|
||||
const regionLines = regions
|
||||
.map((region) => {
|
||||
const cropIndex = cropReferences.findIndex(
|
||||
(reference) => reference.regionIndex === region.index
|
||||
);
|
||||
const targetRect = getScaleAdjustTargetRect(region, scalePercent);
|
||||
const targetPixelRect =
|
||||
region.targetPixelRect ||
|
||||
rectToPixels(targetRect, sourceSize.width, sourceSize.height);
|
||||
const sourceInPatch = toPatchRelativePixelBox(
|
||||
region.pixelRect,
|
||||
patchReference.workspacePixelRect
|
||||
);
|
||||
const targetInPatch = toPatchRelativePixelBox(
|
||||
targetPixelRect,
|
||||
patchReference.workspacePixelRect
|
||||
);
|
||||
const widthFactor =
|
||||
region.rect.width > 0 ? targetRect.width / region.rect.width : 1;
|
||||
const heightFactor =
|
||||
region.rect.height > 0 ? targetRect.height / region.rect.height : 1;
|
||||
const action =
|
||||
widthFactor < 0.92 || heightFactor < 0.92
|
||||
? "make the same furniture visibly smaller"
|
||||
: widthFactor > 1.08 || heightFactor > 1.08
|
||||
? "make the same furniture visibly larger"
|
||||
: "move the same furniture to the final footprint";
|
||||
const cropNote =
|
||||
cropIndex >= 0
|
||||
? ` Input image ${cropIndex + 3} is the clean identity reference for this exact furniture.`
|
||||
: "";
|
||||
return `Region ${region.index}: current search area inside input image 1 crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside the crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}.${cropNote}`;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
return [
|
||||
"Local furniture scale edit inside a cropped workspace.",
|
||||
`Return exactly one clean realistic crop with the same framing and aspect as input image 1. The crop represents full-photo pixels left=${patchReference.workspacePixelRect.left}, top=${patchReference.workspacePixelRect.top}, width=${patchReference.workspacePixelRect.width}, height=${patchReference.workspacePixelRect.height}.`,
|
||||
"Input image 1 is the exact crop from the original photo and is the locked crop base.",
|
||||
"Input image 2 is a rough target-layout draft of the same crop. It is not final artwork. Use it only to understand the desired furniture size, position, and newly visible background area. Remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft.",
|
||||
cropLines,
|
||||
"Only edit the selected movable furniture subject. Keep the crop borders, wall, window frame, floor, rug, sofa, lamps, decor, lighting, camera perspective, and all non-target furniture stable.",
|
||||
"When the final footprint is smaller, the selected furniture must clearly shrink; the old larger footprint outside the final footprint must become natural background/floor/rug/wall/shadow that matches the original crop.",
|
||||
"Keep the same furniture identity, material, perspective, object details, contact shadows, and natural aspect ratio. Do not stretch, squash, duplicate, or replace it with a different item.",
|
||||
"Do not output boxes, lines, labels, coordinate marks, UI marks, text, arrows, overlays, rough layout artifacts, rectangular patch edges, or a full-room image.",
|
||||
"中文硬约束：只输出输入图1同一裁切范围的干净局部照片；输入图2只是目标大小/位置草稿，不是要照抄的图层。目标家具必须相对原图产生明显比例变化，非目标背景和裁切边缘要稳定自然。",
|
||||
"",
|
||||
"Region instructions:",
|
||||
regionLines,
|
||||
"",
|
||||
"Success check before returning: the selected furniture footprint must be clearly closer to the final footprint than the current search area, and the crop must look like one natural photograph.",
|
||||
].filter(Boolean).join("\n");
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth();
|
||||
@ -2098,8 +2284,15 @@ export async function POST(request: NextRequest) {
|
||||
!useProviderTextOnlyCoordinatePrompt;
|
||||
const useScaleAdjustMarkedPhotoGuide =
|
||||
useScaleAdjustCoordinatePrompt && !useScaleAdjustVisualLayoutGuide;
|
||||
const useScaleAdjustEditMask =
|
||||
isScaleAdjustOperation &&
|
||||
isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry);
|
||||
const useScaleAdjustPatchWorkflow =
|
||||
isScaleAdjustOperation && !useScaleAdjustEditMask;
|
||||
const sendScaleAdjustAnnotatedReferenceToModel =
|
||||
isScaleAdjustOperation && !useProviderTextOnlyCoordinatePrompt;
|
||||
isScaleAdjustOperation &&
|
||||
!useScaleAdjustPatchWorkflow &&
|
||||
!useProviderTextOnlyCoordinatePrompt;
|
||||
const annotatedBuffer = isScaleAdjustOperation
|
||||
? useScaleAdjustCoordinatePrompt
|
||||
? useScaleAdjustVisualLayoutGuide
|
||||
@ -2156,9 +2349,6 @@ export async function POST(request: NextRequest) {
|
||||
annotatedFilename,
|
||||
toArrayBuffer(annotatedBuffer)
|
||||
);
|
||||
const useScaleAdjustEditMask =
|
||||
isScaleAdjustOperation &&
|
||||
isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry);
|
||||
const scaleAdjustEditMaskUpload = useScaleAdjustEditMask
|
||||
? await uploadFile(
|
||||
`canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random()
|
||||
@ -2174,9 +2364,19 @@ export async function POST(request: NextRequest) {
|
||||
)
|
||||
)
|
||||
: null;
|
||||
const scaleAdjustPatchReference = useScaleAdjustPatchWorkflow
|
||||
? await createScaleAdjustPatchReference({
|
||||
regions,
|
||||
scalePercent,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
: null;
|
||||
const shouldCreateScaleAdjustCropReferences =
|
||||
isScaleAdjustOperation &&
|
||||
(useScaleAdjustVisualLayoutGuide ||
|
||||
(useScaleAdjustPatchWorkflow ||
|
||||
useScaleAdjustVisualLayoutGuide ||
|
||||
!useScaleAdjustCoordinatePrompt ||
|
||||
useProviderTextOnlyCoordinatePrompt ||
|
||||
(useScaleAdjustMarkedPhotoGuide && !useFalScaleAdjustCoordinatePrompt));
|
||||
@ -2190,7 +2390,15 @@ export async function POST(request: NextRequest) {
|
||||
})
|
||||
: [];
|
||||
const prompt = isScaleAdjustOperation
|
||||
? useProviderTextOnlyCoordinatePrompt
|
||||
? scaleAdjustPatchReference
|
||||
? buildScaleAdjustPatchPrompt(
|
||||
regions,
|
||||
scalePercent,
|
||||
scaleAdjustPatchReference,
|
||||
{ width: sourceWidth, height: sourceHeight },
|
||||
scaleAdjustCropReferences
|
||||
)
|
||||
: useProviderTextOnlyCoordinatePrompt
|
||||
? buildScaleAdjustTextOnlyPrompt(
|
||||
regions,
|
||||
scalePercent,
|
||||
@ -2224,6 +2432,8 @@ export async function POST(request: NextRequest) {
|
||||
: "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target furniture, old-size target furniture, stretched furniture, squashed furniture, duplicate furniture, second copy, extra legs, extra rails, side panels, shelves, cabinets, lattice panels";
|
||||
const scaleAdjustTextOnlyNegativePrompt =
|
||||
"artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, full image redesign, full-frame zoom, crop, pan, pasted crop, hard patch edge, color mismatch, changed sofa, changed lamp, changed wall, changed rug, duplicate furniture, stretched furniture, distorted furniture";
|
||||
const scaleAdjustPatchNegativePrompt =
|
||||
"full room image, changed crop border, artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, guide mark, pasted crop, hard patch edge, color mismatch, duplicate furniture, unchanged target furniture, stretched furniture, distorted furniture";
|
||||
const manualRegionReferenceImageUrls = regions.flatMap((region) =>
|
||||
region.referenceImages.map((reference) => reference.url)
|
||||
);
|
||||
@ -2231,7 +2441,14 @@ export async function POST(request: NextRequest) {
|
||||
(reference) => reference.url
|
||||
);
|
||||
const referenceImages = isScaleAdjustOperation
|
||||
? [
|
||||
? scaleAdjustPatchReference
|
||||
? [
|
||||
scaleAdjustPatchReference.sourceCropUrl,
|
||||
scaleAdjustPatchReference.targetDraftUrl,
|
||||
...scaleAdjustCropReferenceImageUrls,
|
||||
...manualRegionReferenceImageUrls,
|
||||
]
|
||||
: [
|
||||
sourceImageUrl,
|
||||
...(sendScaleAdjustAnnotatedReferenceToModel
|
||||
? [annotatedUpload.url]
|
||||
@ -2243,11 +2460,15 @@ export async function POST(request: NextRequest) {
|
||||
const scaleAdjustCompositeRegions = isScaleAdjustOperation
|
||||
? buildScaleAdjustCompositeRegions(regions, scalePercent)
|
||||
: [];
|
||||
const scaleAdjustProtectedRegions =
|
||||
isScaleAdjustOperation && scaleAdjustPatchReference
|
||||
? [...scaleAdjustCompositeRegions, scaleAdjustPatchReference.workspaceRect]
|
||||
: scaleAdjustCompositeRegions;
|
||||
const scaleAdjustVerificationConfig = isScaleAdjustOperation
|
||||
? {
|
||||
allowOutsideChangedWhenChangedInside: false,
|
||||
allowDeterministicFallback: false,
|
||||
protectedRegions: scaleAdjustCompositeRegions,
|
||||
protectedRegions: scaleAdjustProtectedRegions,
|
||||
rejectFailedCandidates: true,
|
||||
sourceImageUrl,
|
||||
regions: regions.map((region) => region.rect),
|
||||
@ -2269,9 +2490,19 @@ export async function POST(request: NextRequest) {
|
||||
: {}),
|
||||
}
|
||||
: undefined;
|
||||
const resolvedRequestImageSize = scaleAdjustPatchReference
|
||||
? "1K"
|
||||
: typeof body.imageSize === "string" && body.imageSize.trim()
|
||||
? body.imageSize.trim()
|
||||
: "2K";
|
||||
const imageRequest: CanvasGenerateImageRequest = {
|
||||
allowFailover: isScaleAdjustOperation ? false : undefined,
|
||||
aspectRatio: resolvedAspectRatio,
|
||||
aspectRatio: scaleAdjustPatchReference
|
||||
? inferClosestAspectRatio(
|
||||
scaleAdjustPatchReference.workspacePixelRect.width,
|
||||
scaleAdjustPatchReference.workspacePixelRect.height
|
||||
)
|
||||
: resolvedAspectRatio,
|
||||
billingFeature: "repaint",
|
||||
debugLabel:
|
||||
operation === SCALE_ADJUST_OPERATION ? "canvas-scale-adjust" : "canvas-local-edit",
|
||||
@ -2303,6 +2534,15 @@ export async function POST(request: NextRequest) {
|
||||
operation,
|
||||
requestedModel: requestedModel || null,
|
||||
resolvedAspectRatio,
|
||||
scaleAdjustPatchWorkflowApplied: Boolean(scaleAdjustPatchReference),
|
||||
scaleAdjustPatchSourceCropUrl:
|
||||
scaleAdjustPatchReference?.sourceCropUrl || null,
|
||||
scaleAdjustPatchTargetDraftUrl:
|
||||
scaleAdjustPatchReference?.targetDraftUrl || null,
|
||||
scaleAdjustPatchWorkspacePixelRect:
|
||||
scaleAdjustPatchReference?.workspacePixelRect || null,
|
||||
scaleAdjustPatchWorkspaceRect:
|
||||
scaleAdjustPatchReference?.workspaceRect || null,
|
||||
resolvedModel: resolvedModel || null,
|
||||
resolvedScaleAdjustProviderType:
|
||||
resolvedScaleAdjustModelEntry?.provider?.type || null,
|
||||
@ -2349,7 +2589,9 @@ export async function POST(request: NextRequest) {
|
||||
scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect)
|
||||
.length,
|
||||
scaleAdjustPromptStrategy: isScaleAdjustOperation
|
||||
? useFalScaleAdjustCoordinatePrompt
|
||||
? scaleAdjustPatchReference
|
||||
? "ai-patch-workspace-draft-v70-one-shot"
|
||||
: useFalScaleAdjustCoordinatePrompt
|
||||
? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot"
|
||||
: useScaleAdjustVisualLayoutGuide
|
||||
? "same-name-focused-layout-crop-guide-v54-one-shot"
|
||||
@ -2359,8 +2601,9 @@ export async function POST(request: NextRequest) {
|
||||
? "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot"
|
||||
: "locked-plate-semantic-target-map-no-old-crop-v15"
|
||||
: undefined,
|
||||
scaleAdjustPostCompositeApplied: false,
|
||||
scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation,
|
||||
scaleAdjustPostCompositeApplied: Boolean(scaleAdjustPatchReference),
|
||||
scaleAdjustUsesFullFrameOutput:
|
||||
isScaleAdjustOperation && !scaleAdjustPatchReference,
|
||||
sourceHeight,
|
||||
sourceNodeId: body.sourceNodeId || null,
|
||||
sourceWidth,
|
||||
@ -2373,14 +2616,13 @@ export async function POST(request: NextRequest) {
|
||||
: undefined,
|
||||
falThinkingLevel: useScaleAdjustCoordinateGuide ? "high" : undefined,
|
||||
imageCount: 1,
|
||||
imageSize:
|
||||
typeof body.imageSize === "string" && body.imageSize.trim()
|
||||
? body.imageSize.trim()
|
||||
: "2K",
|
||||
imageSize: resolvedRequestImageSize,
|
||||
model: resolvedModel || undefined,
|
||||
negativePrompt:
|
||||
isScaleAdjustOperation
|
||||
? useProviderTextOnlyCoordinatePrompt
|
||||
? scaleAdjustPatchReference
|
||||
? scaleAdjustPatchNegativePrompt
|
||||
: useProviderTextOnlyCoordinatePrompt
|
||||
? scaleAdjustTextOnlyNegativePrompt
|
||||
: useScaleAdjustCoordinatePrompt
|
||||
? scaleAdjustCoordinateNegativePrompt
|
||||
@ -2392,6 +2634,15 @@ export async function POST(request: NextRequest) {
|
||||
? { maskImageUrl: scaleAdjustEditMaskUpload.url }
|
||||
: undefined,
|
||||
referenceImages,
|
||||
scaleAdjustPatchComposite: scaleAdjustPatchReference
|
||||
? {
|
||||
featherMaxPx: 42,
|
||||
featherMinPx: 10,
|
||||
featherRatio: 0.018,
|
||||
sourceImageUrl,
|
||||
workspaceRegion: scaleAdjustPatchReference.workspaceRect,
|
||||
}
|
||||
: undefined,
|
||||
scaleAdjustVerification: scaleAdjustVerificationConfig,
|
||||
localEditComposite: isScaleAdjustOperation
|
||||
? undefined
|
||||
|
||||
@ -52,6 +52,14 @@ export interface CanvasLocalEditCompositeConfig {
|
||||
diffCompareBlurPx?: number;
|
||||
}
|
||||
|
||||
export interface CanvasScaleAdjustPatchCompositeConfig {
|
||||
sourceImageUrl: string;
|
||||
workspaceRegion: CanvasLocalEditCompositeRegion;
|
||||
featherMinPx?: number;
|
||||
featherMaxPx?: number;
|
||||
featherRatio?: number;
|
||||
}
|
||||
|
||||
export interface CanvasScaleAdjustVerificationConfig {
|
||||
allowDeterministicFallback?: boolean;
|
||||
allowOutsideChangedWhenChangedInside?: boolean;
|
||||
@ -79,6 +87,7 @@ export interface CanvasGenerateImageRetryVariant {
|
||||
providerImageParams?: Record<string, unknown>;
|
||||
referenceImage?: string;
|
||||
referenceImages?: string[];
|
||||
scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig;
|
||||
scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
|
||||
}
|
||||
|
||||
@ -102,6 +111,7 @@ export interface CanvasGenerateImageRequest {
|
||||
debugLabel?: string;
|
||||
debugMeta?: Record<string, unknown>;
|
||||
localEditComposite?: CanvasLocalEditCompositeConfig;
|
||||
scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig;
|
||||
scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig;
|
||||
retryVariants?: CanvasGenerateImageRetryVariant[];
|
||||
internalRetryVariantIndex?: number;
|
||||
@ -352,6 +362,33 @@ function normalizeLocalEditCompositeConfig(
|
||||
: null;
|
||||
}
|
||||
|
||||
function normalizeScaleAdjustPatchCompositeConfig(
|
||||
value: CanvasGenerateImageRequest["scaleAdjustPatchComposite"]
|
||||
): CanvasScaleAdjustPatchCompositeConfig | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const sourceImageUrl = String(value.sourceImageUrl || "").trim();
|
||||
if (!/^https?:\/\//i.test(sourceImageUrl)) return null;
|
||||
const workspaceRegion = normalizeCanvasRegion(value.workspaceRegion);
|
||||
if (!workspaceRegion) return null;
|
||||
|
||||
return {
|
||||
sourceImageUrl,
|
||||
workspaceRegion,
|
||||
featherMaxPx: normalizeOptionalNumber(value.featherMaxPx, {
|
||||
max: 180,
|
||||
min: 0,
|
||||
}),
|
||||
featherMinPx: normalizeOptionalNumber(value.featherMinPx, {
|
||||
max: 100,
|
||||
min: 0,
|
||||
}),
|
||||
featherRatio: normalizeOptionalNumber(value.featherRatio, {
|
||||
max: 0.12,
|
||||
min: 0,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeCanvasRegion(region: unknown): CanvasLocalEditCompositeRegion | null {
|
||||
if (!region || typeof region !== "object") return null;
|
||||
const raw = region as Partial<CanvasLocalEditCompositeRegion>;
|
||||
@ -1896,6 +1933,54 @@ function buildLocalEditCompositeMaskSvg(params: {
|
||||
`);
|
||||
}
|
||||
|
||||
function buildScaleAdjustPatchCompositeMaskSvg(params: {
|
||||
width: number;
|
||||
height: number;
|
||||
featherPx: number;
|
||||
touchesLeft: boolean;
|
||||
touchesTop: boolean;
|
||||
touchesRight: boolean;
|
||||
touchesBottom: boolean;
|
||||
}) {
|
||||
const { height, width } = params;
|
||||
const featherPx = Math.max(0, Math.round(params.featherPx));
|
||||
if (featherPx <= 0) {
|
||||
return Buffer.from(`
|
||||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect x="0" y="0" width="${width}" height="${height}" fill="#ffffff" />
|
||||
</svg>
|
||||
`);
|
||||
}
|
||||
|
||||
const spill = Math.max(2, Math.round(featherPx * 2.5));
|
||||
const softLeft = params.touchesLeft ? -spill : 0;
|
||||
const softTop = params.touchesTop ? -spill : 0;
|
||||
const softRight = params.touchesRight ? width + spill : width;
|
||||
const softBottom = params.touchesBottom ? height + spill : height;
|
||||
const coreInset = Math.max(0, Math.round(featherPx * 1.25));
|
||||
const coreLeft = params.touchesLeft ? 0 : Math.min(coreInset, Math.floor(width / 3));
|
||||
const coreTop = params.touchesTop ? 0 : Math.min(coreInset, Math.floor(height / 3));
|
||||
const coreRight = params.touchesRight
|
||||
? width
|
||||
: Math.max(coreLeft + 1, width - Math.min(coreInset, Math.floor(width / 3)));
|
||||
const coreBottom = params.touchesBottom
|
||||
? height
|
||||
: Math.max(coreTop + 1, height - Math.min(coreInset, Math.floor(height / 3)));
|
||||
|
||||
return Buffer.from(`
|
||||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs>
|
||||
<filter id="scale-adjust-patch-soft-mask" x="-50%" y="-50%" width="200%" height="200%" color-interpolation-filters="sRGB">
|
||||
<feGaussianBlur stdDeviation="${featherPx}" />
|
||||
</filter>
|
||||
</defs>
|
||||
<rect x="0" y="0" width="${width}" height="${height}" fill="rgba(0,0,0,0)" />
|
||||
<rect x="${softLeft}" y="${softTop}" width="${Math.max(1, softRight - softLeft)}" height="${Math.max(1, softBottom - softTop)}" fill="#ffffff" filter="url(#scale-adjust-patch-soft-mask)" />
|
||||
<rect x="${coreLeft}" y="${coreTop}" width="${Math.max(1, coreRight - coreLeft)}" height="${Math.max(1, coreBottom - coreTop)}" fill="#ffffff" />
|
||||
</svg>
|
||||
`);
|
||||
}
|
||||
|
||||
function resolveLocalEditCompositeFeatherPx(params: {
|
||||
composite: CanvasLocalEditCompositeConfig;
|
||||
width: number;
|
||||
@ -2131,6 +2216,133 @@ async function applyLocalEditCompositeToImages(params: {
|
||||
return mergedImages;
|
||||
}
|
||||
|
||||
function resolveScaleAdjustPatchCompositeFeatherPx(params: {
|
||||
composite: CanvasScaleAdjustPatchCompositeConfig;
|
||||
width: number;
|
||||
height: number;
|
||||
}) {
|
||||
const { composite, height, width } = params;
|
||||
const minPx =
|
||||
typeof composite.featherMinPx === "number" ? composite.featherMinPx : 10;
|
||||
const maxPx =
|
||||
typeof composite.featherMaxPx === "number"
|
||||
? Math.max(minPx, composite.featherMaxPx)
|
||||
: 42;
|
||||
const ratio =
|
||||
typeof composite.featherRatio === "number" ? composite.featherRatio : 0.018;
|
||||
|
||||
return Math.max(
|
||||
minPx,
|
||||
Math.min(maxPx, Math.round(Math.min(width, height) * ratio))
|
||||
);
|
||||
}
|
||||
|
||||
async function applyScaleAdjustPatchCompositeToImages(params: {
|
||||
images: ParsedImage[];
|
||||
composite: CanvasScaleAdjustPatchCompositeConfig | null;
|
||||
debugLabel: string;
|
||||
}) {
|
||||
const { images, composite, debugLabel } = params;
|
||||
if (!composite || images.length === 0) return images;
|
||||
|
||||
const sourceBuffer = await sharp(
|
||||
await fetchImageBufferFromUrl(composite.sourceImageUrl),
|
||||
{ failOn: "none" }
|
||||
)
|
||||
.rotate()
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata();
|
||||
const sourceWidth = Number(sourceMetadata.width || 0);
|
||||
const sourceHeight = Number(sourceMetadata.height || 0);
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) {
|
||||
throw new Error("比例调整局部结果合成失败：无法识别原图尺寸");
|
||||
}
|
||||
|
||||
const workspaceBox = rectToPixelBox({
|
||||
height: sourceHeight,
|
||||
region: composite.workspaceRegion,
|
||||
width: sourceWidth,
|
||||
});
|
||||
const featherPx = resolveScaleAdjustPatchCompositeFeatherPx({
|
||||
composite,
|
||||
height: workspaceBox.height,
|
||||
width: workspaceBox.width,
|
||||
});
|
||||
const touchesLeft = workspaceBox.left <= 0;
|
||||
const touchesTop = workspaceBox.top <= 0;
|
||||
const touchesRight = workspaceBox.left + workspaceBox.width >= sourceWidth;
|
||||
const touchesBottom = workspaceBox.top + workspaceBox.height >= sourceHeight;
|
||||
const patchMask = buildScaleAdjustPatchCompositeMaskSvg({
|
||||
featherPx,
|
||||
height: workspaceBox.height,
|
||||
touchesBottom,
|
||||
touchesLeft,
|
||||
touchesRight,
|
||||
touchesTop,
|
||||
width: workspaceBox.width,
|
||||
});
|
||||
|
||||
const mergedImages: ParsedImage[] = [];
|
||||
for (const [index, image] of images.entries()) {
|
||||
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||||
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
||||
const patchOverlayBuffer = await sharp(generatedBuffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.resize(workspaceBox.width, workspaceBox.height, {
|
||||
fit: "cover",
|
||||
position: "center",
|
||||
})
|
||||
.ensureAlpha()
|
||||
.composite([{ input: patchMask, blend: "dest-in" }])
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.ensureAlpha()
|
||||
.composite([
|
||||
{
|
||||
input: patchOverlayBuffer,
|
||||
left: workspaceBox.left,
|
||||
top: workspaceBox.top,
|
||||
},
|
||||
])
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" })
|
||||
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const suffix = `${Date.now()}-${index + 1}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2, 8)}`;
|
||||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||||
uploadFile(
|
||||
`canvas-scale-adjust-ai-patch-composited-${suffix}.png`,
|
||||
bufferToArrayBuffer(mergedBuffer)
|
||||
),
|
||||
uploadFile(
|
||||
`canvas-scale-adjust-ai-patch-composited-thumb-${suffix}.png`,
|
||||
bufferToArrayBuffer(thumbnailBuffer)
|
||||
),
|
||||
]);
|
||||
mergedImages.push({
|
||||
originalUrl: originalUpload.url,
|
||||
thumbnailUrl: thumbnailUpload.url,
|
||||
});
|
||||
}
|
||||
|
||||
console.info("[Canvas Generate Image] Scale adjust AI patch composite applied:", {
|
||||
debugLabel,
|
||||
featherPx,
|
||||
imageCount: mergedImages.length,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
workspaceBox,
|
||||
});
|
||||
|
||||
return mergedImages;
|
||||
}
|
||||
|
||||
function getProviderBatchLimit(providerType: string) {
|
||||
switch (providerType) {
|
||||
case "ruxa-image":
|
||||
@ -2561,6 +2773,9 @@ export async function generateCanvasImage(
|
||||
const localEditComposite = normalizeLocalEditCompositeConfig(
|
||||
request.localEditComposite
|
||||
);
|
||||
const scaleAdjustPatchComposite = normalizeScaleAdjustPatchCompositeConfig(
|
||||
request.scaleAdjustPatchComposite
|
||||
);
|
||||
const scaleAdjustVerification = normalizeScaleAdjustVerificationConfig(
|
||||
request.scaleAdjustVerification
|
||||
);
|
||||
@ -2731,6 +2946,7 @@ export async function generateCanvasImage(
|
||||
providerType: provider.type,
|
||||
...debugMeta,
|
||||
hasLocalEditComposite: Boolean(localEditComposite),
|
||||
hasScaleAdjustPatchComposite: Boolean(scaleAdjustPatchComposite),
|
||||
hasScaleAdjustVerification: Boolean(scaleAdjustVerification),
|
||||
});
|
||||
|
||||
@ -3058,10 +3274,16 @@ export async function generateCanvasImage(
|
||||
debugLabel,
|
||||
images: baseImages,
|
||||
});
|
||||
const scaleAdjustPatchCompositedImages =
|
||||
await applyScaleAdjustPatchCompositeToImages({
|
||||
composite: scaleAdjustPatchComposite,
|
||||
debugLabel,
|
||||
images: compositedImages,
|
||||
});
|
||||
|
||||
const finalImages = await applyScaleAdjustVerificationToImages({
|
||||
debugLabel,
|
||||
images: compositedImages,
|
||||
images: scaleAdjustPatchCompositedImages,
|
||||
verification: scaleAdjustVerification,
|
||||
});
|
||||
console.info("[Canvas Generate Image] Completed:", {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user