fix: clarify scale adjust guide and retry failed FAL edits
This commit is contained in:
134
parent
d9f63bd00c
commit
d001dba00c
@ -232,6 +232,36 @@ function isFalImageModelEntry(
|
||||
return entry?.provider?.type === "fal-image";
|
||||
}
|
||||
|
||||
function isHealthyImageModelEntry(
|
||||
entry: AiModelWithProviderForSelection | null | undefined
|
||||
) {
|
||||
return (
|
||||
Boolean(entry?.model?.supportsImageGen) &&
|
||||
entry?.model?.supportsVision !== false &&
|
||||
entry?.model?.healthStatus === "healthy"
|
||||
);
|
||||
}
|
||||
|
||||
function findScaleAdjustRetryModelEntry(params: {
|
||||
currentModelId?: string | null;
|
||||
models: AiModelWithProviderForSelection[];
|
||||
}) {
|
||||
const excluded = String(params.currentModelId || "").trim();
|
||||
const isExcluded = (entry: AiModelWithProviderForSelection) =>
|
||||
Boolean(excluded) &&
|
||||
(entry.model.id === excluded || entry.model.modelId === excluded);
|
||||
|
||||
return (
|
||||
params.models.find(
|
||||
(entry) =>
|
||||
!isExcluded(entry) &&
|
||||
isHealthyImageModelEntry(entry) &&
|
||||
entry.provider.type === "snapgen-image" &&
|
||||
/gpt[-_ ]?image[-_ ]?2/i.test(entry.model.modelId || "")
|
||||
) || null
|
||||
);
|
||||
}
|
||||
|
||||
function resolveScaleAdjustPreferredModelId(params: {
|
||||
configuredModelId: string;
|
||||
requestedModelId: string;
|
||||
@ -781,6 +811,56 @@ function buildScaleAdjustAnnotationSvg(params: {
|
||||
`);
|
||||
}
|
||||
|
||||
function buildScaleAdjustPhotoGuideSvg(params: {
|
||||
width: number;
|
||||
height: number;
|
||||
regions: NormalizedLocalEditRegion[];
|
||||
scalePercent?: ScaleAdjustPercent | null;
|
||||
}) {
|
||||
const { width, height, regions, scalePercent } = params;
|
||||
const minSide = Math.max(1, Math.min(width, height));
|
||||
const strokeWidth = Math.max(4, Math.round(minSide * 0.006));
|
||||
const fontSize = Math.max(18, Math.round(minSide * 0.022));
|
||||
const labelPadX = Math.round(fontSize * 0.5);
|
||||
const labelPadY = Math.round(fontSize * 0.28);
|
||||
const sourceColor = "#ff3b30";
|
||||
const targetColor = "#00c8ff";
|
||||
|
||||
const label = (text: string, x: number, y: number, color: string) => {
|
||||
const labelWidth = Math.round(text.length * fontSize * 0.68 + labelPadX * 2);
|
||||
const labelHeight = Math.round(fontSize * 1.28 + labelPadY * 2);
|
||||
const left = clamp(x, 6, Math.max(6, width - labelWidth - 6));
|
||||
const top = clamp(y, 6, Math.max(6, height - labelHeight - 6));
|
||||
return `
|
||||
<rect x="${left}" y="${top}" width="${labelWidth}" height="${labelHeight}" rx="${Math.round(labelHeight * 0.26)}" fill="${color}" opacity="0.9" />
|
||||
<text x="${left + labelPadX}" y="${top + labelPadY + fontSize}" fill="#ffffff" font-size="${fontSize}" font-family="Arial, Helvetica, sans-serif" font-weight="800">${escapeXml(text)}</text>
|
||||
`;
|
||||
};
|
||||
|
||||
const regionSvg = regions
|
||||
.map((region) => {
|
||||
const source = region.pixelRect;
|
||||
const targetRect = getScaleAdjustTargetRect(region, scalePercent);
|
||||
const target =
|
||||
region.targetPixelRect || rectToPixels(targetRect, width, height);
|
||||
const radius = Math.max(8, Math.round(strokeWidth * 1.8));
|
||||
const sourceDash = `${Math.round(strokeWidth * 2.2)} ${Math.round(strokeWidth * 1.6)}`;
|
||||
return `
|
||||
<rect x="${source.left}" y="${source.top}" width="${source.width}" height="${source.height}" rx="${radius}" fill="none" stroke="${sourceColor}" stroke-width="${strokeWidth}" stroke-dasharray="${sourceDash}" opacity="0.92" />
|
||||
<rect x="${target.left}" y="${target.top}" width="${target.width}" height="${target.height}" rx="${radius}" fill="none" stroke="${targetColor}" stroke-width="${Math.max(4, Math.round(strokeWidth * 1.12))}" opacity="0.95" />
|
||||
${label(`OLD ${region.index}`, source.left + strokeWidth, source.top + strokeWidth, sourceColor)}
|
||||
${label(`TARGET ${region.index}`, target.left + strokeWidth, target.top + strokeWidth, targetColor)}
|
||||
`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
return Buffer.from(`
|
||||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||||
${regionSvg}
|
||||
</svg>
|
||||
`);
|
||||
}
|
||||
|
||||
function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) {
|
||||
const regionLines = regions
|
||||
.map((region) => {
|
||||
@ -942,7 +1022,7 @@ function buildFalScaleAdjustCoordinatePrompt(
|
||||
return [
|
||||
"Local furniture scale edit. Use input image 1 as the clean original photo and the locked camera frame.",
|
||||
`Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`,
|
||||
"Input image 2 is only a coordinate map. It is not output content. The OLD outline locates the current furniture in input image 1; the TARGET outline gives the final size and center.",
|
||||
"Input image 2 is an annotated copy of input image 1. It is only a visual instruction guide, never output content. The red dashed OLD outline locates the current furniture in input image 1; the cyan TARGET outline gives the final size and center.",
|
||||
"",
|
||||
"Task:",
|
||||
"Find the main movable furniture subject inside each OLD box in input image 1. Resize and, if needed, move that subject only so its final visible footprint matches the TARGET box.",
|
||||
@ -953,7 +1033,7 @@ function buildFalScaleAdjustCoordinatePrompt(
|
||||
regionLines,
|
||||
"",
|
||||
"Success check: the selected furniture size must clearly change from the original, while the rest of the photo stays stable.",
|
||||
"Do not paste rectangular image areas. Do not output coordinate lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.",
|
||||
"Do not paste rectangular image areas. Do not output the red/cyan guide lines, labels, boxes, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.",
|
||||
"Return only the clean edited photo.",
|
||||
].join("\n");
|
||||
}
|
||||
@ -964,7 +1044,7 @@ function buildFalScaleAdjustSystemPrompt() {
|
||||
"Interpret the input image as a soft layout guide, not as a pixel layer to copy.",
|
||||
"Generate a clean realistic final photo by editing the target furniture itself and naturally rebuilding only the tiny surrounding background that becomes visible.",
|
||||
"Do not paste, composite, clone, crop, resize, or copy rectangular image patches from the guide.",
|
||||
"Never output guide artifacts: no blurred blocks, seams, rectangular borders, arrows, labels, masks, UI marks, or color-coded annotations.",
|
||||
"Never output guide artifacts from the annotated reference: no red or cyan lines, labels, blurred blocks, seams, rectangular borders, arrows, masks, UI marks, or color-coded annotations.",
|
||||
"Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.",
|
||||
].join("\n");
|
||||
}
|
||||
@ -1128,12 +1208,34 @@ export async function POST(request: NextRequest) {
|
||||
: null;
|
||||
const useFalScaleAdjustCoordinateGuide =
|
||||
isScaleAdjustOperation && isFalImageModelEntry(resolvedScaleAdjustModelEntry);
|
||||
const scaleAdjustRetryModelEntry =
|
||||
useFalScaleAdjustCoordinateGuide
|
||||
? findScaleAdjustRetryModelEntry({
|
||||
currentModelId: resolvedModel || null,
|
||||
models: scaleAdjustModelCandidates,
|
||||
})
|
||||
: null;
|
||||
if (scaleAdjustRetryModelEntry) {
|
||||
console.info("[Canvas Local Edit] Prepared scale adjust retry model:", {
|
||||
primaryModel: resolvedScaleAdjustModelEntry?.model?.modelId || resolvedModel || null,
|
||||
primaryProvider: resolvedScaleAdjustModelEntry?.provider?.type || null,
|
||||
retryModel: scaleAdjustRetryModelEntry.model.modelId,
|
||||
retryProvider: scaleAdjustRetryModelEntry.provider.type,
|
||||
});
|
||||
}
|
||||
|
||||
const annotatedBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||||
.composite([
|
||||
{
|
||||
input: isScaleAdjustOperation
|
||||
? buildScaleAdjustAnnotationSvg({
|
||||
? useFalScaleAdjustCoordinateGuide
|
||||
? buildScaleAdjustPhotoGuideSvg({
|
||||
height: sourceHeight,
|
||||
regions,
|
||||
scalePercent,
|
||||
width: sourceWidth,
|
||||
})
|
||||
: buildScaleAdjustAnnotationSvg({
|
||||
height: sourceHeight,
|
||||
regions,
|
||||
scalePercent,
|
||||
@ -1229,6 +1331,24 @@ export async function POST(request: NextRequest) {
|
||||
: {}),
|
||||
}
|
||||
: undefined;
|
||||
const scaleAdjustRetryVariants =
|
||||
isScaleAdjustOperation && scaleAdjustRetryModelEntry
|
||||
? [
|
||||
{
|
||||
allowFailover: false,
|
||||
debugMeta: {
|
||||
scaleAdjustRetryAfterFalReject: true,
|
||||
scaleAdjustRetryModelId: scaleAdjustRetryModelEntry.model.modelId,
|
||||
scaleAdjustRetryProviderType:
|
||||
scaleAdjustRetryModelEntry.provider.type,
|
||||
scaleAdjustPromptStrategy:
|
||||
"scale-adjust-annotated-photo-snapgen-retry-v1",
|
||||
},
|
||||
label: "snapgen-gpt-image-2-after-fal-reject",
|
||||
model: scaleAdjustRetryModelEntry.model.id,
|
||||
},
|
||||
]
|
||||
: undefined;
|
||||
|
||||
const imageRequest: CanvasGenerateImageRequest = {
|
||||
aspectRatio: resolvedAspectRatio,
|
||||
@ -1240,7 +1360,7 @@ export async function POST(request: NextRequest) {
|
||||
annotatedReferenceSentToModel: isScaleAdjustOperation,
|
||||
annotatedReferenceType: isScaleAdjustOperation
|
||||
? useFalScaleAdjustCoordinateGuide
|
||||
? "scale-adjust-fal-coordinate-guide-only"
|
||||
? "scale-adjust-fal-annotated-photo-guide-only"
|
||||
: "scale-adjust-neutral-coordinate-guide-map"
|
||||
: "source-overlay",
|
||||
cleanSourceReferenceIndex: 1,
|
||||
@ -1268,7 +1388,7 @@ export async function POST(request: NextRequest) {
|
||||
.length,
|
||||
scaleAdjustPromptStrategy: isScaleAdjustOperation
|
||||
? useFalScaleAdjustCoordinateGuide
|
||||
? "fal-locked-source-coordinate-single-v24"
|
||||
? "fal-locked-source-annotated-photo-single-v25"
|
||||
: "locked-plate-neutral-coordinate-guide-map-pixel-anchors-v8"
|
||||
: undefined,
|
||||
scaleAdjustPostCompositeApplied: false,
|
||||
@ -1298,7 +1418,7 @@ export async function POST(request: NextRequest) {
|
||||
prompt,
|
||||
promptExtend: false,
|
||||
referenceImages,
|
||||
retryVariants: undefined,
|
||||
retryVariants: scaleAdjustRetryVariants,
|
||||
scaleAdjustVerification: scaleAdjustVerificationConfig,
|
||||
localEditComposite: isScaleAdjustOperation
|
||||
? undefined
|
||||
|
||||
@ -62,12 +62,14 @@ export interface CanvasScaleAdjustVerificationConfig {
|
||||
}
|
||||
|
||||
export interface CanvasGenerateImageRetryVariant {
|
||||
allowFailover?: boolean;
|
||||
debugMeta?: Record<string, unknown>;
|
||||
falSafetyTolerance?: string | number;
|
||||
falSystemPrompt?: string;
|
||||
falThinkingLevel?: "minimal" | "high";
|
||||
imageCount?: number;
|
||||
label?: string;
|
||||
model?: string;
|
||||
negativePrompt?: string;
|
||||
prompt?: string;
|
||||
promptExtend?: boolean;
|
||||
@ -654,6 +656,32 @@ async function getScaleAdjustDiffScores(params: {
|
||||
return scores;
|
||||
}
|
||||
|
||||
function getScaleAdjustEffectiveChangeRegions(
|
||||
verification: CanvasScaleAdjustVerificationConfig
|
||||
) {
|
||||
const regions: CanvasLocalEditCompositeRegion[] = [];
|
||||
for (const [index, region] of verification.regions.entries()) {
|
||||
regions.push(region);
|
||||
const targetRegion = verification.targetRegions?.[index];
|
||||
if (targetRegion) {
|
||||
regions.push(targetRegion);
|
||||
} else if (typeof verification.scalePercent === "number") {
|
||||
regions.push(scaleRegionAroundCenter(region, verification.scalePercent));
|
||||
}
|
||||
}
|
||||
|
||||
return regions.filter((region, index) => {
|
||||
const key = `${region.x.toFixed(4)}:${region.y.toFixed(4)}:${region.width.toFixed(4)}:${region.height.toFixed(4)}`;
|
||||
return (
|
||||
regions.findIndex(
|
||||
(item) =>
|
||||
`${item.x.toFixed(4)}:${item.y.toFixed(4)}:${item.width.toFixed(4)}:${item.height.toFixed(4)}` ===
|
||||
key
|
||||
) === index
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function getScaleAdjustOutsideDiffScores(params: {
|
||||
generatedBuffer: Buffer;
|
||||
sourceBuffer: Buffer;
|
||||
@ -999,6 +1027,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
}
|
||||
|
||||
const evaluatedImages: Array<{
|
||||
changeDiffScores: number[];
|
||||
changedInside: boolean;
|
||||
cleanedImage?: ParsedImage;
|
||||
diffScores: number[];
|
||||
@ -1013,7 +1042,8 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
for (const [index, image] of images.entries()) {
|
||||
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||||
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
||||
const [diffScores, outsideDiffScores, outsidePixelStats] = await Promise.all([
|
||||
const changeRegions = getScaleAdjustEffectiveChangeRegions(verification);
|
||||
const [diffScores, changeDiffScores, outsideDiffScores, outsidePixelStats] = await Promise.all([
|
||||
getScaleAdjustDiffScores({
|
||||
generatedBuffer,
|
||||
regions: verification.regions,
|
||||
@ -1021,6 +1051,13 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
sourceHeight: sourceImage.sourceHeight,
|
||||
sourceWidth: sourceImage.sourceWidth,
|
||||
}),
|
||||
getScaleAdjustDiffScores({
|
||||
generatedBuffer,
|
||||
regions: changeRegions,
|
||||
sourceBuffer: sourceImage.sourceBuffer,
|
||||
sourceHeight: sourceImage.sourceHeight,
|
||||
sourceWidth: sourceImage.sourceWidth,
|
||||
}),
|
||||
getScaleAdjustOutsideDiffScores({
|
||||
generatedBuffer,
|
||||
sourceBuffer: sourceImage.sourceBuffer,
|
||||
@ -1036,7 +1073,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
verification,
|
||||
}),
|
||||
]);
|
||||
const changedInside = diffScores.some(
|
||||
const changedInside = changeDiffScores.some(
|
||||
(score) => score >= SCALE_ADJUST_VISUAL_DIFF_THRESHOLD
|
||||
);
|
||||
const outsideChanged =
|
||||
@ -1056,6 +1093,9 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
sampledPixelCount: outsidePixelStats.sampledPixelCount,
|
||||
step: outsidePixelStats.step,
|
||||
},
|
||||
changeRegionDiffScores: changeDiffScores.map((score) =>
|
||||
Number(score.toFixed(2))
|
||||
),
|
||||
protectedRegionCount: verification.protectedRegions?.length || 0,
|
||||
regionCount: verification.regions.length,
|
||||
regionDiffScores: diffScores.map((score) => Number(score.toFixed(2))),
|
||||
@ -1082,6 +1122,9 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
sampledPixelCount: outsidePixelStats.sampledPixelCount,
|
||||
step: outsidePixelStats.step,
|
||||
},
|
||||
changeRegionDiffScores: changeDiffScores.map((score) =>
|
||||
Number(score.toFixed(2))
|
||||
),
|
||||
regionDiffScores: diffScores.map((score) => Number(score.toFixed(2))),
|
||||
scalePercent: verification.scalePercent ?? null,
|
||||
targetRegionCount: verification.targetRegions?.length || 0,
|
||||
@ -1101,6 +1144,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
});
|
||||
if (cleanedImage) {
|
||||
evaluatedImages.push({
|
||||
changeDiffScores,
|
||||
changedInside,
|
||||
cleanedImage,
|
||||
diffScores,
|
||||
@ -1121,6 +1165,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
}
|
||||
|
||||
evaluatedImages.push({
|
||||
changeDiffScores,
|
||||
changedInside,
|
||||
cleanedImage,
|
||||
diffScores,
|
||||
@ -1135,7 +1180,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
if (evaluatedImages.length === 0) return images;
|
||||
|
||||
const scoreEvaluation = (item: (typeof evaluatedImages)[number]) => {
|
||||
const insideScore = Math.max(...item.diffScores, 0);
|
||||
const insideScore = Math.max(...item.changeDiffScores, ...item.diffScores, 0);
|
||||
const outsideMean = item.outsidePixelStats.meanDiff;
|
||||
const outsidePenalty = item.outsideChanged ? 100 : 0;
|
||||
const noChangePenalty = item.changedInside ? 0 : 60;
|
||||
@ -1156,6 +1201,9 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
passed: selected.changedInside && !selected.outsideChanged,
|
||||
changedInside: selected.changedInside,
|
||||
outsideChanged: selected.outsideChanged,
|
||||
changeRegionDiffScores: selected.changeDiffScores.map((score) =>
|
||||
Number(score.toFixed(2))
|
||||
),
|
||||
regionDiffScores: selected.diffScores.map((score) => Number(score.toFixed(2))),
|
||||
outsidePixelStats: {
|
||||
changedRatio: Number(selected.outsidePixelStats.changedRatio.toFixed(4)),
|
||||
@ -1176,6 +1224,9 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
changedRatio: Number(selected.outsidePixelStats.changedRatio.toFixed(4)),
|
||||
meanDiff: Number(selected.outsidePixelStats.meanDiff.toFixed(2)),
|
||||
},
|
||||
changeRegionDiffScores: selected.changeDiffScores.map((score) =>
|
||||
Number(score.toFixed(2))
|
||||
),
|
||||
regionDiffScores: selected.diffScores.map((score) => Number(score.toFixed(2))),
|
||||
});
|
||||
}
|
||||
@ -1188,7 +1239,7 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
reason: selected.outsideChanged
|
||||
? "outside-frame-changed"
|
||||
: "no-effective-scale-change",
|
||||
diffScores: selected.diffScores,
|
||||
diffScores: selected.changeDiffScores,
|
||||
outsideDiffScores: selected.outsideDiffScores,
|
||||
});
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user