fix: improve canvas local edit scale adjustment
This commit is contained in:
134
parent
f7d18f20db
commit
2b96fdb3e7
@ -697,28 +697,14 @@ function buildScaleAdjustPrompt(
|
||||
: "最终输出必须保持与输入图片1完全相同的画幅和像素尺寸。";
|
||||
const regionLines = regions
|
||||
.map((region) => {
|
||||
const left = Math.round(region.rect.x * 1000) / 10;
|
||||
const top = Math.round(region.rect.y * 1000) / 10;
|
||||
const width = Math.round(region.rect.width * 1000) / 10;
|
||||
const height = Math.round(region.rect.height * 1000) / 10;
|
||||
const targetRect = scaleRectAroundCenter(region.rect, scalePercent);
|
||||
const targetLeft = Math.round(targetRect.x * 1000) / 10;
|
||||
const targetTop = Math.round(targetRect.y * 1000) / 10;
|
||||
const targetWidth = Math.round(targetRect.width * 1000) / 10;
|
||||
const targetHeight = Math.round(targetRect.height * 1000) / 10;
|
||||
const pixel = region.pixelRect;
|
||||
const right = pixel.left + pixel.width;
|
||||
const bottom = pixel.top + pixel.height;
|
||||
const centerX = pixel.left + Math.round(pixel.width / 2);
|
||||
const centerY = pixel.top + Math.round(pixel.height / 2);
|
||||
const cropIndex = cropReferences.findIndex(
|
||||
(reference) => reference.regionIndex === region.index
|
||||
);
|
||||
const cropNote =
|
||||
cropIndex >= 0
|
||||
? `；输入图片${cropIndex + 3}是该区域的干净局部裁切参考图，只用于识别目标主体的轮廓与原始比例`
|
||||
? `；输入图片${cropIndex + 3}是该区域的干净局部裁切参考图，只用于识别目标主体的身份、轮廓与原始比例`
|
||||
: "";
|
||||
return `区域 ${region.index}（原始红色实线框：x=${left}%，y=${top}%，w=${width}%，h=${height}%；目标虚线框：x=${targetLeft}%，y=${targetTop}%，w=${targetWidth}%，h=${targetHeight}%；像素范围 left=${pixel.left}, top=${pixel.top}, right=${right}, bottom=${bottom}, center=(${centerX}, ${centerY})${cropNote}）：红色实线框只是“寻找目标主体”的范围，不是要缩放或贴回的矩形补丁；目标虚线框只是表达目标尺寸范围，不是要求主体填满矩形或变成矩形比例。请识别此范围内最主要的商品/物体主体，以主体自身轮廓、原始宽高比和视觉中心为准，执行整体等比${actionLabel}，宽度与高度必须使用同一个缩放倍率，目标主体视觉尺寸约为原主体的 ${targetLabel}。严禁宽度变窄但高度变高，严禁横向挤压、纵向拉伸或改变主体透视形体。如果框中只覆盖到主体的一部分，请根据连接关系推断该主体的完整可见结构，只调整该主体，不调整矩形框内背景。`;
|
||||
return `区域 ${region.index}${cropNote}：请在输入图片1中找到输入图片2红色实线框所标记范围内最主要、可独立缩放的商品/物体主体。红色实线框只是“寻找主体”的范围，不是要缩放或贴回的矩形补丁；目标虚线框只是表达目标尺寸范围，不是要求主体填满矩形或变成矩形比例。请以主体自身轮廓、原始宽高比和视觉中心为准，执行整体等比${actionLabel}，宽度与高度必须使用同一个缩放倍率，目标主体视觉尺寸约为原主体的 ${targetLabel}。严禁宽度变窄但高度变高，严禁横向挤压、纵向拉伸或改变主体透视形体。如果框中只覆盖到主体的一部分，请根据连接关系推断该主体的完整可见结构，只调整该主体，不调整矩形框内背景。`;
|
||||
})
|
||||
.join("\n");
|
||||
const cropLines = cropReferences
|
||||
@ -732,23 +718,23 @@ function buildScaleAdjustPrompt(
|
||||
"这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。",
|
||||
sourceSizeLine,
|
||||
"输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。",
|
||||
"输入图片2是低噪声定位参考图：红色实线框用于寻找当前主体，青绿色或绿色虚线框用于表达目标尺寸范围。输入图片2不是输出底图，不得学习或复刻它的框线、编号、半透明遮罩或 UI 标记。",
|
||||
"输入图片2是白底选区参考图，只用于说明输入图片1上的红色实线框和目标虚线框位置；它不包含场景内容，不是输出底图，不得学习或复刻它的框线、编号、半透明遮罩或 UI 标记。",
|
||||
cropLines
|
||||
? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}`
|
||||
: "",
|
||||
"输入图片2中的红色实线框表示目标主体当前大致位置；青绿色或绿色虚线框只表示比例调整后的安全尺寸范围。最终画面不得出现这些框线，主体也不需要填满虚线矩形；必须保持主体自身原始宽高比和透视形体。",
|
||||
"输入图片2中的红色实线框表示输入图片1里目标主体当前大致位置；青绿色或绿色虚线框只表示比例调整后的安全尺寸范围。最终画面不得出现这些框线，主体也不需要填满虚线矩形；必须保持主体自身原始宽高比和透视形体。",
|
||||
"如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。",
|
||||
"系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片或蒙版边界。",
|
||||
"系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是要整体缩放的矩形图层。",
|
||||
"最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度；严禁整张图整体缩放、裁切、平移、重新排版或重新生成。",
|
||||
"Highest priority rule: do NOT zoom, crop, resize, reframe, pan, or regenerate the full canvas. Treat image 1 as a locked background plate and only inpaint the selected subject footprint.",
|
||||
"请不要生成任何彩色框、矩形边框、编号圆点、提示词气泡、标注文字、UI 控件或半透明色块。",
|
||||
"",
|
||||
`任务：先在输入图片1中识别红色实线框附近最主要、可独立缩放的主体物体，再只对这个主体做局部比例调整，统一执行${actionLabel}。目标主体调整后的视觉尺寸必须约为原主体的 ${targetLabel}，普通用户肉眼应能清楚看出主体尺寸已经变化；如果输出看起来和原图一样，就是失败。`,
|
||||
`任务：先在输入图片1中识别红色实线框范围内最主要、可独立缩放的主体物体，再只对这个“主体物体”做局部比例调整，统一执行${actionLabel}。目标主体调整后的视觉尺寸必须约为原主体的 ${targetLabel}，普通用户肉眼应能清楚看出主体尺寸已经变化；如果输出看起来和原图一样，就是失败。`,
|
||||
regionLines,
|
||||
"",
|
||||
"关键要求：",
|
||||
`1. 只调整目标主体物体本身的整体视觉比例，必须等比缩放到约 ${targetLabel}，不得保留原尺寸；主体轮廓宽度和高度必须使用同一个倍率同步变化，宽高比必须与原主体一致，不能为了贴合虚线框而拉宽、压窄、拉高或压扁。`,
|
||||
"2. 严禁把整个矩形框选区域当成图片块整体缩放、平移、复制、裁切或贴回；最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。",
|
||||
"2. 严禁把整个矩形框选区域、输入图片1整张图或任意裁切图当成图片块整体缩放、平移、复制、裁切或贴回；最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。",
|
||||
"2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图；如果画面中的标题、Logo、页面文字、边框、背景家具或非目标主体发生整体位移/缩放/消失，必须视为失败并重新生成。",
|
||||
`3. 缩小时，目标主体应围绕原位置视觉中心自然变小到约 ${targetLabel}；主体缩小后，原来被遮挡的背景、桌面、墙面、阴影、纹理和透视必须被重新生成并无缝补全，选区内应出现明显新增留白或背景。`,
|
||||
`4. 放大时，目标主体应围绕原位置视觉中心自然变大到约 ${targetLabel}；宽度和高度都要同步变大，不能出现“高度变高但宽度变窄”的非等比变形；主体边缘与遮挡关系要合理，不能产生突兀裁切。`,
|
||||
@ -876,7 +862,17 @@ export async function POST(request: NextRequest) {
|
||||
regions,
|
||||
width: sourceWidth,
|
||||
});
|
||||
const annotatedBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||||
const annotatedBase = isScaleAdjustOperation
|
||||
? sharp({
|
||||
create: {
|
||||
background: "#ffffff",
|
||||
channels: 3,
|
||||
height: sourceHeight,
|
||||
width: sourceWidth,
|
||||
},
|
||||
})
|
||||
: sharp(sourceBuffer, { failOn: "none" });
|
||||
const annotatedBuffer = await annotatedBase
|
||||
.composite([{ input: annotationSvg, left: 0, top: 0 }])
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
@ -963,6 +959,9 @@ export async function POST(request: NextRequest) {
|
||||
debugMeta: {
|
||||
annotatedReferenceUrl: annotatedUpload.url,
|
||||
annotatedReferenceSentToModel: isScaleAdjustOperation,
|
||||
annotatedReferenceType: isScaleAdjustOperation
|
||||
? "white-selection-guide"
|
||||
: "source-overlay",
|
||||
cleanSourceReferenceIndex: 1,
|
||||
configuredDetailModelId: configuredDetailModel || null,
|
||||
detailFixedModelApplied: Boolean(detailFixedModel),
|
||||
@ -980,7 +979,7 @@ export async function POST(request: NextRequest) {
|
||||
scalePercent,
|
||||
scaleAdjustCropReferenceCount: scaleAdjustCropReferenceImageUrls.length,
|
||||
scaleAdjustPromptStrategy: isScaleAdjustOperation
|
||||
? "locked-plate-local-subject-v2"
|
||||
? "locked-plate-local-subject-v3"
|
||||
: undefined,
|
||||
scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation,
|
||||
sourceHeight,
|
||||
|
||||
1525
docs/无限画布对话框Agent模式升级开发文档-借鉴Cowart-2026-06-22.md
Normal file
1525
docs/无限画布对话框Agent模式升级开发文档-借鉴Cowart-2026-06-22.md
Normal file
File diff suppressed because it is too large
Load Diff
@ -129,6 +129,9 @@ const LOCAL_EDIT_COMPOSITE_DIFF_HIGH_THRESHOLD = 34;
|
||||
const SCALE_ADJUST_VISUAL_DIFF_THRESHOLD = 14;
|
||||
const SCALE_ADJUST_OUTSIDE_DIFF_THRESHOLD = 18;
|
||||
const SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO = 0.28;
|
||||
const SCALE_ADJUST_OUTSIDE_PIXEL_DIFF_THRESHOLD = 16;
|
||||
const SCALE_ADJUST_OUTSIDE_PIXEL_CHANGED_RATIO_THRESHOLD = 0.045;
|
||||
const SCALE_ADJUST_OUTSIDE_PIXEL_MEAN_DIFF_THRESHOLD = 4.5;
|
||||
const SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN = 0.015;
|
||||
const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112;
|
||||
|
||||
@ -632,6 +635,89 @@ function isScaleAdjustOutsideChanged(scores: number[]) {
|
||||
return changedCount >= requiredCount;
|
||||
}
|
||||
|
||||
async function getScaleAdjustOutsidePixelChangeStats(params: {
|
||||
generatedBuffer: Buffer;
|
||||
sourceBuffer: Buffer;
|
||||
sourceWidth: number;
|
||||
sourceHeight: number;
|
||||
verification: CanvasScaleAdjustVerificationConfig;
|
||||
}) {
|
||||
const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } =
|
||||
params;
|
||||
const protectedRegions = getScaleAdjustProtectedRegions(verification);
|
||||
const normalizeToRaw = (buffer: Buffer) =>
|
||||
sharp(buffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.resize(sourceWidth, sourceHeight, {
|
||||
fit: "cover",
|
||||
position: "center",
|
||||
})
|
||||
.removeAlpha()
|
||||
.toColorspace("srgb")
|
||||
.raw()
|
||||
.toBuffer();
|
||||
const [sourceRaw, generatedRaw] = await Promise.all([
|
||||
normalizeToRaw(sourceBuffer),
|
||||
normalizeToRaw(generatedBuffer),
|
||||
]);
|
||||
const step = Math.max(2, Math.ceil(Math.max(sourceWidth, sourceHeight) / 1200));
|
||||
let changedPixelCount = 0;
|
||||
let sampledPixelCount = 0;
|
||||
let diffSum = 0;
|
||||
|
||||
for (let y = 0; y < sourceHeight; y += step) {
|
||||
const normalizedY = y / sourceHeight;
|
||||
for (let x = 0; x < sourceWidth; x += step) {
|
||||
const normalizedX = x / sourceWidth;
|
||||
if (
|
||||
protectedRegions.some(
|
||||
(region) =>
|
||||
normalizedX >= region.x &&
|
||||
normalizedX <= region.x + region.width &&
|
||||
normalizedY >= region.y &&
|
||||
normalizedY <= region.y + region.height
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const rawIndex = (y * sourceWidth + x) * 3;
|
||||
const diff =
|
||||
Math.abs(generatedRaw[rawIndex] - sourceRaw[rawIndex]) * 0.299 +
|
||||
Math.abs(generatedRaw[rawIndex + 1] - sourceRaw[rawIndex + 1]) * 0.587 +
|
||||
Math.abs(generatedRaw[rawIndex + 2] - sourceRaw[rawIndex + 2]) * 0.114;
|
||||
diffSum += diff;
|
||||
sampledPixelCount += 1;
|
||||
if (diff >= SCALE_ADJUST_OUTSIDE_PIXEL_DIFF_THRESHOLD) {
|
||||
changedPixelCount += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const changedRatio =
|
||||
sampledPixelCount > 0 ? changedPixelCount / sampledPixelCount : 0;
|
||||
const meanDiff = sampledPixelCount > 0 ? diffSum / sampledPixelCount : 0;
|
||||
|
||||
return {
|
||||
changedPixelCount,
|
||||
changedRatio,
|
||||
meanDiff,
|
||||
sampledPixelCount,
|
||||
step,
|
||||
};
|
||||
}
|
||||
|
||||
function isScaleAdjustOutsidePixelChanged(params: {
|
||||
changedRatio: number;
|
||||
meanDiff: number;
|
||||
sampledPixelCount: number;
|
||||
}) {
|
||||
if (params.sampledPixelCount < 100) return false;
|
||||
return (
|
||||
params.changedRatio >= SCALE_ADJUST_OUTSIDE_PIXEL_CHANGED_RATIO_THRESHOLD &&
|
||||
params.meanDiff >= SCALE_ADJUST_OUTSIDE_PIXEL_MEAN_DIFF_THRESHOLD
|
||||
);
|
||||
}
|
||||
|
||||
async function loadScaleAdjustSourceImage(
|
||||
verification: CanvasScaleAdjustVerificationConfig
|
||||
) {
|
||||
@ -664,13 +750,86 @@ async function applyScaleAdjustVerificationToImages(params: {
|
||||
const { images, verification, debugLabel } = params;
|
||||
if (!verification || images.length === 0) return images;
|
||||
|
||||
console.info("[Canvas Generate Image] Scale adjust verification disabled:", {
|
||||
debugLabel,
|
||||
imageCount: images.length,
|
||||
protectedRegionCount: verification.protectedRegions?.length || 0,
|
||||
regionCount: verification.regions.length,
|
||||
scalePercent: verification.scalePercent,
|
||||
});
|
||||
const sourceImage = await loadScaleAdjustSourceImage(verification);
|
||||
if (!sourceImage) {
|
||||
console.warn("[Canvas Generate Image] Skip scale adjust result guard: source image unavailable", {
|
||||
debugLabel,
|
||||
imageCount: images.length,
|
||||
regionCount: verification.regions.length,
|
||||
scalePercent: verification.scalePercent,
|
||||
});
|
||||
return images;
|
||||
}
|
||||
|
||||
for (const [index, image] of images.entries()) {
|
||||
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||||
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
||||
const [diffScores, outsideDiffScores, outsidePixelStats] = await Promise.all([
|
||||
getScaleAdjustDiffScores({
|
||||
generatedBuffer,
|
||||
regions: verification.regions,
|
||||
sourceBuffer: sourceImage.sourceBuffer,
|
||||
sourceHeight: sourceImage.sourceHeight,
|
||||
sourceWidth: sourceImage.sourceWidth,
|
||||
}),
|
||||
getScaleAdjustOutsideDiffScores({
|
||||
generatedBuffer,
|
||||
sourceBuffer: sourceImage.sourceBuffer,
|
||||
sourceHeight: sourceImage.sourceHeight,
|
||||
sourceWidth: sourceImage.sourceWidth,
|
||||
verification,
|
||||
}),
|
||||
getScaleAdjustOutsidePixelChangeStats({
|
||||
generatedBuffer,
|
||||
sourceBuffer: sourceImage.sourceBuffer,
|
||||
sourceHeight: sourceImage.sourceHeight,
|
||||
sourceWidth: sourceImage.sourceWidth,
|
||||
verification,
|
||||
}),
|
||||
]);
|
||||
const changedInside = diffScores.some(
|
||||
(score) => score >= SCALE_ADJUST_VISUAL_DIFF_THRESHOLD
|
||||
);
|
||||
const outsideChanged =
|
||||
isScaleAdjustOutsideChanged(outsideDiffScores) ||
|
||||
isScaleAdjustOutsidePixelChanged(outsidePixelStats);
|
||||
|
||||
console.info("[Canvas Generate Image] Scale adjust result guard:", {
|
||||
debugLabel,
|
||||
changedInside,
|
||||
imageIndex: index,
|
||||
outsideChanged,
|
||||
outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))),
|
||||
outsidePixelStats: {
|
||||
changedPixelCount: outsidePixelStats.changedPixelCount,
|
||||
changedRatio: Number(outsidePixelStats.changedRatio.toFixed(4)),
|
||||
meanDiff: Number(outsidePixelStats.meanDiff.toFixed(2)),
|
||||
sampledPixelCount: outsidePixelStats.sampledPixelCount,
|
||||
step: outsidePixelStats.step,
|
||||
},
|
||||
protectedRegionCount: verification.protectedRegions?.length || 0,
|
||||
regionCount: verification.regions.length,
|
||||
regionDiffScores: diffScores.map((score) => Number(score.toFixed(2))),
|
||||
scalePercent: verification.scalePercent,
|
||||
});
|
||||
|
||||
if (outsideChanged) {
|
||||
throw new ScaleAdjustOutputRejectedError({
|
||||
diffScores,
|
||||
outsideDiffScores,
|
||||
reason: "outside-frame-changed",
|
||||
});
|
||||
}
|
||||
|
||||
if (!changedInside) {
|
||||
throw new ScaleAdjustOutputRejectedError({
|
||||
diffScores,
|
||||
outsideDiffScores,
|
||||
reason: "no-effective-scale-change",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return images;
|
||||
}
|
||||
|
||||
@ -1710,30 +1869,17 @@ export async function generateCanvasImage(
|
||||
}
|
||||
|
||||
const baseImages = collectedImages.slice(0, targetImageCount);
|
||||
const shouldCompositeBeforeScaleAdjustVerification = Boolean(
|
||||
scaleAdjustVerification && localEditComposite
|
||||
);
|
||||
const protectedImages = shouldCompositeBeforeScaleAdjustVerification
|
||||
? await applyLocalEditCompositeToImages({
|
||||
composite: localEditComposite,
|
||||
debugLabel,
|
||||
images: baseImages,
|
||||
})
|
||||
: baseImages;
|
||||
|
||||
const scaleCheckedImages = await applyScaleAdjustVerificationToImages({
|
||||
debugLabel,
|
||||
images: protectedImages,
|
||||
images: baseImages,
|
||||
verification: scaleAdjustVerification,
|
||||
});
|
||||
|
||||
const finalImages = shouldCompositeBeforeScaleAdjustVerification
|
||||
? scaleCheckedImages
|
||||
: await applyLocalEditCompositeToImages({
|
||||
composite: localEditComposite,
|
||||
debugLabel,
|
||||
images: scaleCheckedImages,
|
||||
});
|
||||
const finalImages = await applyLocalEditCompositeToImages({
|
||||
composite: localEditComposite,
|
||||
debugLabel,
|
||||
images: scaleCheckedImages,
|
||||
});
|
||||
console.info("[Canvas Generate Image] Completed:", {
|
||||
debugLabel,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user