fix: pass scale adjust crop references
This commit is contained in:
134
parent
eac5bb9296
commit
618bc2bd97
@ -77,11 +77,18 @@ type NormalizedLocalEditRegion = {
|
||||
|
||||
const MAX_LOCAL_EDIT_REGIONS = 8;
|
||||
const MAX_REFERENCE_IMAGES_PER_REGION = 3;
|
||||
const MAX_SCALE_ADJUST_CROP_REFERENCES = 4;
|
||||
const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18;
|
||||
const SCALE_ADJUST_CROP_MAX_EDGE = 1536;
|
||||
const MIN_REGION_EDGE_PX = 24;
|
||||
const SCALE_ADJUST_OPERATION = "scale-adjust";
|
||||
type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION;
|
||||
const SCALE_ADJUST_PERCENT_OPTIONS = [-30, -20, -10, 10, 20, 30] as const;
|
||||
type ScaleAdjustPercent = (typeof SCALE_ADJUST_PERCENT_OPTIONS)[number];
|
||||
type ScaleAdjustCropReference = {
|
||||
regionIndex: number;
|
||||
url: string;
|
||||
};
|
||||
const DEFAULT_REGION_COLORS = [
|
||||
"#ff3b30",
|
||||
"#1677ff",
|
||||
@ -316,6 +323,75 @@ function scaleRectAroundCenter(
|
||||
};
|
||||
}
|
||||
|
||||
function expandPixelRectForCrop(params: {
|
||||
height: number;
|
||||
pixelRect: NormalizedLocalEditRegion["pixelRect"];
|
||||
width: number;
|
||||
}) {
|
||||
const { height, pixelRect, width } = params;
|
||||
const padX = Math.round(pixelRect.width * SCALE_ADJUST_CROP_PADDING_RATIO);
|
||||
const padY = Math.round(pixelRect.height * SCALE_ADJUST_CROP_PADDING_RATIO);
|
||||
const left = clamp(pixelRect.left - padX, 0, Math.max(0, width - 1));
|
||||
const top = clamp(pixelRect.top - padY, 0, Math.max(0, height - 1));
|
||||
const right = clamp(
|
||||
pixelRect.left + pixelRect.width + padX,
|
||||
left + 1,
|
||||
width
|
||||
);
|
||||
const bottom = clamp(
|
||||
pixelRect.top + pixelRect.height + padY,
|
||||
top + 1,
|
||||
height
|
||||
);
|
||||
|
||||
return {
|
||||
left,
|
||||
top,
|
||||
width: Math.max(1, right - left),
|
||||
height: Math.max(1, bottom - top),
|
||||
};
|
||||
}
|
||||
|
||||
async function createScaleAdjustCropReferences(params: {
|
||||
regions: NormalizedLocalEditRegion[];
|
||||
sourceBuffer: Buffer;
|
||||
sourceHeight: number;
|
||||
sourceWidth: number;
|
||||
}): Promise<ScaleAdjustCropReference[]> {
|
||||
const references: ScaleAdjustCropReference[] = [];
|
||||
const regionsForCrop = params.regions.slice(0, MAX_SCALE_ADJUST_CROP_REFERENCES);
|
||||
|
||||
for (const region of regionsForCrop) {
|
||||
const cropBox = expandPixelRectForCrop({
|
||||
height: params.sourceHeight,
|
||||
pixelRect: region.pixelRect,
|
||||
width: params.sourceWidth,
|
||||
});
|
||||
const cropBuffer = await sharp(params.sourceBuffer, { failOn: "none" })
|
||||
.extract(cropBox)
|
||||
.resize({
|
||||
fit: "inside",
|
||||
height: SCALE_ADJUST_CROP_MAX_EDGE,
|
||||
withoutEnlargement: true,
|
||||
width: SCALE_ADJUST_CROP_MAX_EDGE,
|
||||
})
|
||||
.png({ compressionLevel: 6 })
|
||||
.toBuffer();
|
||||
const upload = await uploadFile(
|
||||
`canvas-scale-adjust-crop-r${region.index}-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2, 8)}.png`,
|
||||
toArrayBuffer(cropBuffer)
|
||||
);
|
||||
references.push({
|
||||
regionIndex: region.index,
|
||||
url: upload.url,
|
||||
});
|
||||
}
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
function wrapText(text: string, maxChars: number, maxLines = 3) {
|
||||
const compact = text.replace(/\s+/g, " ").trim();
|
||||
if (!compact) return [];
|
||||
@ -582,7 +658,8 @@ function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) {
|
||||
|
||||
function buildScaleAdjustPrompt(
|
||||
regions: NormalizedLocalEditRegion[],
|
||||
scalePercent: ScaleAdjustPercent
|
||||
scalePercent: ScaleAdjustPercent,
|
||||
cropReferences: ScaleAdjustCropReference[] = []
|
||||
) {
|
||||
const actionLabel = getScaleAdjustLabel(scalePercent);
|
||||
const targetLabel = getScaleAdjustTargetLabel(scalePercent);
|
||||
@ -602,13 +679,29 @@ function buildScaleAdjustPrompt(
|
||||
const bottom = pixel.top + pixel.height;
|
||||
const centerX = pixel.left + Math.round(pixel.width / 2);
|
||||
const centerY = pixel.top + Math.round(pixel.height / 2);
|
||||
return `区域 ${region.index}（原始红色实线框：x=${left}%，y=${top}%，w=${width}%，h=${height}%；目标虚线框：x=${targetLeft}%，y=${targetTop}%，w=${targetWidth}%，h=${targetHeight}%；像素范围 left=${pixel.left}, top=${pixel.top}, right=${right}, bottom=${bottom}, center=(${centerX}, ${centerY})）：红色实线框只是“寻找目标主体”的范围，不是要缩放或贴回的矩形补丁；目标虚线框只是表达目标尺寸范围，不是要求主体填满矩形或变成矩形比例。请识别此范围内最主要的商品/物体主体，以主体自身轮廓、原始宽高比和视觉中心为准，执行整体等比${actionLabel}，宽度与高度必须使用同一个缩放倍率，目标主体视觉尺寸约为原主体的 ${targetLabel}。严禁宽度变窄但高度变高，严禁横向挤压、纵向拉伸或改变主体透视形体。如果框中只覆盖到主体的一部分，请根据连接关系推断该主体的完整可见结构，只调整该主体，不调整矩形框内背景。`;
|
||||
const cropIndex = cropReferences.findIndex(
|
||||
(reference) => reference.regionIndex === region.index
|
||||
);
|
||||
const cropNote =
|
||||
cropIndex >= 0
|
||||
? `；输入图片${cropIndex + 3}是该区域的干净局部裁切参考图，只用于识别目标主体的轮廓与原始比例`
|
||||
: "";
|
||||
return `区域 ${region.index}（原始红色实线框：x=${left}%，y=${top}%，w=${width}%，h=${height}%；目标虚线框：x=${targetLeft}%，y=${targetTop}%，w=${targetWidth}%，h=${targetHeight}%；像素范围 left=${pixel.left}, top=${pixel.top}, right=${right}, bottom=${bottom}, center=(${centerX}, ${centerY})${cropNote}）：红色实线框只是“寻找目标主体”的范围，不是要缩放或贴回的矩形补丁；目标虚线框只是表达目标尺寸范围，不是要求主体填满矩形或变成矩形比例。请识别此范围内最主要的商品/物体主体，以主体自身轮廓、原始宽高比和视觉中心为准，执行整体等比${actionLabel}，宽度与高度必须使用同一个缩放倍率，目标主体视觉尺寸约为原主体的 ${targetLabel}。严禁宽度变窄但高度变高，严禁横向挤压、纵向拉伸或改变主体透视形体。如果框中只覆盖到主体的一部分，请根据连接关系推断该主体的完整可见结构，只调整该主体，不调整矩形框内背景。`;
|
||||
})
|
||||
.join("\n");
|
||||
const cropLines = cropReferences
|
||||
.map(
|
||||
(reference, index) =>
|
||||
`输入图片${index + 3}：区域 ${reference.regionIndex} 的原图干净局部裁切参考图，只用于确认要缩放的主体身份、轮廓和宽高比，不能当成输出底图，也不能把裁切图整体贴回。`
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
return [
|
||||
"输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。",
|
||||
"输入图片2是定位参考图：它只用于告诉你用户框选的目标主体位置和目标尺寸范围，不是输出底图。输入图片2里的彩色框、编号、标签、半透明遮罩、文字和 UI 只是定位标记，绝不能出现在最终画面中，也不得影响输入图片1的文字和排版。",
|
||||
cropLines
|
||||
? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}`
|
||||
: "",
|
||||
"输入图片2中的红色实线框表示目标主体当前大致位置；青绿色或绿色虚线框只表示比例调整后的安全尺寸范围。最终画面不得出现这些框线，主体也不需要填满虚线矩形；必须保持主体自身原始宽高比和透视形体。",
|
||||
"如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。",
|
||||
"系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片或蒙版边界。",
|
||||
@ -621,6 +714,7 @@ function buildScaleAdjustPrompt(
|
||||
"关键要求：",
|
||||
`1. 只调整目标主体物体本身的整体视觉比例，必须等比缩放到约 ${targetLabel}，不得保留原尺寸；主体轮廓宽度和高度必须使用同一个倍率同步变化，宽高比必须与原主体一致，不能为了贴合虚线框而拉宽、压窄、拉高或压扁。`,
|
||||
"2. 严禁把整个矩形框选区域当成图片块整体缩放、平移、复制、裁切或贴回；最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。",
|
||||
"2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图；如果画面中的标题、Logo、页面文字、边框、背景家具或非目标主体发生整体位移/缩放/消失，必须视为失败并重新生成。",
|
||||
`3. 缩小时，目标主体应围绕原位置视觉中心自然变小到约 ${targetLabel}；主体缩小后，原来被遮挡的背景、桌面、墙面、阴影、纹理和透视必须被重新生成并无缝补全，选区内应出现明显新增留白或背景。`,
|
||||
`4. 放大时，目标主体应围绕原位置视觉中心自然变大到约 ${targetLabel}；宽度和高度都要同步变大，不能出现“高度变高但宽度变窄”的非等比变形；主体边缘与遮挡关系要合理，不能产生突兀裁切。`,
|
||||
"5. 如果主体靠近画布边界导致目标尺寸空间不足，请用能完整放入画面的最大统一倍率整体缩放；绝不能为了塞进画面而只压缩宽度或只拉伸高度。",
|
||||
@ -741,16 +835,36 @@ export async function POST(request: NextRequest) {
|
||||
annotatedFilename,
|
||||
toArrayBuffer(annotatedBuffer)
|
||||
);
|
||||
const scaleAdjustCropReferences = isScaleAdjustOperation
|
||||
? await createScaleAdjustCropReferences({
|
||||
regions,
|
||||
sourceBuffer,
|
||||
sourceHeight,
|
||||
sourceWidth,
|
||||
})
|
||||
: [];
|
||||
const prompt =
|
||||
isScaleAdjustOperation
|
||||
? buildScaleAdjustPrompt(regions, scalePercent as ScaleAdjustPercent)
|
||||
? buildScaleAdjustPrompt(
|
||||
regions,
|
||||
scalePercent as ScaleAdjustPercent,
|
||||
scaleAdjustCropReferences
|
||||
)
|
||||
: buildLocalEditPrompt(regions);
|
||||
const regionReferenceImageUrls = regions.flatMap((region) =>
|
||||
const manualRegionReferenceImageUrls = regions.flatMap((region) =>
|
||||
region.referenceImages.map((reference) => reference.url)
|
||||
);
|
||||
const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map(
|
||||
(reference) => reference.url
|
||||
);
|
||||
const referenceImages = isScaleAdjustOperation
|
||||
? [sourceImageUrl, annotatedUpload.url, ...regionReferenceImageUrls]
|
||||
: [sourceImageUrl, ...regionReferenceImageUrls];
|
||||
? [
|
||||
sourceImageUrl,
|
||||
annotatedUpload.url,
|
||||
...scaleAdjustCropReferenceImageUrls,
|
||||
...manualRegionReferenceImageUrls,
|
||||
]
|
||||
: [sourceImageUrl, ...manualRegionReferenceImageUrls];
|
||||
const resolvedAspectRatio = resolveLocalEditAspectRatio(
|
||||
body.aspectRatio,
|
||||
sourceWidth,
|
||||
@ -782,9 +896,13 @@ export async function POST(request: NextRequest) {
|
||||
requestedModel: requestedModel || null,
|
||||
resolvedAspectRatio,
|
||||
resolvedModel: resolvedModel || null,
|
||||
regionReferenceImageCount: regionReferenceImageUrls.length,
|
||||
manualRegionReferenceImageCount: manualRegionReferenceImageUrls.length,
|
||||
regionReferenceImageCount:
|
||||
manualRegionReferenceImageUrls.length +
|
||||
scaleAdjustCropReferenceImageUrls.length,
|
||||
regionCount: regions.length,
|
||||
scalePercent,
|
||||
scaleAdjustCropReferenceCount: scaleAdjustCropReferenceImageUrls.length,
|
||||
scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation,
|
||||
sourceHeight,
|
||||
sourceNodeId: body.sourceNodeId || null,
|
||||
|
||||
@ -116,8 +116,8 @@ const LOCAL_EDIT_COMPOSITE_FEATHER_MIN_PX = 10;
|
||||
const LOCAL_EDIT_COMPOSITE_FEATHER_MAX_PX = 44;
|
||||
const SCALE_ADJUST_VISUAL_DIFF_THRESHOLD = 14;
|
||||
const SCALE_ADJUST_OUTSIDE_DIFF_THRESHOLD = 18;
|
||||
const SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO = 0.45;
|
||||
const SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN = 0.035;
|
||||
const SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO = 0.28;
|
||||
const SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN = 0.015;
|
||||
const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112;
|
||||
|
||||
const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
|
||||
@ -362,7 +362,7 @@ function getScaleAdjustOutsideSampleRegions(
|
||||
verification: CanvasScaleAdjustVerificationConfig
|
||||
) {
|
||||
const protectedRegions = getScaleAdjustProtectedRegions(verification);
|
||||
const candidates: CanvasLocalEditCompositeRegion[] = [
|
||||
const fixedCandidates: CanvasLocalEditCompositeRegion[] = [
|
||||
{ x: 0.03, y: 0.03, width: 0.2, height: 0.16 },
|
||||
{ x: 0.4, y: 0.03, width: 0.2, height: 0.16 },
|
||||
{ x: 0.77, y: 0.03, width: 0.2, height: 0.16 },
|
||||
@ -378,13 +378,42 @@ function getScaleAdjustOutsideSampleRegions(
|
||||
{ x: 0.28, y: 0.55, width: 0.18, height: 0.16 },
|
||||
{ x: 0.54, y: 0.55, width: 0.18, height: 0.16 },
|
||||
];
|
||||
const gridCandidates: CanvasLocalEditCompositeRegion[] = [];
|
||||
const columns = 7;
|
||||
const rows = 7;
|
||||
const cellWidth = 0.12;
|
||||
const cellHeight = 0.1;
|
||||
|
||||
return candidates.filter(
|
||||
(candidate) =>
|
||||
!protectedRegions.some((protectedRegion) =>
|
||||
canvasRegionsIntersect(candidate, protectedRegion)
|
||||
)
|
||||
);
|
||||
for (let row = 0; row < rows; row += 1) {
|
||||
for (let column = 0; column < columns; column += 1) {
|
||||
gridCandidates.push({
|
||||
x: (column / Math.max(1, columns - 1)) * (1 - cellWidth),
|
||||
y: (row / Math.max(1, rows - 1)) * (1 - cellHeight),
|
||||
width: cellWidth,
|
||||
height: cellHeight,
|
||||
});
|
||||
}
|
||||
}
|
||||
const candidates = [...fixedCandidates, ...gridCandidates];
|
||||
const uniqueCandidates = candidates.filter((candidate, index) => {
|
||||
const key = `${candidate.x.toFixed(3)}:${candidate.y.toFixed(3)}:${candidate.width.toFixed(3)}:${candidate.height.toFixed(3)}`;
|
||||
return (
|
||||
candidates.findIndex(
|
||||
(item) =>
|
||||
`${item.x.toFixed(3)}:${item.y.toFixed(3)}:${item.width.toFixed(3)}:${item.height.toFixed(3)}` ===
|
||||
key
|
||||
) === index
|
||||
);
|
||||
});
|
||||
|
||||
return uniqueCandidates
|
||||
.filter(
|
||||
(candidate) =>
|
||||
!protectedRegions.some((protectedRegion) =>
|
||||
canvasRegionsIntersect(candidate, protectedRegion)
|
||||
)
|
||||
)
|
||||
.slice(0, 28);
|
||||
}
|
||||
|
||||
function rectToPixelBox(params: {
|
||||
@ -524,7 +553,7 @@ function isScaleAdjustOutsideChanged(scores: number[]) {
|
||||
(score) => score >= SCALE_ADJUST_OUTSIDE_DIFF_THRESHOLD
|
||||
).length;
|
||||
const requiredCount = Math.max(
|
||||
2,
|
||||
3,
|
||||
Math.ceil(scores.length * SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO)
|
||||
);
|
||||
return changedCount >= requiredCount;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user