diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index faed4c1c0..497b972e1 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -77,11 +77,18 @@ type NormalizedLocalEditRegion = { const MAX_LOCAL_EDIT_REGIONS = 8; const MAX_REFERENCE_IMAGES_PER_REGION = 3; +const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; +const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; +const SCALE_ADJUST_CROP_MAX_EDGE = 1536; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; const SCALE_ADJUST_PERCENT_OPTIONS = [-30, -20, -10, 10, 20, 30] as const; type ScaleAdjustPercent = (typeof SCALE_ADJUST_PERCENT_OPTIONS)[number]; +type ScaleAdjustCropReference = { + regionIndex: number; + url: string; +}; const DEFAULT_REGION_COLORS = [ "#ff3b30", "#1677ff", @@ -316,6 +323,75 @@ function scaleRectAroundCenter( }; } +function expandPixelRectForCrop(params: { + height: number; + pixelRect: NormalizedLocalEditRegion["pixelRect"]; + width: number; +}) { + const { height, pixelRect, width } = params; + const padX = Math.round(pixelRect.width * SCALE_ADJUST_CROP_PADDING_RATIO); + const padY = Math.round(pixelRect.height * SCALE_ADJUST_CROP_PADDING_RATIO); + const left = clamp(pixelRect.left - padX, 0, Math.max(0, width - 1)); + const top = clamp(pixelRect.top - padY, 0, Math.max(0, height - 1)); + const right = clamp( + pixelRect.left + pixelRect.width + padX, + left + 1, + width + ); + const bottom = clamp( + pixelRect.top + pixelRect.height + padY, + top + 1, + height + ); + + return { + left, + top, + width: Math.max(1, right - left), + height: Math.max(1, bottom - top), + }; +} + +async function createScaleAdjustCropReferences(params: { + regions: NormalizedLocalEditRegion[]; + sourceBuffer: Buffer; + sourceHeight: number; + sourceWidth: number; +}): Promise { + const references: ScaleAdjustCropReference[] = []; + const regionsForCrop = params.regions.slice(0, MAX_SCALE_ADJUST_CROP_REFERENCES); + + for (const region of regionsForCrop) { + const cropBox = expandPixelRectForCrop({ + height: params.sourceHeight, + pixelRect: region.pixelRect, + width: params.sourceWidth, + }); + const cropBuffer = await sharp(params.sourceBuffer, { failOn: "none" }) + .extract(cropBox) + .resize({ + fit: "inside", + height: SCALE_ADJUST_CROP_MAX_EDGE, + withoutEnlargement: true, + width: SCALE_ADJUST_CROP_MAX_EDGE, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const upload = await uploadFile( + `canvas-scale-adjust-crop-r${region.index}-${Date.now()}-${Math.random() + .toString(36) + .slice(2, 8)}.png`, + toArrayBuffer(cropBuffer) + ); + references.push({ + regionIndex: region.index, + url: upload.url, + }); + } + + return references; +} + function wrapText(text: string, maxChars: number, maxLines = 3) { const compact = text.replace(/\s+/g, " ").trim(); if (!compact) return []; @@ -582,7 +658,8 @@ function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { function buildScaleAdjustPrompt( regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent + scalePercent: ScaleAdjustPercent, + cropReferences: ScaleAdjustCropReference[] = [] ) { const actionLabel = getScaleAdjustLabel(scalePercent); const targetLabel = getScaleAdjustTargetLabel(scalePercent); @@ -602,13 +679,29 @@ function buildScaleAdjustPrompt( const bottom = pixel.top + pixel.height; const centerX = pixel.left + Math.round(pixel.width / 2); const centerY = pixel.top + Math.round(pixel.height / 2); - return `区域 ${region.index}(原始红色实线框:x=${left}%,y=${top}%,w=${width}%,h=${height}%;目标虚线框:x=${targetLeft}%,y=${targetTop}%,w=${targetWidth}%,h=${targetHeight}%;像素范围 left=${pixel.left}, top=${pixel.top}, right=${right}, bottom=${bottom}, center=(${centerX}, ${centerY})):红色实线框只是“寻找目标主体”的范围,不是要缩放或贴回的矩形补丁;目标虚线框只是表达目标尺寸范围,不是要求主体填满矩形或变成矩形比例。请识别此范围内最主要的商品/物体主体,以主体自身轮廓、原始宽高比和视觉中心为准,执行整体等比${actionLabel},宽度与高度必须使用同一个缩放倍率,目标主体视觉尺寸约为原主体的 ${targetLabel}。严禁宽度变窄但高度变高,严禁横向挤压、纵向拉伸或改变主体透视形体。如果框中只覆盖到主体的一部分,请根据连接关系推断该主体的完整可见结构,只调整该主体,不调整矩形框内背景。`; + const cropIndex = cropReferences.findIndex( + (reference) => reference.regionIndex === region.index + ); + const cropNote = + cropIndex >= 0 + ? `;输入图片${cropIndex + 3}是该区域的干净局部裁切参考图,只用于识别目标主体的轮廓与原始比例` + : ""; + return `区域 ${region.index}(原始红色实线框:x=${left}%,y=${top}%,w=${width}%,h=${height}%;目标虚线框:x=${targetLeft}%,y=${targetTop}%,w=${targetWidth}%,h=${targetHeight}%;像素范围 left=${pixel.left}, top=${pixel.top}, right=${right}, bottom=${bottom}, center=(${centerX}, ${centerY})${cropNote}):红色实线框只是“寻找目标主体”的范围,不是要缩放或贴回的矩形补丁;目标虚线框只是表达目标尺寸范围,不是要求主体填满矩形或变成矩形比例。请识别此范围内最主要的商品/物体主体,以主体自身轮廓、原始宽高比和视觉中心为准,执行整体等比${actionLabel},宽度与高度必须使用同一个缩放倍率,目标主体视觉尺寸约为原主体的 ${targetLabel}。严禁宽度变窄但高度变高,严禁横向挤压、纵向拉伸或改变主体透视形体。如果框中只覆盖到主体的一部分,请根据连接关系推断该主体的完整可见结构,只调整该主体,不调整矩形框内背景。`; }) .join("\n"); + const cropLines = cropReferences + .map( + (reference, index) => + `输入图片${index + 3}:区域 ${reference.regionIndex} 的原图干净局部裁切参考图,只用于确认要缩放的主体身份、轮廓和宽高比,不能当成输出底图,也不能把裁切图整体贴回。` + ) + .join("\n"); return [ "输入图片1是原始干净图,也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外,输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", "输入图片2是定位参考图:它只用于告诉你用户框选的目标主体位置和目标尺寸范围,不是输出底图。输入图片2里的彩色框、编号、标签、半透明遮罩、文字和 UI 只是定位标记,绝不能出现在最终画面中,也不得影响输入图片1的文字和排版。", + cropLines + ? `输入图片3及之后是框选区域的干净局部裁切参考图,作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图,不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` + : "", "输入图片2中的红色实线框表示目标主体当前大致位置;青绿色或绿色虚线框只表示比例调整后的安全尺寸范围。最终画面不得出现这些框线,主体也不需要填满虚线矩形;必须保持主体自身原始宽高比和透视形体。", "如果系统或模型只能读取一张参考图,必须优先以输入图片1为输出基底,不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", "系统已根据用户框选区域计算出下方每个区域的原图坐标;框选区域只是定位目标主体的搜索范围,不是最终画面的裁切块、贴片或蒙版边界。", @@ -621,6 +714,7 @@ function buildScaleAdjustPrompt( "关键要求:", `1. 只调整目标主体物体本身的整体视觉比例,必须等比缩放到约 ${targetLabel},不得保留原尺寸;主体轮廓宽度和高度必须使用同一个倍率同步变化,宽高比必须与原主体一致,不能为了贴合虚线框而拉宽、压窄、拉高或压扁。`, "2. 严禁把整个矩形框选区域当成图片块整体缩放、平移、复制、裁切或贴回;最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。", + "2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图;如果画面中的标题、Logo、页面文字、边框、背景家具或非目标主体发生整体位移/缩放/消失,必须视为失败并重新生成。", `3. 缩小时,目标主体应围绕原位置视觉中心自然变小到约 ${targetLabel};主体缩小后,原来被遮挡的背景、桌面、墙面、阴影、纹理和透视必须被重新生成并无缝补全,选区内应出现明显新增留白或背景。`, `4. 放大时,目标主体应围绕原位置视觉中心自然变大到约 ${targetLabel};宽度和高度都要同步变大,不能出现“高度变高但宽度变窄”的非等比变形;主体边缘与遮挡关系要合理,不能产生突兀裁切。`, "5. 如果主体靠近画布边界导致目标尺寸空间不足,请用能完整放入画面的最大统一倍率整体缩放;绝不能为了塞进画面而只压缩宽度或只拉伸高度。", @@ -741,16 +835,36 @@ export async function POST(request: NextRequest) { annotatedFilename, toArrayBuffer(annotatedBuffer) ); + const scaleAdjustCropReferences = isScaleAdjustOperation + ? await createScaleAdjustCropReferences({ + regions, + sourceBuffer, + sourceHeight, + sourceWidth, + }) + : []; const prompt = isScaleAdjustOperation - ? buildScaleAdjustPrompt(regions, scalePercent as ScaleAdjustPercent) + ? buildScaleAdjustPrompt( + regions, + scalePercent as ScaleAdjustPercent, + scaleAdjustCropReferences + ) : buildLocalEditPrompt(regions); - const regionReferenceImageUrls = regions.flatMap((region) => + const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); + const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map( + (reference) => reference.url + ); const referenceImages = isScaleAdjustOperation - ? [sourceImageUrl, annotatedUpload.url, ...regionReferenceImageUrls] - : [sourceImageUrl, ...regionReferenceImageUrls]; + ? [ + sourceImageUrl, + annotatedUpload.url, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] + : [sourceImageUrl, ...manualRegionReferenceImageUrls]; const resolvedAspectRatio = resolveLocalEditAspectRatio( body.aspectRatio, sourceWidth, @@ -782,9 +896,13 @@ export async function POST(request: NextRequest) { requestedModel: requestedModel || null, resolvedAspectRatio, resolvedModel: resolvedModel || null, - regionReferenceImageCount: regionReferenceImageUrls.length, + manualRegionReferenceImageCount: manualRegionReferenceImageUrls.length, + regionReferenceImageCount: + manualRegionReferenceImageUrls.length + + scaleAdjustCropReferenceImageUrls.length, regionCount: regions.length, scalePercent, + scaleAdjustCropReferenceCount: scaleAdjustCropReferenceImageUrls.length, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation, sourceHeight, sourceNodeId: body.sourceNodeId || null, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index f9621e4b2..b0b2d90e6 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -116,8 +116,8 @@ const LOCAL_EDIT_COMPOSITE_FEATHER_MIN_PX = 10; const LOCAL_EDIT_COMPOSITE_FEATHER_MAX_PX = 44; const SCALE_ADJUST_VISUAL_DIFF_THRESHOLD = 14; const SCALE_ADJUST_OUTSIDE_DIFF_THRESHOLD = 18; -const SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO = 0.45; -const SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN = 0.035; +const SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO = 0.28; +const SCALE_ADJUST_OUTSIDE_PROTECTED_MARGIN = 0.015; const SCALE_ADJUST_DIFF_SAMPLE_SIZE = 112; const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ @@ -362,7 +362,7 @@ function getScaleAdjustOutsideSampleRegions( verification: CanvasScaleAdjustVerificationConfig ) { const protectedRegions = getScaleAdjustProtectedRegions(verification); - const candidates: CanvasLocalEditCompositeRegion[] = [ + const fixedCandidates: CanvasLocalEditCompositeRegion[] = [ { x: 0.03, y: 0.03, width: 0.2, height: 0.16 }, { x: 0.4, y: 0.03, width: 0.2, height: 0.16 }, { x: 0.77, y: 0.03, width: 0.2, height: 0.16 }, @@ -378,13 +378,42 @@ function getScaleAdjustOutsideSampleRegions( { x: 0.28, y: 0.55, width: 0.18, height: 0.16 }, { x: 0.54, y: 0.55, width: 0.18, height: 0.16 }, ]; + const gridCandidates: CanvasLocalEditCompositeRegion[] = []; + const columns = 7; + const rows = 7; + const cellWidth = 0.12; + const cellHeight = 0.1; - return candidates.filter( - (candidate) => - !protectedRegions.some((protectedRegion) => - canvasRegionsIntersect(candidate, protectedRegion) - ) - ); + for (let row = 0; row < rows; row += 1) { + for (let column = 0; column < columns; column += 1) { + gridCandidates.push({ + x: (column / Math.max(1, columns - 1)) * (1 - cellWidth), + y: (row / Math.max(1, rows - 1)) * (1 - cellHeight), + width: cellWidth, + height: cellHeight, + }); + } + } + const candidates = [...fixedCandidates, ...gridCandidates]; + const uniqueCandidates = candidates.filter((candidate, index) => { + const key = `${candidate.x.toFixed(3)}:${candidate.y.toFixed(3)}:${candidate.width.toFixed(3)}:${candidate.height.toFixed(3)}`; + return ( + candidates.findIndex( + (item) => + `${item.x.toFixed(3)}:${item.y.toFixed(3)}:${item.width.toFixed(3)}:${item.height.toFixed(3)}` === + key + ) === index + ); + }); + + return uniqueCandidates + .filter( + (candidate) => + !protectedRegions.some((protectedRegion) => + canvasRegionsIntersect(candidate, protectedRegion) + ) + ) + .slice(0, 28); } function rectToPixelBox(params: { @@ -524,7 +553,7 @@ function isScaleAdjustOutsideChanged(scores: number[]) { (score) => score >= SCALE_ADJUST_OUTSIDE_DIFF_THRESHOLD ).length; const requiredCount = Math.max( - 2, + 3, Math.ceil(scores.length * SCALE_ADJUST_OUTSIDE_DIFF_MIN_CHANGED_RATIO) ); return changedCount >= requiredCount;