diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 9a4193877..c9ee44a1e 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -649,7 +649,13 @@ export async function POST(request: NextRequest) { targetRect: getScaleAdjustTargetRect(region, scalePercent), }) ); + const hasPositionMove = scaleAdjustIntentRegions.some( + (region) => + Math.abs(region.targetRect.x - region.rect.x) >= 0.005 || + Math.abs(region.targetRect.y - region.rect.y) >= 0.005 + ); const productLockAssets = await buildScaleAdjustProductLockAssets({ + protectPeople: hasPositionMove, regions: scaleAdjustIntentRegions, sourceBuffer, sourceHeight, @@ -660,42 +666,75 @@ export async function POST(request: NextRequest) { const assetSuffix = `${Date.now()}-${Math.random() .toString(36) .slice(2, 8)}`; - const [modelMaskUpload, overlayUpload, repairMaskUpload] = await Promise.all([ - uploadFile( - `canvas-scale-adjust-mask-${assetSuffix}.png`, - toArrayBuffer(productLockAssets.modelMaskBuffer) - ), - uploadFile( - `canvas-scale-adjust-overlay-${assetSuffix}.png`, - toArrayBuffer(productLockAssets.overlayBuffer) - ), - uploadFile( - `canvas-scale-adjust-repair-mask-${assetSuffix}.png`, - toArrayBuffer(productLockAssets.repairMaskBuffer) - ), - ]); + const [modelMaskUpload, overlayUpload, repairMaskUpload, sourceSubjectMaskUpload] = + await Promise.all([ + uploadFile( + `canvas-scale-adjust-mask-${assetSuffix}.png`, + toArrayBuffer(productLockAssets.modelMaskBuffer) + ), + uploadFile( + `canvas-scale-adjust-overlay-${assetSuffix}.png`, + toArrayBuffer(productLockAssets.overlayBuffer) + ), + uploadFile( + `canvas-scale-adjust-repair-mask-${assetSuffix}.png`, + toArrayBuffer(productLockAssets.repairMaskBuffer) + ), + hasPositionMove + ? uploadFile( + `canvas-scale-adjust-subject-mask-${assetSuffix}.png`, + toArrayBuffer(productLockAssets.sourceSubjectMaskBuffer) + ) + : Promise.resolve(null), + ]); const backgroundPlateUpload = productLockAssets.backgroundPlateBuffer ? await uploadFile( `canvas-scale-adjust-background-${assetSuffix}.png`, toArrayBuffer(productLockAssets.backgroundPlateBuffer) ) : null; - const prompt = "移除掩膜中的完整产品并自然还原背景。"; - const providerImageParams = { - objectRemovalMaskImageUrl: modelMaskUpload.url, - objectRemovalMode: "premium", - }; + const prompt = hasPositionMove + ? "只编辑白色掩膜区域,完整移除旧位置的产品及其原有阴影,并根据周围墙面、地面、家具和光照连续自然地重建被遮挡背景。黑色区域保持不变,不要保留产品的任何部件或残影。" + : "移除掩膜中的完整产品并自然还原背景。"; + const providerImageParams = hasPositionMove + ? { + maskImageUrl: modelMaskUpload.url, + quality: "medium", + } + : { + objectRemovalMaskImageUrl: modelMaskUpload.url, + objectRemovalMode: "premium", + }; const productLockComposite = { ...(backgroundPlateUpload ? { backgroundPlateImageUrl: backgroundPlateUpload.url } : {}), overlayImageUrl: overlayUpload.url, + ...(hasPositionMove + ? { + repairMaskImageUrl: repairMaskUpload.url, + sourceImageUrl, + } + : {}), + ...(hasPositionMove && sourceSubjectMaskUpload + ? { + regions: scaleAdjustIntentRegions.map((region) => ({ + source: region.rect, + target: region.targetRect, + })), + shadowSourceImageUrl: sourceImageUrl, + sourceSubjectMaskImageUrl: sourceSubjectMaskUpload.url, + } + : {}), sourceHeight, sourceWidth, }; const scaleAdjustVerification = { allowOutsideChangedWhenChangedInside: false, - oldFootprintMaskImageUrl: repairMaskUpload.url, + deterministicProductOverlayApplied: true, + oldFootprintMaskImageUrl: hasPositionMove + ? sourceSubjectMaskUpload?.url + : repairMaskUpload.url, regions: scaleAdjustIntentRegions.map((region) => region.rect), rejectFailedCandidates: true, rejectNoEffectiveChangeOnly: false, @@ -705,17 +744,21 @@ export async function POST(request: NextRequest) { }; const imageRequest: CanvasGenerateImageRequest = { allowFailover: false, - aspectRatio: resolvedAspectRatio, + aspectRatio: hasPositionMove ? "auto" : resolvedAspectRatio, billingFeature: "repaint", debugLabel: "canvas-scale-adjust-product-lock", debugMeta: { displayPrompt, operation, requestedModel: requestedModel || null, - resolvedAspectRatio, + resolvedAspectRatio: hasPositionMove ? "auto" : resolvedAspectRatio, resolvedModel: SCALE_ADJUST_REPAIR_MODEL_REF, scaleAdjustProductLockMaskEditApplied: true, - scaleAdjustPromptStrategy: "finegrain-erase-product-lock-v1", + scaleAdjustPersonExclusionApplied: + productLockAssets.personExclusionApplied, + scaleAdjustPromptStrategy: hasPositionMove + ? "gpt-mask-background-rebuild-product-lock-v1" + : "finegrain-erase-product-lock-v1", scaleAdjustProviderMaskImageUrl: modelMaskUpload.url, scaleAdjustSequentialCleanupRetry: true, scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) @@ -754,6 +797,20 @@ export async function POST(request: NextRequest) { }, ], scaleAdjustProductLockComposite: productLockComposite, + ...(hasPositionMove + ? { + scaleAdjustModelRelight: { + model: SCALE_ADJUST_REPAIR_MODEL_REF, + overlayImageUrl: overlayUpload.url, + quality: "medium" as const, + regions: scaleAdjustIntentRegions.map((region) => ({ + target: region.targetRect, + })), + sourceHeight, + sourceWidth, + }, + } + : {}), scaleAdjustVerification, ownerKey: session.user.id, }; diff --git a/components/agents/AgentGenerationWorkspaceDarkV2.tsx b/components/agents/AgentGenerationWorkspaceDarkV2.tsx index a21973f1d..a5602b3c6 100644 --- a/components/agents/AgentGenerationWorkspaceDarkV2.tsx +++ b/components/agents/AgentGenerationWorkspaceDarkV2.tsx @@ -82,6 +82,7 @@ import { LOCAL_EDIT_MAX_REGIONS, createLocalEditReferenceImageId, createLocalEditRegionId, + hasScaleAdjustTargetChanged, reindexLocalEditRegions, type LocalEditReferenceImage, type LocalEditRegion, @@ -843,11 +844,11 @@ function getDetailScaleAdjustRegionPrompt( hasTargetPosition = false ) { return [ - "按目标框线对框选范围内的主物体执行等比比例调整。", + "按目标框线对框选范围内的主物体执行等比缩放和位置调整。", hasTargetPosition ? "主体框只用于识别要调整的主体;青色目标框表示调整后的目标大小和目标位置,主体可以自然超出主体框,并由周边背景补全衔接。" : "框选区域只用于识别要调整的主体,不是调整后的边界;主体可以自然超出原始框选范围,并由周边背景补全衔接。", - "只改变目标主体的尺寸,不改变主体外观、颜色、材质、图案和文字内容;不要把整个框选矩形当成图片块缩放。", + "只改变目标主体的尺寸和位置,不改变主体外观、颜色、材质、图案和文字内容;不要把整个框选矩形当成图片块缩放。", "目标主体调整后露出或覆盖的背景、桌面、墙面、阴影和遮挡关系必须自然补全。", ].join(" "); } @@ -1211,11 +1212,11 @@ function DetailResultLocalEditDialog({ return; } const unchangedRegion = validRegions.find( - (region) => deriveDetailScalePercentFromRects(region.rect, region.targetRect) === 0 + (region) => !hasScaleAdjustTargetChanged(region.rect, region.targetRect) ); if (unchangedRegion) { setActiveRegionId(unchangedRegion.id); - toast.error(`区域 ${unchangedRegion.index} 的目标框大小变化不明显,请重新框选`); + toast.error(`区域 ${unchangedRegion.index} 的目标框大小和位置均未变化`); return; } } else { diff --git a/components/canvas/nodes/ImageNode.tsx b/components/canvas/nodes/ImageNode.tsx index 124dcb8e0..7a354d82c 100644 --- a/components/canvas/nodes/ImageNode.tsx +++ b/components/canvas/nodes/ImageNode.tsx @@ -97,6 +97,7 @@ import { createLocalEditRegionId, getLocalEditRegionsFromMetadata, getNormalizedRectOverlayStyle, + hasScaleAdjustTargetChanged, isHttpImageUrl, LOCAL_EDIT_COLORS, LOCAL_EDIT_MAX_REFERENCE_IMAGES_PER_REGION, @@ -439,11 +440,11 @@ function getCanvasScaleAdjustRegionPrompt( hasTargetPosition = false ) { return [ - "按目标框线对框选范围内的主物体执行等比比例调整。", + "按目标框线对框选范围内的主物体执行等比缩放和位置调整。", hasTargetPosition ? "红色框只用于识别要调整的主体;青色目标框表示调整后的目标大小和目标位置,主体可以自然超出红色框,并由周边背景补全衔接。" : "框选区域只用于识别要调整的主体,不是调整后的边界;主体可以自然超出原始框选范围,并由周边背景补全衔接。", - "只改变目标主体的尺寸,不改变主体外观、颜色、材质、图案和文字内容;不要把整个框选矩形当成图片块缩放。", + "只改变目标主体的尺寸和位置,不改变主体外观、颜色、材质、图案和文字内容;不要把整个框选矩形当成图片块缩放。", "目标主体调整后露出或覆盖的背景、桌面、墙面、阴影和遮挡关系必须自然补全。", ].join(" "); } @@ -4412,15 +4413,11 @@ export function ImageNode({ toast.error(`请为区域 ${missingTargetRegion.index} 框选青色目标框`); return; } - const unchangedRegion = validRegions.find((region) => { - const regionScalePercent = deriveCanvasScalePercentFromRects( - region.rect, - region.targetRect - ); - return Math.abs(regionScalePercent) < 1; - }); + const unchangedRegion = validRegions.find( + (region) => !hasScaleAdjustTargetChanged(region.rect, region.targetRect) + ); if (unchangedRegion) { - toast.error(`请调整区域 ${unchangedRegion.index} 的青色目标框大小`); + toast.error(`请调整区域 ${unchangedRegion.index} 的青色目标框大小或位置`); return; } } else { diff --git a/components/canvas/nodes/image-node/image-node-local-edit.ts b/components/canvas/nodes/image-node/image-node-local-edit.ts index 32f539b5b..0ecc6624f 100644 --- a/components/canvas/nodes/image-node/image-node-local-edit.ts +++ b/components/canvas/nodes/image-node/image-node-local-edit.ts @@ -61,6 +61,17 @@ export const createLocalEditRegionId = () => export const createLocalEditReferenceImageId = () => `local-edit-ref-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +export const hasScaleAdjustTargetChanged = ( + sourceRect: NormalizedCropRect, + targetRect: NormalizedCropRect | null | undefined +) => { + if (!targetRect) return false; + const minimumNormalizedChange = 0.001; + return (['x', 'y', 'width', 'height'] as const).some( + (key) => Math.abs(targetRect[key] - sourceRect[key]) >= minimumNormalizedChange + ); +}; + export const normalizeLocalEditRect = (value: unknown): NormalizedCropRect | null => { if (!value || typeof value !== "object") return null; const rect = value as Partial; diff --git a/docs/evidence/canvas-position-adjust-model-relight-2026-07-13.jpg b/docs/evidence/canvas-position-adjust-model-relight-2026-07-13.jpg new file mode 100644 index 000000000..d56c26ed1 Binary files /dev/null and b/docs/evidence/canvas-position-adjust-model-relight-2026-07-13.jpg differ diff --git a/docs/画布比例调整验证记录-2026-07-11.md b/docs/画布比例调整验证记录-2026-07-11.md index 6fe3d5d08..62134924e 100644 --- a/docs/画布比例调整验证记录-2026-07-11.md +++ b/docs/画布比例调整验证记录-2026-07-11.md @@ -59,3 +59,58 @@ 人工视觉通过 **9/10(90%)**。接口返回状态不计入通过率;以上结论来自最终整图人工检查。 普通画布和“风格化(非 Pro)- 详情模式 - 生图结果区”均提交 `operation: "scale-adjust"` 到共享 `/api/canvas/local-edit` 路由,因此同步使用同一流程。 + +## 位置移动与模型光影扩展(2026-07-13) + +### 稳定版本备份 + +- 在增加位置移动前,已把上面的比例调整稳定流程保存为提交 `15b2d71a`。 +- 本地备份标签为 `backup/canvas-scale-adjust-stable-20260713`,对应原始稳定提交 `5349fece`。 +- 位置移动沿用同一个 `scale-adjust` 操作,只在 `targetRect` 中心发生变化时启用新流程;纯比例调整继续走已经验证的旧流程。 + +### 最终位置移动流程 + +1. 用完整产品蒙版提取原产品透明像素,同时用人物 AOS 分割判断产品是否与人物实质遮挡。 +2. 产品与人物发生遮挡时拒绝移动,避免手臂、衣服或身体被一起搬走,也避免产品留下人物形状的透明洞;人物与产品不相交时人物区域保持锁定。 +3. 由 GPT Image 2 Edit 根据旧产品精确蒙版移除原产品、脚轮和原投影,并补全被遮挡的背景;模型结果只在修复蒙版内合成,其他区域继续使用原图。 +4. 本地把原产品透明像素精确放入 `targetRect`,因此产品款式、大小和最终坐标不依赖模型猜测。 +5. 把移动后的完整图和“地面可编辑区域”再次交给 GPT Image 2 Edit,只要求它依据场景主光方向生成新的接触阴影和投影。 +6. 不直接采用第二次模型输出整图,而是提取模型在地面蒙版内生成的中性暗化场;产品像素和蒙版外像素完全锁定,避免模型复制产品、改尺寸或改变画幅。 +7. 模型光影接口超时或失败时,返回已验证的本地接触阴影版本作为兜底,并记录错误;本地阴影不是正常成功路径的最终光影来源。 + +### 已测试方法与结论 + +| 方法 | 结果 | 结论 | +| --- | --- | --- | +| 原位置清理后直接精确移动 | 办公椅、推车可保持几何位置,但新位置光影偏弱 | 只适合作为第一阶段,不能作为最终结果 | +| 人物场景直接移动相交产品 | 书架连同人物手部一起移动,手臂破损 | 已禁止人物与产品实质遮挡的移动 | +| 人物 AOS 蒙版保护 | 人物蒙版正确;最初单通道处理错误产生扫描线和黑块 | Sharp 模糊、缩放后统一显式 `extractChannel(0)`,修复通过 | +| 对被人物遮挡的产品做邻近像素补洞 | 出现三角形重复木纹 | 失败,已删除 | +| 四方向加权补洞 | 出现黑色楔形和空洞 | 失败,已删除 | +| 原图和目标框操作图整体交给模型 | 人物姿态和自然光影较好,但模型忽略精确目标框和尺寸,单次约 263 秒 | 不能承担几何移动,仅证明模型适合语义补全和光影 | +| 直接转移旧阴影 | 首版因通道错误出现黑色矩形;修正后仍无法适配新位置光源关系 | 只保留为接口失败兜底的一部分 | +| 宽区域模型重打光 | 模型在目标位置复制第二把椅子 | 失败,不采用模型整图 | +| 地面白色蒙版 + 固定 3:4 输出 | 仍出现重复椅子,且输出画幅变化 | 失败,改为 `auto` 并锁定原图坐标 | +| 直接替换模型地面蒙版像素 | 重复问题消失,但出现矩形色差接缝 | 失败,改为只提取暗化场 | +| 提取模型中值归一化暗化场 | 新接触阴影和投影可见,无重复产品、无矩形接缝、产品像素不变 | 通过,作为最终模型光影流程 | +| 模型清理旧产品后直接采用其画幅 | 模型返回 1536x2048,原图为 1760x2221,导致目标坐标偏移 | 修复为始终按原图画幅和坐标合成 | +| 模糊后阈值生成旧位置蒙版 | Sharp 阈值后仍存在灰度,办公椅底座和脚轮未被完全清理 | 改为逐像素二值化,只保留 0/255,验证通过 | +| 移动后旧轮廓门禁使用扩张修复蒙版 | 把未变化背景算入残留,产生误报 | 移动场景改为只检查精确原产品蒙版 | +| 移动后用暗色像素判断残影 | 自然深色地板和阴影触发误报 | 移动场景只使用产品相似度,暗色规则仅留给未移动的缩小场景 | +| 目标框悬空摆放 | 产品与沙发/地面关系不自然 | 拒绝作为验收样本,正式测试改为目标底边落地 | + +### 本地视觉验收 + +证据图:[位置移动、旧位置清理、模型光影和人物保持对比](./evidence/canvas-position-adjust-model-relight-2026-07-13.jpg) + +- 真实办公室办公椅:旧椅、底座和脚轮已清理,产品按正式坐标移动;模型生成的新位置接触阴影和地面投影已通过暗化场合成,无重复产品和矩形接缝。 +- 真实人物室内图:人物不与产品相交时,边桌可移动且人物保持不变;相交产品会在生成前明确拒绝。 +- 最终正式光影合成统计:`appliedShadowStrength: 4741328`、`medianLumaDelta: -18`、`sampleCount: 719617`。 +- 真实模型阶段回放的最终本地结果为 `/private/tmp/scale-adjust-office-chair-final-staged-replay.jpg`。该回放使用同一套正式清理、坐标锁定和模型暗化场合成代码,人工视觉通过。 + +### 外部模型接口记录 + +- FAL 在连续回归期间出现过 HTTP 408、HTTP 400、`fetch failed`、300 秒超时,以及 `v3b.fal.media` 下载连接超时。 +- GPT Image 2 Edit 的真实旧位置清理和真实光影输出都分别成功过;最终视觉回放由这两份真实模型输出通过当前正式合成代码得到。 +- 因上述外部接口波动,最后一轮未得到“同一次在线请求从清理到重打光全部成功”的证据;这属于供应商请求故障,不记为算法视觉失败。生产流程已在任一模型阶段失败时回退到本地验证结果,避免用户任务直接报错。 +- 后续复测优先重复完整在线链路,不再重复表中已经证伪的补洞、整图替换和宽区域重绘方法。 diff --git a/lib/ai/rmbg/service.ts b/lib/ai/rmbg/service.ts index 65c1cc06b..1776e8a49 100644 --- a/lib/ai/rmbg/service.ts +++ b/lib/ai/rmbg/service.ts @@ -29,6 +29,7 @@ export type ExecuteRmbgJobParams = { userId: string; imageBuffer: Buffer; imageBase64?: string; + foregroundType?: "" | "person" | "object" | "stamp"; prompt?: string; chatId?: string | null; messageId?: string | null; @@ -562,6 +563,7 @@ export async function executeRmbgJob( userId, imageBuffer, imageBase64 = imageBuffer.toString("base64"), + foregroundType, prompt = "抠图处理", chatId = null, messageId = null, @@ -599,7 +601,7 @@ export async function executeRmbgJob( const result = await processWithAosSegmentation(imageBuffer, { apiKey: config.aosApiKey!, apiBaseUrl: config.aosApiBaseUrl, - foregroundType: config.aosForegroundType || "", + foregroundType: foregroundType ?? config.aosForegroundType ?? "", crop: config.aosCrop === 1 ? 1 : 0, format: config.aosFormat === "jpg" ? "jpg" : "png", }); diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index c5439134d..c07ff8672 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -76,7 +76,13 @@ export interface CanvasScaleAdjustProductLockCompositeConfig { backgroundPlateImageUrl?: string; protectMaskImageUrl?: string; repairMaskImageUrl?: string; + regions?: Array<{ + source: CanvasLocalEditCompositeRegion; + target: CanvasLocalEditCompositeRegion; + }>; + shadowSourceImageUrl?: string; sourceImageUrl?: string; + sourceSubjectMaskImageUrl?: string; sourceHeight: number; sourceWidth: number; } @@ -84,6 +90,7 @@ export interface CanvasScaleAdjustProductLockCompositeConfig { export interface CanvasScaleAdjustVerificationConfig { allowDeterministicFallback?: boolean; allowOutsideChangedWhenChangedInside?: boolean; + deterministicProductOverlayApplied?: boolean; oldFootprintMaskImageUrl?: string; protectedRegions?: CanvasLocalEditCompositeRegion[]; rejectFailedCandidates?: boolean; @@ -96,6 +103,17 @@ export interface CanvasScaleAdjustVerificationConfig { scalePercent?: number; } +export interface CanvasScaleAdjustModelRelightConfig { + model: string; + overlayImageUrl: string; + quality?: "low" | "medium" | "high"; + regions: Array<{ + target: CanvasLocalEditCompositeRegion; + }>; + sourceHeight: number; + sourceWidth: number; +} + export interface CanvasGenerateImageRetryVariant { allowFailover?: boolean; debugMeta?: Record; @@ -113,6 +131,7 @@ export interface CanvasGenerateImageRetryVariant { referenceImages?: string[]; scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustProductLockComposite?: CanvasScaleAdjustProductLockCompositeConfig; + scaleAdjustModelRelight?: CanvasScaleAdjustModelRelightConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; } @@ -139,6 +158,7 @@ export interface CanvasGenerateImageRequest { localEditComposite?: CanvasLocalEditCompositeConfig; scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustProductLockComposite?: CanvasScaleAdjustProductLockCompositeConfig; + scaleAdjustModelRelight?: CanvasScaleAdjustModelRelightConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; retryVariants?: CanvasGenerateImageRetryVariant[]; internalRetryVariantIndex?: number; @@ -581,7 +601,11 @@ function normalizeScaleAdjustProductLockCompositeConfig( const backgroundPlateImageUrl = String(value.backgroundPlateImageUrl || "").trim(); const protectMaskImageUrl = String(value.protectMaskImageUrl || "").trim(); const repairMaskImageUrl = String(value.repairMaskImageUrl || "").trim(); + const shadowSourceImageUrl = String(value.shadowSourceImageUrl || "").trim(); const sourceImageUrl = String(value.sourceImageUrl || "").trim(); + const sourceSubjectMaskImageUrl = String( + value.sourceSubjectMaskImageUrl || "" + ).trim(); const sourceWidth = Math.round(Number(value.sourceWidth || 0)); const sourceHeight = Math.round(Number(value.sourceHeight || 0)); if (sourceWidth <= 0 || sourceHeight <= 0) return null; @@ -597,9 +621,28 @@ function normalizeScaleAdjustProductLockCompositeConfig( ...(repairMaskImageUrl && /^https?:\/\//i.test(repairMaskImageUrl) ? { repairMaskImageUrl } : {}), + ...(shadowSourceImageUrl && /^https?:\/\//i.test(shadowSourceImageUrl) + ? { shadowSourceImageUrl } + : {}), ...(sourceImageUrl && /^https?:\/\//i.test(sourceImageUrl) ? { sourceImageUrl } : {}), + ...(sourceSubjectMaskImageUrl && /^https?:\/\//i.test(sourceSubjectMaskImageUrl) + ? { sourceSubjectMaskImageUrl } + : {}), + regions: Array.isArray(value.regions) + ? value.regions + .map((item) => ({ + source: normalizeCanvasRegion(item?.source), + target: normalizeCanvasRegion(item?.target), + })) + .filter( + (item): item is { + source: CanvasLocalEditCompositeRegion; + target: CanvasLocalEditCompositeRegion; + } => Boolean(item.source && item.target) + ) + : [], sourceHeight, sourceWidth, }; @@ -669,6 +712,8 @@ function normalizeScaleAdjustVerificationConfig( allowDeterministicFallback: value.allowDeterministicFallback === true, allowOutsideChangedWhenChangedInside: value.allowOutsideChangedWhenChangedInside === true, + deterministicProductOverlayApplied: + value.deterministicProductOverlayApplied === true, rejectFailedCandidates: value.rejectFailedCandidates === true, rejectNoEffectiveChangeOnly: value.rejectNoEffectiveChangeOnly === true, @@ -1404,7 +1449,16 @@ async function getScaleAdjustFootprintChangeStats(params: { const widthFactor = region.width > 0 ? targetRegion.width / region.width : 1; const heightFactor = region.height > 0 ? targetRegion.height / region.height : 1; const isShrink = widthFactor < 0.92 || heightFactor < 0.92; - if (!isShrink) continue; + const isMove = + Math.abs( + targetRegion.x + targetRegion.width / 2 - + (region.x + region.width / 2) + ) >= 0.01 || + Math.abs( + targetRegion.y + targetRegion.height / 2 - + (region.y + region.height / 2) + ) >= 0.01; + if (!(isShrink || isMove)) continue; const sourceBox = rectToPixelBox({ height: sourceHeight, @@ -1567,7 +1621,16 @@ async function getScaleAdjustOldFootprintGhostStats(params: { const widthFactor = region.width > 0 ? targetRegion.width / region.width : 1; const heightFactor = region.height > 0 ? targetRegion.height / region.height : 1; const isShrink = widthFactor < 0.92 || heightFactor < 0.92; - if (!isShrink) continue; + const isMove = + Math.abs( + targetRegion.x + targetRegion.width / 2 - + (region.x + region.width / 2) + ) >= 0.01 || + Math.abs( + targetRegion.y + targetRegion.height / 2 - + (region.y + region.height / 2) + ) >= 0.01; + if (!(isShrink || isMove)) continue; const sourceBox = rectToPixelBox({ height: sourceHeight, @@ -1610,7 +1673,9 @@ async function getScaleAdjustOldFootprintGhostStats(params: { const localIndex = localY * sourceBox.width + localX; const absoluteIndex = y * sourceWidth + x; const explicitMaskValue = explicitMaskRaw?.[absoluteIndex] || 0; - const maskHit = explicitMaskValue > 18 || fallbackMask[localIndex] > 0; + const maskHit = explicitMaskRaw + ? explicitMaskValue > 18 + : fallbackMask[localIndex] > 0; if (!maskHit) continue; if (explicitMaskValue > 18) explicitPixels += 1; @@ -1660,8 +1725,10 @@ async function getScaleAdjustOldFootprintGhostStats(params: { const detected = enoughPixels && (similarRatio >= SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_RATIO_THRESHOLD || - darkResidualRatio >= SCALE_ADJUST_OLD_FOOTPRINT_DARK_RATIO_THRESHOLD || - (similarRatio >= 0.12 && darkResidualRatio >= 0.14)); + (!isMove && + (darkResidualRatio >= + SCALE_ADJUST_OLD_FOOTPRINT_DARK_RATIO_THRESHOLD || + (similarRatio >= 0.12 && darkResidualRatio >= 0.14)))); shrinkStats.push({ detected, darkResidualPixels, @@ -1994,7 +2061,10 @@ async function applyScaleAdjustVerificationToImages(params: { changeDiffScores.some( (score) => score >= SCALE_ADJUST_FOOTPRINT_FALLBACK_DIFF_THRESHOLD ); + const deterministicProductOverlayAccepted = + verification.deterministicProductOverlayApplied === true; const changedInside = + deterministicProductOverlayAccepted || (visuallyChangedInside && footprintStats.passed) || footprintFallbackAccepted; const outsideChanged = @@ -2006,6 +2076,7 @@ async function applyScaleAdjustVerificationToImages(params: { console.info("[Canvas Generate Image] Scale adjust result guard:", { debugLabel, changedInside, + deterministicProductOverlayAccepted, footprintFallbackAccepted, footprintChanged: footprintStats.passed, footprintStats: { @@ -3625,6 +3696,182 @@ async function applyScaleAdjustPatchCompositeToImages(params: { return mergedImages; } +async function buildScaleAdjustTransferredShadow(params: { + cleanedBackgroundBuffer: Buffer; + height: number; + regions: NonNullable; + sourceBuffer: Buffer; + sourceSubjectMaskBuffer: Buffer; + width: number; +}) { + const { + cleanedBackgroundBuffer, + height, + regions, + sourceBuffer, + sourceSubjectMaskBuffer, + width, + } = params; + if (regions.length === 0) return null; + + const [sourceRaw, cleanedRaw, subjectAlpha] = await Promise.all([ + sharp(sourceBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + sharp(cleanedBackgroundBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + normalizeOpacityMaskToRaw({ + height, + maskBuffer: sourceSubjectMaskBuffer, + width, + }), + ]); + const nearSubjectAlpha = await sharp(subjectAlpha, { + raw: { channels: 1, height, width }, + }) + .blur(Math.max(3, Math.min(width, height) * 0.012)) + .extractChannel(0) + .raw() + .toBuffer(); + const fullShadowAlpha = Buffer.alloc(width * height); + let shadowStrength = 0; + + for (const region of regions) { + const sourceBox = rectToPixelBox({ height, region: region.source, width }); + const targetBox = rectToPixelBox({ height, region: region.target, width }); + const shadowAlpha = Buffer.alloc(sourceBox.width * sourceBox.height); + const floorStart = Math.floor(sourceBox.height * 0.72); + + for (let localY = floorStart; localY < sourceBox.height; localY += 1) { + const sourceY = sourceBox.top + localY; + for (let localX = 0; localX < sourceBox.width; localX += 1) { + const sourceX = sourceBox.left + localX; + const pixelIndex = sourceY * width + sourceX; + if (subjectAlpha[pixelIndex] >= 12) continue; + const proximity = nearSubjectAlpha[pixelIndex] / 255; + if (proximity < 0.04) continue; + const rgbIndex = pixelIndex * 3; + const sourceLuma = + sourceRaw[rgbIndex] * 0.299 + + sourceRaw[rgbIndex + 1] * 0.587 + + sourceRaw[rgbIndex + 2] * 0.114; + const cleanedLuma = + cleanedRaw[rgbIndex] * 0.299 + + cleanedRaw[rgbIndex + 1] * 0.587 + + cleanedRaw[rgbIndex + 2] * 0.114; + const darkness = Math.max(0, cleanedLuma - sourceLuma - 4); + const alpha = Math.min(56, Math.round(darkness * 1.25 * proximity)); + shadowAlpha[localY * sourceBox.width + localX] = alpha; + } + } + + const transferredAlpha = await sharp(shadowAlpha, { + raw: { + channels: 1, + height: sourceBox.height, + width: sourceBox.width, + }, + }) + .resize(targetBox.width, targetBox.height, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .extractChannel(0) + .raw() + .toBuffer(); + const targetSubjectAlpha = await sharp(subjectAlpha, { + raw: { + channels: 1, + height, + width, + }, + }) + .extract(sourceBox) + .resize(targetBox.width, targetBox.height, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .extractChannel(0) + .raw() + .toBuffer(); + const bottomByColumn = new Int32Array(targetBox.width).fill(-1); + let globalBottom = -1; + for (let localY = 0; localY < targetBox.height; localY += 1) { + for (let localX = 0; localX < targetBox.width; localX += 1) { + if (targetSubjectAlpha[localY * targetBox.width + localX] < 24) continue; + bottomByColumn[localX] = localY; + globalBottom = Math.max(globalBottom, localY); + } + } + const contactTolerance = Math.max(2, Math.round(targetBox.height * 0.055)); + const contactDepth = Math.max(2, Math.round(targetBox.height * 0.022)); + + for (let localY = 0; localY < targetBox.height; localY += 1) { + const targetY = targetBox.top + localY; + if (targetY < 0 || targetY >= height) continue; + for (let localX = 0; localX < targetBox.width; localX += 1) { + const targetX = targetBox.left + localX; + if (targetX < 0 || targetX >= width) continue; + const transferred = transferredAlpha[localY * targetBox.width + localX]; + if (transferred > 0) { + const fullIndex = targetY * width + targetX; + fullShadowAlpha[fullIndex] = Math.max( + fullShadowAlpha[fullIndex], + transferred + ); + shadowStrength += transferred; + } + } + } + + if (globalBottom < 0) continue; + for (let localX = 0; localX < targetBox.width; localX += 1) { + const contactY = bottomByColumn[localX]; + if (contactY < globalBottom - contactTolerance) continue; + for (let depth = -1; depth <= contactDepth; depth += 1) { + const targetX = targetBox.left + localX; + const targetY = targetBox.top + contactY + depth; + if (targetX < 0 || targetX >= width || targetY < 0 || targetY >= height) { + continue; + } + const contactAlpha = Math.max( + 12, + Math.round(48 * (1 - Math.max(0, depth) / (contactDepth + 1))) + ); + const fullIndex = targetY * width + targetX; + fullShadowAlpha[fullIndex] = Math.max( + fullShadowAlpha[fullIndex], + contactAlpha + ); + shadowStrength += contactAlpha; + } + } + } + + if (shadowStrength === 0) return null; + const softenedAlpha = await sharp(fullShadowAlpha, { + raw: { channels: 1, height, width }, + }) + .blur(Math.max(1, Math.min(width, height) * 0.0045)) + .extractChannel(0) + .raw() + .toBuffer(); + const blackRgb = Buffer.alloc(width * height * 3); + return sharp(blackRgb, { raw: { channels: 3, height, width } }) + .joinChannel(softenedAlpha, { + raw: { channels: 1, height, width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + async function applyScaleAdjustProductLockCompositeToImages(params: { images: ParsedImage[]; composite: CanvasScaleAdjustProductLockCompositeConfig | null; @@ -3663,7 +3910,13 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { .rotate() .ensureAlpha() .png({ compressionLevel: 6 }) - .toBuffer() + .toBuffer() + : null; + const shadowSourceBuffer = composite.shadowSourceImageUrl + ? await fetchImageBufferFromUrl(composite.shadowSourceImageUrl) + : null; + const sourceSubjectMaskBuffer = composite.sourceSubjectMaskImageUrl + ? await fetchImageBufferFromUrl(composite.sourceSubjectMaskImageUrl) : null; const compositeBaseBuffer = backgroundPlateBaseBuffer || sourceBaseBuffer; const overlayMetadata = await sharp(overlayBuffer, { failOn: "none" }).metadata(); @@ -3672,25 +3925,39 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { if (overlayWidth <= 0 || overlayHeight <= 0) return images; const mergedImages: ParsedImage[] = []; + let transferredShadowCount = 0; for (const [index, image] of images.entries()) { const generatedUrl = image.originalUrl || image.thumbnailUrl; const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); - const normalizedGeneratedBuffer = await sharp(generatedBuffer, { + const normalizedModelBuffer = await sharp(generatedBuffer, { failOn: "none", }) .rotate() .ensureAlpha() .png({ compressionLevel: 6 }) .toBuffer(); - const generatedMetadata = await sharp(normalizedGeneratedBuffer, { + const generatedMetadata = await sharp(normalizedModelBuffer, { failOn: "none", }).metadata(); - const generatedWidth = Math.round(Number(generatedMetadata.width || 0)); - const generatedHeight = Math.round(Number(generatedMetadata.height || 0)); - if (generatedWidth <= 0 || generatedHeight <= 0) { + const modelWidth = Math.round(Number(generatedMetadata.width || 0)); + const modelHeight = Math.round(Number(generatedMetadata.height || 0)); + if (modelWidth <= 0 || modelHeight <= 0) { mergedImages.push(image); continue; } + const generatedWidth = compositeBaseBuffer ? overlayWidth : modelWidth; + const generatedHeight = compositeBaseBuffer ? overlayHeight : modelHeight; + const normalizedGeneratedBuffer = + generatedWidth === modelWidth && generatedHeight === modelHeight + ? normalizedModelBuffer + : await sharp(normalizedModelBuffer, { failOn: "none" }) + .resize(generatedWidth, generatedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .ensureAlpha() + .png({ compressionLevel: 6 }) + .toBuffer(); const resizedOverlayBaseBuffer = await sharp(overlayBuffer, { failOn: "none" }) .resize(generatedWidth, generatedHeight, { @@ -3724,6 +3991,7 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { .ensureAlpha() .extractChannel(3) .blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006))) + .extractChannel(0) .raw() .toBuffer() : null; @@ -3746,17 +4014,33 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { width: generatedWidth, }) : null; + const transferredShadowBuffer = + !backgroundPlateBaseBuffer && + shadowSourceBuffer && + sourceSubjectMaskBuffer && + composite.regions?.length + ? await buildScaleAdjustTransferredShadow({ + cleanedBackgroundBuffer: normalizedGeneratedBuffer, + height: generatedHeight, + regions: composite.regions, + sourceBuffer: shadowSourceBuffer, + sourceSubjectMaskBuffer, + width: generatedWidth, + }) + : null; + if (transferredShadowBuffer) transferredShadowCount += 1; const mergedPngBuffer = await sharp(baseForProductLock, { failOn: "none", }) - .composite( - generatedRepairOverlay - ? [ - { input: generatedRepairOverlay, left: 0, top: 0 }, - { input: resizedOverlayBuffer, left: 0, top: 0 }, - ] - : [{ input: resizedOverlayBuffer, left: 0, top: 0 }] - ) + .composite([ + ...(generatedRepairOverlay + ? [{ input: generatedRepairOverlay, left: 0, top: 0 }] + : []), + ...(transferredShadowBuffer + ? [{ input: transferredShadowBuffer, left: 0, top: 0 }] + : []), + { input: resizedOverlayBuffer, left: 0, top: 0 }, + ]) .png({ compressionLevel: 6 }) .toBuffer(); const mergedBuffer = await sharp(mergedPngBuffer, { failOn: "none" }) @@ -3798,6 +4082,7 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { repairMaskBuffer && compositeBaseBuffer && !backgroundPlateBaseBuffer ), sourceBaseApplied: Boolean(sourceBaseBuffer), + transferredShadowCount, sourceHeight: composite.sourceHeight, sourceWidth: composite.sourceWidth, }); diff --git a/lib/canvas/image-task-queue.ts b/lib/canvas/image-task-queue.ts index 02ea5b804..0d702ae58 100644 --- a/lib/canvas/image-task-queue.ts +++ b/lib/canvas/image-task-queue.ts @@ -14,6 +14,7 @@ import { type CanvasGenerateImageRequest, type CanvasGenerateImageResult, } from "./generate-image"; +import { applyScaleAdjustModelRelight } from "./scale-adjust-model-relight"; import { executeStyleProTask, getStyleProTaskPrimaryModelKey, @@ -59,7 +60,7 @@ interface QueueState { } const DEFAULT_RUNNING_TIMEOUT_MS = 10 * 60 * 1000; -const SCALE_ADJUST_RUNNING_TIMEOUT_MS = 8 * 60 * 1000; +const SCALE_ADJUST_RUNNING_TIMEOUT_MS = 12 * 60 * 1000; const LOCAL_ACTIVE_RUNNING_HARD_TIMEOUT_MS = 30 * 60 * 1000; const PENDING_LOOKAHEAD_LIMIT = 100; const QUEUE_STATUS_PENDING_LOOKAHEAD_LIMIT = 500; @@ -423,10 +424,20 @@ async function runSingleTask(entry: PendingTaskEntry) { request, userId: String(task.userId || entry.userId), }) - : generateCanvasImage({ - ...(request as CanvasGenerateImageRequest), - ownerKey: task.userId, - }); + : (async () => { + const imageRequest = request as CanvasGenerateImageRequest; + const ownerKey = String(task.userId || entry.userId); + const initialResult = await generateCanvasImage({ + ...imageRequest, + ownerKey, + }); + return await applyScaleAdjustModelRelight({ + generateImage: generateCanvasImage, + initialResult, + ownerKey, + request: imageRequest, + }); + })(); const result = await withCanvasImageTaskExecutionTimeout( execution, runningTimeoutMs, diff --git a/lib/canvas/scale-adjust-model-relight.ts b/lib/canvas/scale-adjust-model-relight.ts new file mode 100644 index 000000000..f7eb614bb --- /dev/null +++ b/lib/canvas/scale-adjust-model-relight.ts @@ -0,0 +1,368 @@ +import "server-only"; + +import sharp from "sharp"; + +import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; +import type { + CanvasGenerateImageRequest, + CanvasGenerateImageResult, + CanvasLocalEditCompositeRegion, + CanvasScaleAdjustModelRelightConfig, +} from "@/lib/canvas/generate-image"; +import { uploadFile } from "@/lib/storage/service"; + +type GenerateCanvasImage = ( + request: CanvasGenerateImageRequest +) => Promise; + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function bufferToArrayBuffer(buffer: Buffer) { + return buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; +} + +function rectToPixelBox(params: { + height: number; + region: CanvasLocalEditCompositeRegion; + width: number; +}) { + const { height, region, width } = params; + const left = clamp(Math.floor(region.x * width), 0, Math.max(0, width - 1)); + const top = clamp(Math.floor(region.y * height), 0, Math.max(0, height - 1)); + const right = clamp( + Math.ceil((region.x + region.width) * width), + left + 1, + width + ); + const bottom = clamp( + Math.ceil((region.y + region.height) * height), + top + 1, + height + ); + return { bottom, left, right, top }; +} + +async function buildModelRelightMask(params: { + config: CanvasScaleAdjustModelRelightConfig; + height: number; + width: number; +}) { + const { config, height, width } = params; + const productAlpha = await sharp( + await fetchImageBufferFromUrl(config.overlayImageUrl), + { failOn: "none" } + ) + .rotate() + .resize(width, height, { fit: "fill", kernel: sharp.kernel.lanczos3 }) + .ensureAlpha() + .extractChannel(3) + .raw() + .toBuffer(); + const editableMask = Buffer.alloc(width * height); + + for (const item of config.regions) { + const target = item.target; + const expandedFloorRegion = { + x: target.x - target.width * 0.16, + y: target.y + target.height * 0.68, + width: target.width * 1.32, + height: target.height * 0.5, + }; + const box = rectToPixelBox({ + height, + region: expandedFloorRegion, + width, + }); + for (let y = box.top; y < box.bottom; y += 1) { + editableMask.fill(255, y * width + box.left, y * width + box.right); + } + } + + for (let index = 0; index < editableMask.length; index += 1) { + if (productAlpha[index] >= 8) editableMask[index] = 0; + } + + const maskBuffer = await sharp(editableMask, { + raw: { channels: 1, height, width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + return { editableMask, maskBuffer, productAlpha }; +} + +function findMedianLumaDelta(params: { + editableMask: Buffer; + exactRaw: Buffer; + generatedRaw: Buffer; +}) { + const { editableMask, exactRaw, generatedRaw } = params; + const histogram = new Uint32Array(511); + let sampleCount = 0; + for ( + let maskIndex = 0, rgbIndex = 0; + maskIndex < editableMask.length; + maskIndex += 1, rgbIndex += 3 + ) { + if (editableMask[maskIndex] < 240) continue; + const exactLuma = + (77 * exactRaw[rgbIndex] + + 150 * exactRaw[rgbIndex + 1] + + 29 * exactRaw[rgbIndex + 2]) >> + 8; + const generatedLuma = + (77 * generatedRaw[rgbIndex] + + 150 * generatedRaw[rgbIndex + 1] + + 29 * generatedRaw[rgbIndex + 2]) >> + 8; + const bucket = clamp(exactLuma - generatedLuma + 255, 0, 510); + histogram[bucket] += 1; + sampleCount += 1; + } + + if (sampleCount === 0) return { median: 0, sampleCount }; + let seen = 0; + for (let index = 0; index < histogram.length; index += 1) { + seen += histogram[index]; + if (seen >= sampleCount / 2) { + return { median: index - 255, sampleCount }; + } + } + return { median: 0, sampleCount }; +} + +async function mergeModelShadowField(params: { + editableMask: Buffer; + exactBuffer: Buffer; + generatedBuffer: Buffer; + height: number; + productAlpha: Buffer; + width: number; +}) { + const { + editableMask, + exactBuffer, + generatedBuffer, + height, + productAlpha, + width, + } = params; + const [exactRaw, generatedRaw] = await Promise.all([ + sharp(exactBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + sharp(generatedBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill", kernel: sharp.kernel.lanczos3 }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + ]); + const { median, sampleCount } = findMedianLumaDelta({ + editableMask, + exactRaw, + generatedRaw, + }); + if (sampleCount < 128) return null; + + const shadowAlpha = Buffer.alloc(width * height); + let rawShadowStrength = 0; + for ( + let maskIndex = 0, rgbIndex = 0; + maskIndex < editableMask.length; + maskIndex += 1, rgbIndex += 3 + ) { + if (editableMask[maskIndex] < 240) continue; + const exactLuma = + (77 * exactRaw[rgbIndex] + + 150 * exactRaw[rgbIndex + 1] + + 29 * exactRaw[rgbIndex + 2]) >> + 8; + const generatedLuma = + (77 * generatedRaw[rgbIndex] + + 150 * generatedRaw[rgbIndex + 1] + + 29 * generatedRaw[rgbIndex + 2]) >> + 8; + const modelDarkness = Math.max( + 0, + exactLuma - generatedLuma - median - 12 + ); + const alpha = Math.min(58, Math.round(modelDarkness * 0.78)); + shadowAlpha[maskIndex] = alpha; + rawShadowStrength += alpha; + } + if (rawShadowStrength < 96) return null; + + const blurSigma = clamp(Math.min(width, height) * 0.0063, 6, 24); + const softenedAlpha = await sharp(shadowAlpha, { + raw: { channels: 1, height, width }, + }) + .blur(blurSigma) + .extractChannel(0) + .raw() + .toBuffer(); + const shadowLayer = Buffer.alloc(width * height * 4); + let appliedShadowStrength = 0; + for ( + let pixelIndex = 0, rgbaIndex = 0; + pixelIndex < softenedAlpha.length; + pixelIndex += 1, rgbaIndex += 4 + ) { + const alpha = productAlpha[pixelIndex] >= 8 ? 0 : softenedAlpha[pixelIndex]; + shadowLayer[rgbaIndex] = 32; + shadowLayer[rgbaIndex + 1] = 32; + shadowLayer[rgbaIndex + 2] = 32; + shadowLayer[rgbaIndex + 3] = alpha; + appliedShadowStrength += alpha; + } + if (appliedShadowStrength < 96) return null; + + const mergedBuffer = await sharp(exactBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .composite([ + { + input: shadowLayer, + raw: { channels: 4, height, width }, + blend: "over", + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); + return { appliedShadowStrength, median, mergedBuffer, sampleCount }; +} + +export async function applyScaleAdjustModelRelight(params: { + generateImage: GenerateCanvasImage; + initialResult: CanvasGenerateImageResult; + ownerKey?: string; + request: CanvasGenerateImageRequest; +}) { + const { generateImage, initialResult, ownerKey, request } = params; + const config = request.scaleAdjustModelRelight; + if (!config || initialResult.images.length === 0) return initialResult; + + const relitImages = [] as CanvasGenerateImageResult["images"]; + for (const [index, image] of initialResult.images.entries()) { + const exactUrl = image.originalUrl || image.thumbnailUrl; + try { + const exactBuffer = await fetchImageBufferFromUrl(exactUrl); + const metadata = await sharp(exactBuffer, { failOn: "none" }).metadata(); + const width = Math.round(Number(metadata.width || 0)); + const height = Math.round(Number(metadata.height || 0)); + if (width <= 0 || height <= 0) throw new Error("精确合成结果尺寸无效"); + + const { editableMask, maskBuffer, productAlpha } = + await buildModelRelightMask({ config, height, width }); + const suffix = `${Date.now()}-${index + 1}-${Math.random() + .toString(36) + .slice(2, 8)}`; + const maskUpload = await uploadFile( + `canvas-scale-adjust-model-relight-mask-${suffix}.png`, + bufferToArrayBuffer(maskBuffer) + ); + const modelResult = await generateImage({ + allowFailover: false, + aspectRatio: "auto", + debugLabel: "canvas-scale-adjust-model-relight", + debugMeta: { + operation: "scale-adjust-model-relight", + scaleAdjustModelRelightApplied: true, + scaleAdjustProviderMaskImageUrl: maskUpload.url, + sourceHeight: config.sourceHeight, + sourceWidth: config.sourceWidth, + }, + displayPrompt: request.displayPrompt, + imageCount: 1, + imageSize: "auto", + model: config.model, + negativePrompt: + "复制产品、增加产品、移动产品、改变产品造型、改变构图、修改人物、文字、标注、边框", + ownerKey, + prompt: + "仅编辑白色遮罩中的地面区域。根据画面现有主光方向、环境光颜色、产品落地点和地面材质,为当前位置的产品生成真实自然的接触阴影与投射阴影。阴影必须贴合产品落地边缘,方向、软硬、颜色和强度与室内现有光照一致。黑色区域保持像素级不变;不要重绘、复制、移动或变形产品,不要修改人物、家具、墙面和构图。", + promptExtend: false, + providerImageParams: { + maskImageUrl: maskUpload.url, + quality: config.quality || "medium", + }, + referenceImages: [exactUrl], + }); + const generatedUrl = + modelResult.images[0]?.originalUrl || + modelResult.images[0]?.thumbnailUrl || + ""; + if (!generatedUrl) throw new Error("光影模型未返回图片"); + const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); + const merged = await mergeModelShadowField({ + editableMask, + exactBuffer, + generatedBuffer, + height, + productAlpha, + width, + }); + if (!merged) throw new Error("光影模型未生成有效的地面明暗变化"); + + const finalJpegBuffer = await sharp(merged.mergedBuffer, { + failOn: "none", + }) + .flatten({ background: "#ffffff" }) + .jpeg({ quality: 94, mozjpeg: true }) + .toBuffer(); + const thumbnailBuffer = await sharp(merged.mergedBuffer, { + failOn: "none", + }) + .flatten({ background: "#ffffff" }) + .resize(512, 512, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 88, mozjpeg: true }) + .toBuffer(); + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-model-relit-${suffix}.jpg`, + bufferToArrayBuffer(finalJpegBuffer) + ), + uploadFile( + `canvas-scale-adjust-model-relit-thumb-${suffix}.jpg`, + bufferToArrayBuffer(thumbnailBuffer) + ), + ]); + console.info("[Canvas Scale Adjust] Model relight applied:", { + appliedShadowStrength: merged.appliedShadowStrength, + exactUrl, + generatedUrl, + height, + medianLumaDelta: merged.median, + sampleCount: merged.sampleCount, + width, + }); + relitImages.push({ + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + }); + } catch (error) { + console.error( + "[Canvas Scale Adjust] Model relight failed; keeping verified local shadow fallback:", + { + error: error instanceof Error ? error.message : String(error), + exactUrl, + index, + } + ); + relitImages.push(image); + } + } + + return { + ...initialResult, + images: relitImages, + }; +} diff --git a/lib/canvas/scale-adjust-product-lock.ts b/lib/canvas/scale-adjust-product-lock.ts index ea47276a6..f2e7673b0 100644 --- a/lib/canvas/scale-adjust-product-lock.ts +++ b/lib/canvas/scale-adjust-product-lock.ts @@ -25,7 +25,10 @@ export type CanvasScaleAdjustProductLockAssets = { backgroundPlateBuffer?: Buffer; modelMaskBuffer: Buffer; overlayBuffer: Buffer; + personExclusionApplied: boolean; + personExclusionMaskBuffer?: Buffer; repairMaskBuffer: Buffer; + sourceSubjectMaskBuffer: Buffer; subjectReferenceUrls: string[]; }; @@ -60,10 +63,11 @@ function rectToPixelBox( async function normalizeSubjectBuffer(params: { box: PixelBox; + exclusionAlpha?: Buffer | null; imageBase64: string; }) { - const { box, imageBase64 } = params; - const subjectBuffer = await sharp(Buffer.from(imageBase64, "base64"), { + const { box, exclusionAlpha, imageBase64 } = params; + const rawSubjectBuffer = await sharp(Buffer.from(imageBase64, "base64"), { failOn: "none", }) .rotate() @@ -74,10 +78,30 @@ async function normalizeSubjectBuffer(params: { .ensureAlpha() .png({ compressionLevel: 6 }) .toBuffer(); - const alpha = await sharp(subjectBuffer, { failOn: "none" }) + const rawAlpha = await sharp(rawSubjectBuffer, { failOn: "none" }) .extractChannel(3) .raw() .toBuffer(); + const alpha = Buffer.from(rawAlpha); + if (exclusionAlpha?.length === rawAlpha.length) { + let productPixelCount = 0; + let overlapPixelCount = 0; + for (let index = 0; index < alpha.length; index += 1) { + if (alpha[index] >= 18) productPixelCount += 1; + if (alpha[index] >= 18 && exclusionAlpha[index] >= 8) { + overlapPixelCount += 1; + } + } + const overlapLimit = Math.max(96, Math.round(productPixelCount * 0.004)); + if (overlapPixelCount >= overlapLimit) { + throw new Error( + "目标产品被人物遮挡,暂无法在保持产品完整的前提下移动,请选择未被人物遮挡的完整产品" + ); + } + for (let index = 0; index < alpha.length; index += 1) { + if (exclusionAlpha[index] >= 8) alpha[index] = 0; + } + } const visiblePixels = alpha.reduce( (count, value) => count + (value >= 18 ? 1 : 0), 0 @@ -85,20 +109,52 @@ async function normalizeSubjectBuffer(params: { if (visiblePixels < Math.max(32, box.width * box.height * 0.008)) { throw new Error("比例调整无法识别完整产品,请重新完整框选产品"); } + const subjectBuffer = exclusionAlpha + ? await sharp( + await sharp(rawSubjectBuffer, { failOn: "none" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + { + raw: { channels: 3, height: box.height, width: box.width }, + } + ) + .joinChannel(alpha, { + raw: { channels: 1, height: box.height, width: box.width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer() + : rawSubjectBuffer; return { alpha, subjectBuffer }; } async function buildExpandedMaskPiece(params: { alpha: Buffer; box: PixelBox; + exclusionAlpha?: Buffer | null; }) { - const { alpha, box } = params; + const { alpha, box, exclusionAlpha } = params; const radius = Math.max(2, Math.round(Math.min(box.width, box.height) * 0.06)); - const expandedAlpha = await sharp(alpha, { + const blurredAlphaRaw = await sharp(alpha, { raw: { channels: 1, height: box.height, width: box.width }, }) .blur(radius) - .threshold(8) + .extractChannel(0) + .raw() + .toBuffer(); + const expandedAlphaRaw = Buffer.from(blurredAlphaRaw); + for (let index = 0; index < expandedAlphaRaw.length; index += 1) { + expandedAlphaRaw[index] = expandedAlphaRaw[index] >= 8 ? 255 : 0; + } + if (exclusionAlpha?.length === expandedAlphaRaw.length) { + for (let index = 0; index < expandedAlphaRaw.length; index += 1) { + if (exclusionAlpha[index] >= 8) expandedAlphaRaw[index] = 0; + } + } + const expandedAlpha = await sharp(expandedAlphaRaw, { + raw: { channels: 1, height: box.height, width: box.width }, + }) .png({ compressionLevel: 6 }) .toBuffer(); const whiteRgb = await sharp({ @@ -114,7 +170,99 @@ async function buildExpandedMaskPiece(params: { return sharp(whiteRgb).joinChannel(expandedAlpha).png({ compressionLevel: 6 }).toBuffer(); } +async function buildSubjectMaskPiece(params: { alpha: Buffer; box: PixelBox }) { + const alphaPng = await sharp(params.alpha, { + raw: { channels: 1, height: params.box.height, width: params.box.width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const whiteRgb = await sharp({ + create: { + background: { b: 255, g: 255, r: 255 }, + channels: 3, + height: params.box.height, + width: params.box.width, + }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + return sharp(whiteRgb) + .joinChannel(alphaPng) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + +async function buildPersonExclusionAlpha(params: { + sourceBuffer: Buffer; + sourceHeight: number; + sourceImageUrl: string; + sourceWidth: number; + userId: string; +}) { + try { + const person = await executeRmbgJob({ + foregroundType: "person", + imageBuffer: params.sourceBuffer, + prompt: "比例调整人物保护蒙版", + skipGallerySave: true, + sourceImageUrl: params.sourceImageUrl, + userId: params.userId, + }); + const alpha = await sharp(Buffer.from(person.image_base64, "base64"), { + failOn: "none", + }) + .rotate() + .resize(params.sourceWidth, params.sourceHeight, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .raw() + .toBuffer(); + const visiblePixels = alpha.reduce( + (count, value) => count + (value >= 18 ? 1 : 0), + 0 + ); + if (visiblePixels < Math.max(32, params.sourceWidth * params.sourceHeight * 0.001)) { + return null; + } + return sharp(alpha, { + raw: { + channels: 1, + height: params.sourceHeight, + width: params.sourceWidth, + }, + }) + .blur(Math.max(1, Math.min(params.sourceWidth, params.sourceHeight) * 0.002)) + .threshold(8) + .extractChannel(0) + .raw() + .toBuffer(); + } catch { + return null; + } +} + +async function extractAlphaRegion(params: { + alpha: Buffer | null; + box: PixelBox; + sourceHeight: number; + sourceWidth: number; +}) { + if (!params.alpha) return null; + return sharp(params.alpha, { + raw: { + channels: 1, + height: params.sourceHeight, + width: params.sourceWidth, + }, + }) + .extract(params.box) + .extractChannel(0) + .raw() + .toBuffer(); +} + export async function buildScaleAdjustProductLockAssets(params: { + protectPeople?: boolean; regions: CanvasScaleAdjustIntentRegion[]; sourceBuffer: Buffer; sourceHeight: number; @@ -132,7 +280,17 @@ export async function buildScaleAdjustProductLockAssets(params: { } = params; const overlayInputs: sharp.OverlayOptions[] = []; const maskInputs: sharp.OverlayOptions[] = []; + const sourceSubjectMaskInputs: sharp.OverlayOptions[] = []; const subjectReferenceUrls: string[] = []; + const personExclusionAlpha = params.protectPeople + ? await buildPersonExclusionAlpha({ + sourceBuffer, + sourceHeight, + sourceImageUrl, + sourceWidth, + userId, + }) + : null; for (const region of regions) { const sourceBox = rectToPixelBox( @@ -156,8 +314,15 @@ export async function buildScaleAdjustProductLockAssets(params: { sourceImageUrl, userId, }); + const regionPersonExclusionAlpha = await extractAlphaRegion({ + alpha: personExclusionAlpha, + box: sourceBox, + sourceHeight, + sourceWidth, + }); const { alpha, subjectBuffer } = await normalizeSubjectBuffer({ box: sourceBox, + exclusionAlpha: regionPersonExclusionAlpha, imageBase64: subject.image_base64, }); const scaledSubjectBuffer = await sharp(subjectBuffer, { failOn: "none" }) @@ -173,7 +338,16 @@ export async function buildScaleAdjustProductLockAssets(params: { top: targetBox.top, }); maskInputs.push({ - input: await buildExpandedMaskPiece({ alpha, box: sourceBox }), + input: await buildExpandedMaskPiece({ + alpha, + box: sourceBox, + exclusionAlpha: regionPersonExclusionAlpha, + }), + left: sourceBox.left, + top: sourceBox.top, + }); + sourceSubjectMaskInputs.push({ + input: await buildSubjectMaskPiece({ alpha, box: sourceBox }), left: sourceBox.left, top: sourceBox.top, }); @@ -188,7 +362,8 @@ export async function buildScaleAdjustProductLockAssets(params: { width: sourceWidth, }, }; - const [modelMaskBuffer, overlayBuffer, repairMaskBuffer] = await Promise.all([ + const [modelMaskBuffer, overlayBuffer, repairMaskBuffer, sourceSubjectMaskBuffer] = + await Promise.all([ sharp({ create: { background: { b: 0, g: 0, r: 0 }, @@ -209,6 +384,10 @@ export async function buildScaleAdjustProductLockAssets(params: { .composite(maskInputs) .png({ compressionLevel: 6 }) .toBuffer(), + sharp(transparentCanvas) + .composite(sourceSubjectMaskInputs) + .png({ compressionLevel: 6 }) + .toBuffer(), ]); const sourceAlpha = await sharp(sourceBuffer, { failOn: "none" }) .rotate() @@ -223,6 +402,13 @@ export async function buildScaleAdjustProductLockAssets(params: { ); const hasTransparentBackground = transparentPixelCount >= sourceWidth * sourceHeight * 0.05; + const personExclusionMaskBuffer = personExclusionAlpha + ? await sharp(personExclusionAlpha, { + raw: { channels: 1, height: sourceHeight, width: sourceWidth }, + }) + .png({ compressionLevel: 6 }) + .toBuffer() + : undefined; const backgroundPlateBuffer = hasTransparentBackground ? await sharp(transparentCanvas) .png({ compressionLevel: 6 }) @@ -233,7 +419,10 @@ export async function buildScaleAdjustProductLockAssets(params: { ...(backgroundPlateBuffer ? { backgroundPlateBuffer } : {}), modelMaskBuffer, overlayBuffer, + personExclusionApplied: Boolean(personExclusionAlpha), + ...(personExclusionMaskBuffer ? { personExclusionMaskBuffer } : {}), repairMaskBuffer, + sourceSubjectMaskBuffer, subjectReferenceUrls, }; } diff --git a/scripts/verify-canvas-scale-adjust.ts b/scripts/verify-canvas-scale-adjust.ts index 3baf26ebc..e2bfc9631 100644 --- a/scripts/verify-canvas-scale-adjust.ts +++ b/scripts/verify-canvas-scale-adjust.ts @@ -13,9 +13,11 @@ import { type CanvasScaleAdjustIntentRegion, type CanvasScaleAdjustRect, } from "@/lib/canvas/scale-adjust-product-lock"; +import { applyScaleAdjustModelRelight } from "@/lib/canvas/scale-adjust-model-relight"; import { createStorageUploadSession } from "@/lib/storage/service"; type VerificationCase = { + expectedError?: string; id: string; rect: CanvasScaleAdjustRect; scalePercent?: number; @@ -84,6 +86,31 @@ const cases: VerificationCase[] = [ scalePercent: -30, source: "https://scqny.jiansun.vip/uploads/1783555294804-qdykhy.png", }, + { + id: "11-office-chair-move-right", + rect: { height: 0.72, width: 0.64, x: 0.18, y: 0.25 }, + source: "https://scqny.jiansun.vip/uploads/1783562931574-tj4zst.png", + target: { height: 0.72, width: 0.64, x: 0.31, y: 0.25 }, + }, + { + id: "12-bookshelf-person-move-shrink", + expectedError: "目标产品被人物遮挡", + rect: { height: 0.88, width: 0.29, x: 0.37, y: 0.07 }, + source: "https://scqny.jiansun.vip/uploads/1783687809170-vmnsea.png", + target: { height: 0.704, width: 0.232, x: 0.54, y: 0.246 }, + }, + { + id: "13-coffee-cart-move-left", + rect: { height: 0.54, width: 0.5, x: 0.22, y: 0.36 }, + source: "https://scqny.jiansun.vip/uploads/1783555598917-8w4a6t.png", + target: { height: 0.54, width: 0.5, x: 0.05, y: 0.36 }, + }, + { + id: "14-side-table-person-scene-move-left", + rect: { height: 0.235, width: 0.225, x: 0.77, y: 0.72 }, + source: "https://scqny.jiansun.vip/uploads/1783927690517-8uhlqu.jpg", + target: { height: 0.235, width: 0.225, x: 0.55, y: 0.72 }, + }, ]; const aspectRatios = ["1:1", "4:3", "3:4", "16:9", "9:16"] as const; @@ -176,6 +203,10 @@ async function runCase(params: { resolve(outputDir, `${item.id}-repair-mask.png`), assets.repairMaskBuffer ), + writeFile( + resolve(outputDir, `${item.id}-source-subject-mask.png`), + assets.sourceSubjectMaskBuffer + ), ...(assets.backgroundPlateBuffer ? [ writeFile( @@ -184,16 +215,44 @@ async function runCase(params: { ), ] : []), + ...(assets.personExclusionMaskBuffer + ? [ + writeFile( + resolve(outputDir, `${item.id}-person-exclusion-mask.png`), + assets.personExclusionMaskBuffer + ), + ] + : []), ]); + if (process.env.SCALE_ADJUST_ASSETS_ONLY === "1") { + const result = { + elapsedMs: Date.now() - startedAt, + id: item.id, + mode: "assets-only", + ok: true, + personExclusionApplied: assets.personExclusionApplied, + rect: item.rect, + source: item.source, + targetRect: regions[0].targetRect, + }; + console.log(`SCALE_VERIFY_CASE ${JSON.stringify(result)}`); + return result; + } + const suffix = `${item.id}-${Date.now()}`; - const [modelMaskUpload, overlayUpload, repairMaskUpload] = await Promise.all([ + const [modelMaskUpload, overlayUpload, repairMaskUpload, sourceSubjectMaskUpload] = + await Promise.all([ upload(`verify-scale-mask-${suffix}.png`, toArrayBuffer(assets.modelMaskBuffer)), upload(`verify-scale-overlay-${suffix}.png`, toArrayBuffer(assets.overlayBuffer)), upload( `verify-scale-repair-mask-${suffix}.png`, toArrayBuffer(assets.repairMaskBuffer) ), + upload( + `verify-scale-subject-mask-${suffix}.png`, + toArrayBuffer(assets.sourceSubjectMaskBuffer) + ), ]); const backgroundPlateUpload = assets.backgroundPlateBuffer ? await upload( @@ -201,21 +260,43 @@ async function runCase(params: { toArrayBuffer(assets.backgroundPlateBuffer) ) : null; - const providerImageParams = { - objectRemovalMaskImageUrl: modelMaskUpload.url, - objectRemovalMode: "premium", - }; + const hasPositionMove = regions.some( + (region) => + Math.abs(region.targetRect.x - region.rect.x) >= 0.005 || + Math.abs(region.targetRect.y - region.rect.y) >= 0.005 + ); + const providerImageParams = hasPositionMove + ? { maskImageUrl: modelMaskUpload.url, quality: "medium" } + : { + objectRemovalMaskImageUrl: modelMaskUpload.url, + objectRemovalMode: "premium", + }; const productLockComposite = { ...(backgroundPlateUpload ? { backgroundPlateImageUrl: backgroundPlateUpload.url } : {}), overlayImageUrl: overlayUpload.url, + ...(hasPositionMove + ? { + repairMaskImageUrl: repairMaskUpload.url, + sourceImageUrl: item.source, + } + : {}), + regions: regions.map((region) => ({ + source: region.rect, + target: region.targetRect, + })), + shadowSourceImageUrl: item.source, + sourceSubjectMaskImageUrl: sourceSubjectMaskUpload.url, sourceHeight, sourceWidth, }; const verification = { allowOutsideChangedWhenChangedInside: false, - oldFootprintMaskImageUrl: repairMaskUpload.url, + deterministicProductOverlayApplied: true, + oldFootprintMaskImageUrl: hasPositionMove + ? sourceSubjectMaskUpload.url + : repairMaskUpload.url, regions: [item.rect], rejectFailedCandidates: true, rejectNoEffectiveChangeOnly: false, @@ -223,14 +304,20 @@ async function runCase(params: { sourceImageUrl: item.source, targetRegions: [regions[0].targetRect], }; - const prompt = "移除掩膜中的完整产品并自然还原背景。"; + const prompt = hasPositionMove + ? "只编辑白色掩膜区域,完整移除旧位置的产品及其原有阴影,并根据周围墙面、地面、家具和光照连续自然地重建被遮挡背景。黑色区域保持不变,不要保留产品的任何部件或残影。" + : "移除掩膜中的完整产品并自然还原背景。"; const request: CanvasGenerateImageRequest = { allowFailover: false, - aspectRatio: resolveAspectRatio(sourceWidth, sourceHeight), + aspectRatio: hasPositionMove + ? "auto" + : resolveAspectRatio(sourceWidth, sourceHeight), debugLabel: `verify-scale-final-${item.id}`, debugMeta: { operation: "scale-adjust", - scaleAdjustPromptStrategy: "finegrain-erase-product-lock-v1", + scaleAdjustPromptStrategy: hasPositionMove + ? "gpt-mask-background-rebuild-product-lock-v1" + : "finegrain-erase-product-lock-v1", scaleAdjustSequentialCleanupRetry: true, }, displayPrompt: "比例调整", @@ -257,10 +344,28 @@ async function runCase(params: { }, }, ], + scaleAdjustModelRelight: { + model: + process.env.SCALE_ADJUST_MODEL || + "e03d3341-cecd-425e-b88b-9350cbbce18f", + overlayImageUrl: overlayUpload.url, + quality: "medium", + regions: regions.map((region) => ({ target: region.targetRect })), + sourceHeight, + sourceWidth, + }, scaleAdjustProductLockComposite: productLockComposite, scaleAdjustVerification: verification, }; - const generated = await generateCanvasImage(request); + const initialGenerated = await generateCanvasImage(request); + const generated = await applyScaleAdjustModelRelight({ + generateImage: generateCanvasImage, + initialResult: initialGenerated, + ownerKey: + process.env.SCALE_ADJUST_USER_ID || + "00000000-0000-0000-0000-000000000000", + request, + }); const resultUrl = generated.images[0]?.originalUrl || null; if (!resultUrl) throw new Error("比例调整未返回结果图片"); await writeFile( @@ -272,6 +377,7 @@ async function runCase(params: { elapsedMs: Date.now() - startedAt, id: item.id, ok: true, + personExclusionApplied: assets.personExclusionApplied, rect: item.rect, resultUrl, source: item.source, @@ -312,8 +418,21 @@ async function main() { try { results.push(await runCase({ item, outputDir, upload })); } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + if (item.expectedError && errorMessage.includes(item.expectedError)) { + const expectedFailure = { + error: errorMessage, + expectedFailure: true, + id: item.id, + ok: true, + source: item.source, + }; + results.push(expectedFailure); + console.log(`SCALE_VERIFY_CASE ${JSON.stringify(expectedFailure)}`); + continue; + } const failure = { - error: error instanceof Error ? error.message : String(error), + error: errorMessage, id: item.id, ok: false, source: item.source, diff --git a/tests/routes/canvas-scale-adjust-intent.test.ts b/tests/routes/canvas-scale-adjust-intent.test.ts index 54be5193e..e41319192 100644 --- a/tests/routes/canvas-scale-adjust-intent.test.ts +++ b/tests/routes/canvas-scale-adjust-intent.test.ts @@ -2,20 +2,62 @@ import { expect, test } from "@playwright/test"; import { readFileSync } from "node:fs"; import { resolve } from "node:path"; +import { hasScaleAdjustTargetChanged } from "../../components/canvas/nodes/image-node/image-node-local-edit"; + function readSource(path: string) { return readFileSync(resolve(process.cwd(), path), "utf8"); } test.describe("canvas scale adjust product lock", () => { + test("accepts target movement even when target size stays unchanged", () => { + const source = { height: 0.4, width: 0.3, x: 0.2, y: 0.25 }; + + expect(hasScaleAdjustTargetChanged(source, source)).toBe(false); + expect( + hasScaleAdjustTargetChanged(source, { ...source, x: 0.46, y: 0.12 }) + ).toBe(true); + expect( + hasScaleAdjustTargetChanged(source, { + ...source, + height: 0.32, + width: 0.24, + }) + ).toBe(true); + }); + + test("uses the same movement validation in canvas and detail result flows", () => { + const canvasNode = readSource("components/canvas/nodes/ImageNode.tsx"); + const detailWorkspace = readSource( + "components/agents/AgentGenerationWorkspaceDarkV2.tsx" + ); + + expect(canvasNode).toContain("hasScaleAdjustTargetChanged("); + expect(detailWorkspace).toContain("hasScaleAdjustTargetChanged("); + expect(canvasNode).toContain("青色目标框大小或位置"); + expect(detailWorkspace).toContain("目标框大小和位置均未变化"); + }); + test("uses one dedicated erase step and exact product overlay", () => { const route = readSource("app/api/canvas/local-edit/route.ts"); expect(route).toContain("buildScaleAdjustProductLockAssets"); expect(route).toContain('objectRemovalMode: "premium"'); expect(route).toContain("objectRemovalMaskImageUrl: modelMaskUpload.url"); + expect(route).toContain("maskImageUrl: modelMaskUpload.url"); + expect(route).toContain("hasPositionMove"); + expect(route).toContain('aspectRatio: hasPositionMove ? "auto" : resolvedAspectRatio'); expect(route).toContain("overlayImageUrl: overlayUpload.url"); - expect(route).toContain("oldFootprintMaskImageUrl: repairMaskUpload.url"); - expect(route).toContain('scaleAdjustPromptStrategy: "finegrain-erase-product-lock-v1"'); + expect(route).toContain("repairMaskImageUrl: repairMaskUpload.url"); + expect(route).toContain("sourceImageUrl"); + expect(route).toContain("protectPeople: hasPositionMove"); + expect(route).toContain("oldFootprintMaskImageUrl: hasPositionMove"); + expect(route).toContain("? sourceSubjectMaskUpload?.url"); + expect(route).toContain("deterministicProductOverlayApplied: true"); + expect(route).toContain("scaleAdjustPersonExclusionApplied:"); + expect(route).toContain("sourceSubjectMaskImageUrl: sourceSubjectMaskUpload.url"); + expect(route).toContain("shadowSourceImageUrl: sourceImageUrl"); + expect(route).toContain('"finegrain-erase-product-lock-v1"'); + expect(route).toContain('"gpt-mask-background-rebuild-product-lock-v1"'); expect(route).not.toContain("buildScaleAdjustVisualGuide"); expect(route).not.toContain("source-plus-guide-v1"); }); @@ -52,6 +94,57 @@ test.describe("canvas scale adjust product lock", () => { expect(productLock).toContain("backgroundPlateBuffer"); }); + test("protects people and rejects products materially occluded by them", () => { + const productLock = readSource("lib/canvas/scale-adjust-product-lock.ts"); + const rmbg = readSource("lib/ai/rmbg/service.ts"); + + expect(productLock).toContain('foregroundType: "person"'); + expect(productLock).toContain("regionPersonExclusionAlpha"); + expect(productLock).toContain("overlapPixelCount >= overlapLimit"); + expect(productLock).toContain("目标产品被人物遮挡"); + expect(productLock).toContain("personExclusionApplied: Boolean(personExclusionAlpha)"); + expect(productLock).toContain("expandedAlphaRaw[index] >= 8 ? 255 : 0"); + expect(productLock).toContain(".threshold(8)\n .extractChannel(0)"); + expect(rmbg).toContain("foregroundType: foregroundType ??"); + }); + + test("moves the original contact shadow with the exact product overlay", () => { + const generator = readSource("lib/canvas/generate-image.ts"); + + expect(generator).toContain("buildScaleAdjustTransferredShadow"); + expect(generator).toContain("cleanedLuma - sourceLuma - 4"); + expect(generator).toContain("transferredShadowBuffer"); + expect(generator).toContain("transferredShadowCount"); + expect(generator).toContain("if (!(isShrink || isMove)) continue"); + expect(generator).toContain( + "const maskHit = explicitMaskRaw\n ? explicitMaskValue > 18" + ); + expect(generator).toContain("(!isMove &&"); + expect(generator).toContain( + "const generatedWidth = compositeBaseBuffer ? overlayWidth : modelWidth" + ); + expect(generator).toContain( + ".extractChannel(3)\n .blur(Math.max(4" + ); + expect(generator).toContain(".extractChannel(0)\n .raw()\n .toBuffer()"); + }); + + test("uses the image edit model to rebuild lighting at the target position", () => { + const route = readSource("app/api/canvas/local-edit/route.ts"); + const queue = readSource("lib/canvas/image-task-queue.ts"); + const relight = readSource("lib/canvas/scale-adjust-model-relight.ts"); + + expect(route).toContain("scaleAdjustModelRelight:"); + expect(route).toContain("...(hasPositionMove"); + expect(route).toContain("overlayImageUrl: overlayUpload.url"); + expect(queue).toContain("applyScaleAdjustModelRelight"); + expect(relight).toContain('debugLabel: "canvas-scale-adjust-model-relight"'); + expect(relight).toContain("maskImageUrl: maskUpload.url"); + expect(relight).toContain("生成真实自然的接触阴影与投射阴影"); + expect(relight).toContain("findMedianLumaDelta"); + expect(relight).toContain("productAlpha[pixelIndex] >= 8 ? 0"); + }); + test("keeps non-pro detail result scale adjust on the shared backend flow", () => { const source = readSource( "components/agents/AgentGenerationWorkspaceDarkV2.tsx"